@oneuptime/common 11.5.4 → 11.5.6

package/Server/Infrastructure/Queue.ts

@@ -217,6 +217,23 @@ export default class Queue {
     }
   }
 
+  /**
+   * Removes a one-off job by id, and also removes a repeatable whose repeat
+   * KEY equals `jobId` (QueueWorkflow persists `job.repeatJobKey` on the
+   * Workflow row and passes it back here).
+   *
+   * Note the asymmetry between the two removals, which is easy to get wrong:
+   * BullMQ rejects custom job ids containing ":", so the job lookup must use
+   * the sanitized id. But removeRepeatableByKey() expects the repeat KEY
+   * exactly as it appears in the `bull:<queue>:repeat` zset (an md5 hash, or a
+   * legacy "name:id:endDate:tz:pattern" string) — it ZREMs that exact member.
+   * Sanitizing would mangle the legacy colon form into a member that does not
+   * exist, and the removal would silently no-op. So the raw value is passed
+   * through.
+   *
+   * To remove a repeatable by its job NAME (e.g. to retire a renamed cron),
+   * use removeRepeatableByName() — passing a job name here does nothing.
+   */
   @CaptureSpan()
   public static async removeJob(
     queueName: QueueName,
@@ -226,17 +243,78 @@ export default class Queue {
       return;
     }
 
+    const queue: BullQueue = this.getQueue(queueName);
+
     const sanitizedJobId: string = this.sanitizeJobId(jobId.toString());
 
-    const job: Job | undefined =
-      await this.getQueue(queueName).getJob(sanitizedJobId);
+    const job: Job | undefined = await queue.getJob(sanitizedJobId);
 
     if (job) {
       await job.remove();
     }
 
-    // remove existing repeatable job
-    await this.getQueue(queueName).removeRepeatableByKey(sanitizedJobId);
+    /*
+     * Remove an existing repeatable keyed by this value. Raw, not sanitized —
+     * see the note above.
+     */
+    await queue.removeRepeatableByKey(jobId.toString());
+  }
+
+  /**
+   * Removes every repeatable job DEFINITION on `queueName` whose job NAME is
+   * `jobName`, and returns how many were removed. Idempotent: removing a name
+   * that isn't registered is a no-op that returns 0.
+   *
+   * Why this cannot be done with removeJob(): BullMQ keys a repeatable by an
+   * opaque `key` (the member of the `bull:<queue>:repeat` zset — an md5 hash),
+   * never by the job's name. removeRepeatableByKey() ZREMs that exact member,
+   * so handing it a job name (or a job id) matches nothing and silently
+   * succeeds as a no-op. BullMQ's own docs say so:
+   *
+   *   "Removes a repeatable job by its key. Note that the key is the one used
+   *    to store the repeatable job metadata and not one of the job iterations
+   *    themselves. You can use "getRepeatableJobs" in order to get the keys."
+   *
+   * So the only supported name -> key path is to enumerate getRepeatableJobs()
+   * and match on `.name`. Removing by key also deletes the already-materialized
+   * next delayed iteration, so the job stops firing immediately.
+   *
+   * This exists to retire a RENAMED cron: RunCron registers a repeatable keyed
+   * by the new name, and addJob() above only clears a pre-existing repeatable
+   * that matches the NEW name — nothing clears the old one, so a renamed cron
+   * leaves its old definition behind in Redis, firing forever.
+   */
+  @CaptureSpan()
+  public static async removeRepeatableByName(
+    queueName: QueueName,
+    jobName: string,
+  ): Promise<number> {
+    if (!jobName) {
+      return 0;
+    }
+
+    const queue: BullQueue = this.getQueue(queueName);
+
+    const repeatableJobs: RepeatableJob[] = await queue.getRepeatableJobs();
+
+    let removedCount: number = 0;
+
+    for (const repeatableJob of repeatableJobs) {
+      if (repeatableJob.name !== jobName) {
+        continue;
+      }
+
+      // `.key` is the zset member BullMQ expects here — never the name or id.
+      const isRemoved: boolean = await queue.removeRepeatableByKey(
+        repeatableJob.key,
+      );
+
+      if (isRemoved) {
+        removedCount++;
+      }
+    }
+
+    return removedCount;
   }
 
   @CaptureSpan()

package/Server/Infrastructure/Semaphore.ts

@@ -1,8 +1,13 @@
 import Redis, { ClientType } from "./Redis";
-import { Mutex, LockOptions } from "redis-semaphore";
+import {
+  Mutex,
+  LockOptions,
+  Semaphore as RedisSemaphore,
+} from "redis-semaphore";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 
 export type SemaphoreMutex = Mutex;
+export type SemaphorePermit = RedisSemaphore;
 
 export default class Semaphore {
   // returns the mutex id
@@ -60,4 +65,60 @@ export default class Semaphore {
   public static async release(mutex: SemaphoreMutex): Promise<void> {
     await mutex.release();
   }
+
+  /**
+   * Acquire one permit from a distributed, fixed-size concurrency pool.
+   * Unlike lock(), up to `limit` callers may hold the key simultaneously.
+   */
+  @CaptureSpan()
+  public static async acquirePermit(data: {
+    key: string;
+    namespace: string;
+    limit: number;
+    lockTimeout?: number | undefined;
+    acquireTimeout?: number | undefined;
+    acquireAttemptsLimit?: number | undefined;
+    retryInterval?: number | undefined;
+  }): Promise<SemaphorePermit> {
+    if (!Number.isInteger(data.limit) || data.limit <= 0) {
+      throw new Error("Semaphore permit limit must be a positive integer");
+    }
+
+    const client: ClientType | null = Redis.getClient();
+
+    if (!client) {
+      throw new Error("Redis client is not connected");
+    }
+
+    const lockOptions: LockOptions = {
+      lockTimeout: data.lockTimeout || 5000,
+    };
+
+    if (data.acquireTimeout !== undefined) {
+      lockOptions.acquireTimeout = data.acquireTimeout;
+    }
+
+    if (data.acquireAttemptsLimit !== undefined) {
+      lockOptions.acquireAttemptsLimit = data.acquireAttemptsLimit;
+    }
+
+    if (data.retryInterval !== undefined) {
+      lockOptions.retryInterval = data.retryInterval;
+    }
+
+    const permit: SemaphorePermit = new RedisSemaphore(
+      client,
+      `${data.namespace}-${data.key}`,
+      data.limit,
+      lockOptions,
+    );
+
+    await permit.acquire();
+    return permit;
+  }
+
+  @CaptureSpan()
+  public static async releasePermit(permit: SemaphorePermit): Promise<void> {
+    await permit.release();
+  }
 }

package/Server/Middleware/MasterAdminAuthorization.ts

@@ -1,4 +1,5 @@
 import UserMiddleware from "./UserAuthorization";
+import ProjectMiddleware from "./ProjectAuthorization";
 import JSONWebToken from "../Utils/JsonWebToken";
 import Response from "../Utils/Response";
 import {
@@ -8,6 +9,7 @@ import {
 } from "../Utils/Express";
 import NotAuthorizedException from "../../Types/Exception/NotAuthorizedException";
 import JSONWebTokenData from "../../Types/JsonWebTokenData";
+import ObjectID from "../../Types/ObjectID";
 
 export default class MasterAdminAuthorization {
   public static async isAuthorizedMasterAdminMiddleware(
@@ -52,4 +54,39 @@ export default class MasterAdminAuthorization {
       );
     }
   }
+
+  /*
+   * Same as isAuthorizedMasterAdminMiddleware, but ALSO accepts the instance-wide
+   * master API key (Admin Dashboard → Settings → API Key) supplied in the
+   * `apikey` header. The master key has root/master-admin access, so this lets
+   * automated callers reach the master-admin instance-health endpoints with the
+   * key instead of a logged-in master-admin session.
+   *
+   * Deliberately scoped to the read-only health / diagnostics routes only. It is
+   * NOT used on higher-risk master-admin actions (e.g. the read/write query
+   * console or broadcast email), which stay on the JWT-only middleware above so
+   * that a leaked static key cannot trigger them headlessly.
+   */
+  public static async isAuthorizedMasterAdminOrMasterApiKeyMiddleware(
+    req: ExpressRequest,
+    res: ExpressResponse,
+    next: NextFunction,
+  ): Promise<void> {
+    try {
+      const apiKey: ObjectID | null = ProjectMiddleware.getApiKey(req);
+
+      if (apiKey && (await ProjectMiddleware.isMasterApiKey(apiKey))) {
+        next();
+        return;
+      }
+    } catch {
+      // Fall through to the master-admin session (JWT) check below.
+    }
+
+    return MasterAdminAuthorization.isAuthorizedMasterAdminMiddleware(
+      req,
+      res,
+      next,
+    );
+  }
 }

package/Server/Middleware/ProjectAuthorization.ts

@@ -60,6 +60,43 @@ export default class ProjectMiddleware {
     return Boolean(this.getProjectId(req));
   }
 
+  /*
+   * Whether the given key is the instance-wide master API key
+   * (Admin Dashboard → Settings → API Key) and that key is currently enabled.
+   * The master key has root/master-admin access, so this is shared with
+   * MasterAdminAuthorization to let the key reach master-admin-only endpoints
+   * (e.g. instance health) as well.
+   */
+  @CaptureSpan()
+  public static async isMasterApiKey(apiKey: ObjectID): Promise<boolean> {
+    /*
+     * masterApiKey is a Postgres `uuid` column, so a non-UUID header value would
+     * make the lookup raise an "invalid input syntax for type uuid" error on
+     * every request. Reject it cleanly up front — a malformed key is never the
+     * master key anyway — so callers can safely fall through to other auth.
+     */
+    if (!ObjectID.isValidUUID(apiKey.toString())) {
+      return false;
+    }
+
+    const masterKeyGlobalConfig: GlobalConfig | null =
+      await GlobalConfigService.findOneBy({
+        query: {
+          _id: ObjectID.getZeroObjectID().toString(),
+          isMasterApiKeyEnabled: true,
+          masterApiKey: apiKey,
+        },
+        props: {
+          isRoot: true,
+        },
+        select: {
+          _id: true,
+        },
+      });
+
+    return Boolean(masterKeyGlobalConfig);
+  }
+
   @CaptureSpan()
   public static async isValidProjectIdAndApiKeyMiddleware(
     req: ExpressRequest,
@@ -124,22 +161,10 @@ export default class ProjectMiddleware {
 
       if (!apiKeyRow) {
         // check master key.
-        const masterKeyGlobalConfig: GlobalConfig | null =
-          await GlobalConfigService.findOneBy({
-            query: {
-              _id: ObjectID.getZeroObjectID().toString(),
-              isMasterApiKeyEnabled: true,
-              masterApiKey: apiKey,
-            },
-            props: {
-              isRoot: true,
-            },
-            select: {
-              _id: true,
-            },
-          });
+        const isMasterApiKey: boolean =
+          await ProjectMiddleware.isMasterApiKey(apiKey);
 
-        if (masterKeyGlobalConfig) {
+        if (isMasterApiKey) {
           (req as OneUptimeRequest).userType = UserType.MasterAdmin;
 
           // get master admin user

package/Server/Services/AIAgentService.ts

@@ -571,6 +571,38 @@ export class Service extends DatabaseService<Model> {
 
     return null;
   }
+
+  /*
+   * Like getAIAgentForProject, but only returns an agent that is actually
+   * alive (recent heartbeat or Connected status). Used by readiness checks
+   * and the stuck-task sweeper so a task is never created — or left queued
+   * forever — for a fleet that isn't running.
+   */
+  @CaptureSpan()
+  public async getConnectedAIAgentForProject(
+    projectId: ObjectID,
+  ): Promise<Model | null> {
+    const agent: Model | null = await this.getAIAgentForProject(projectId);
+
+    if (!agent) {
+      return null;
+    }
+
+    return this.isAgentAlive(agent) ? agent : null;
+  }
+
+  public isAgentAlive(agent: Model): boolean {
+    if (agent.connectionStatus === AIAgentConnectionStatus.Connected) {
+      return true;
+    }
+
+    if (!agent.lastAlive) {
+      return false;
+    }
+
+    const fiveMinutesAgo: Date = OneUptimeDate.getSomeMinutesAgo(5);
+    return OneUptimeDate.isAfter(agent.lastAlive, fiveMinutesAgo);
+  }
 }
 
 export default new Service();

package/Server/Services/AIAgentTaskPullRequestService.ts

@@ -1,10 +1,127 @@
 import DatabaseService from "./DatabaseService";
 import Model from "../../Models/DatabaseModels/AIAgentTaskPullRequest";
+import BadDataException from "../../Types/Exception/BadDataException";
+import DatabaseCommonInteractionProps from "../../Types/BaseDatabase/DatabaseCommonInteractionProps";
+import PullRequestState from "../../Types/CodeRepository/PullRequestState";
+import FixPullRequestCiStatus from "../../Types/AI/FixPullRequestCiStatus";
+import PositiveNumber from "../../Types/PositiveNumber";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+
+export interface AIFixOutcomeStats {
+  total: number;
+  open: number;
+  merged: number;
+  closedUnmerged: number;
+  /*
+   * merged / (merged + closedUnmerged), in whole percent. Null until at
+   * least one PR reached a terminal state — a rate over zero outcomes is
+   * noise, not signal.
+   */
+  acceptanceRatePercent: number | null;
+  /*
+   * Merged PRs whose last recorded ciStatus was Green or (for should-fail
+   * regression-test PRs) ExpectedFailureObserved — merged is good,
+   * merged-and-CI-green is better. Per gate G9, ONLY those two conclusions
+   * count: a merged PR with no CI (NoCiConfigured), a never-polled null,
+   * Pending or Red is NOT verified and drags the rate down honestly.
+   */
+  verifiedGreen: number;
+  /*
+   * verifiedGreen / merged, in whole percent. Null until at least one PR
+   * merged — same honesty rule as acceptanceRatePercent.
+   */
+  verifiedGreenRatePercent: number | null;
+}
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
   }
+
+  /*
+   * Outcome counts for the project's AI-authored fix PRs — the G11
+   * precision baseline (states kept current by AIAgent:SyncPullRequestStates).
+   * Queries run with the caller's props so tenant scoping and read
+   * permissions come from the model ACLs.
+   */
+  @CaptureSpan()
+  public async getOutcomeStats(
+    props: DatabaseCommonInteractionProps,
+  ): Promise<AIFixOutcomeStats> {
+    if (!props.tenantId) {
+      throw new BadDataException("Project ID is required");
+    }
+
+    const countForState: (
+      state: PullRequestState,
+    ) => Promise<PositiveNumber> = (
+      state: PullRequestState,
+    ): Promise<PositiveNumber> => {
+      return this.countBy({
+        query: {
+          projectId: props.tenantId!,
+          pullRequestState: state,
+        },
+        props,
+      });
+    };
+
+    /*
+     * CI-verified merged PRs (B4 Tier 1). Only Green and
+     * ExpectedFailureObserved count as verified — never NoCiConfigured,
+     * null, Pending or Red (G9: absence of CI reads as unverified).
+     */
+    const countMergedWithCiStatus: (
+      ciStatus: FixPullRequestCiStatus,
+    ) => Promise<PositiveNumber> = (
+      ciStatus: FixPullRequestCiStatus,
+    ): Promise<PositiveNumber> => {
+      return this.countBy({
+        query: {
+          projectId: props.tenantId!,
+          pullRequestState: PullRequestState.Merged,
+          ciStatus: ciStatus,
+        },
+        props,
+      });
+    };
+
+    const [open, merged, closedUnmerged, mergedCiGreen, mergedCiExpectedFail]: [
+      PositiveNumber,
+      PositiveNumber,
+      PositiveNumber,
+      PositiveNumber,
+      PositiveNumber,
+    ] = await Promise.all([
+      countForState(PullRequestState.Open),
+      countForState(PullRequestState.Merged),
+      countForState(PullRequestState.Closed),
+      countMergedWithCiStatus(FixPullRequestCiStatus.Green),
+      countMergedWithCiStatus(FixPullRequestCiStatus.ExpectedFailureObserved),
+    ]);
+
+    const mergedCount: number = merged.toNumber();
+    const closedCount: number = closedUnmerged.toNumber();
+    const terminalCount: number = mergedCount + closedCount;
+    const verifiedGreenCount: number =
+      mergedCiGreen.toNumber() + mergedCiExpectedFail.toNumber();
+
+    return {
+      total: open.toNumber() + terminalCount,
+      open: open.toNumber(),
+      merged: mergedCount,
+      closedUnmerged: closedCount,
+      acceptanceRatePercent:
+        terminalCount === 0
+          ? null
+          : Math.round((mergedCount / terminalCount) * 100),
+      verifiedGreen: verifiedGreenCount,
+      verifiedGreenRatePercent:
+        mergedCount === 0
+          ? null
+          : Math.round((verifiedGreenCount / mergedCount) * 100),
+    };
+  }
 }
 
 export default new Service();

package/Server/Services/AIInsightService.ts

@@ -0,0 +1,174 @@
+import DatabaseService from "./DatabaseService";
+import AIRunService from "./AIRunService";
+import AIRunEventService from "./AIRunEventService";
+import AIInsight from "../../Models/DatabaseModels/AIInsight";
+import AIRun from "../../Models/DatabaseModels/AIRun";
+import AIRunEvent from "../../Models/DatabaseModels/AIRunEvent";
+import ObjectID from "../../Types/ObjectID";
+import OneUptimeDate from "../../Types/Date";
+import SortOrder from "../../Types/BaseDatabase/SortOrder";
+import BadDataException from "../../Types/Exception/BadDataException";
+import AIInsightStatus from "../../Types/AI/AIInsightStatus";
+import AIInsightHumanVerdict from "../../Types/AI/AIInsightHumanVerdict";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+
+/*
+ * Upper bound on triage-run events returned for one insight — mirrors the
+ * investigation panel's cap (AIInvestigationAPI MAX_EVENTS). The engine's
+ * per-run step budgets keep real trails far below this; the cap only
+ * bounds pathological rows.
+ */
+export const MAX_TRIAGE_RUN_EVENTS: number = 500;
+
+export class Service extends DatabaseService<AIInsight> {
+  public constructor() {
+    super(AIInsight);
+  }
+
+  /*
+   * Human verdict capture — this IS the G11 precision measurement:
+   * confirm/dismiss rates per insight type tell us how precise each
+   * deterministic detector is in the field and gate any future automation.
+   *
+   * Overwriting an existing verdict is deliberate — people change their
+   * minds; the latest verdict wins (per-insight state, not an audit trail).
+   * Dismissed is a human terminal state, so it also closes the insight;
+   * Confirmed leaves the status untouched (the insight stays open until a
+   * human resolves it or a fix lands).
+   *
+   * Callers must have already access-checked the insight under the USER's
+   * permissions — insight rows are server-authored (empty update ACL), so
+   * this writes as root.
+   */
+  @CaptureSpan()
+  public async applyHumanVerdict(data: {
+    insightId: ObjectID;
+    verdict: AIInsightHumanVerdict;
+    byUserId: ObjectID;
+  }): Promise<{ insightId: ObjectID; verdict: AIInsightHumanVerdict }> {
+    await this.updateOneById({
+      id: data.insightId,
+      data: {
+        humanVerdict: data.verdict,
+        humanVerdictAt: OneUptimeDate.getCurrentDate(),
+        humanVerdictByUserId: data.byUserId,
+        ...(data.verdict === AIInsightHumanVerdict.Dismissed
+          ? { status: AIInsightStatus.Dismissed }
+          : {}),
+      },
+      props: { isRoot: true },
+    });
+
+    return { insightId: data.insightId, verdict: data.verdict };
+  }
+
+  /*
+   * Marks the insight handled. Resolving implies the finding was real, so
+   * when no verdict was recorded yet this also stamps Confirmed (+ at/by) —
+   * the G11 measurement should count a silently-fixed insight as a true
+   * positive. An existing verdict (either value) is left untouched:
+   * resolve is a lifecycle action, not a verdict change.
+   *
+   * Callers must have already access-checked the insight under the USER's
+   * permissions — this reads and writes as root.
+   */
+  @CaptureSpan()
+  public async resolveInsight(data: {
+    insightId: ObjectID;
+    byUserId: ObjectID;
+  }): Promise<{ insightId: ObjectID; status: AIInsightStatus }> {
+    const insight: AIInsight | null = await this.findOneById({
+      id: data.insightId,
+      select: { _id: true, humanVerdict: true },
+      props: { isRoot: true },
+    });
+
+    if (!insight || !insight.id) {
+      throw new BadDataException("AI insight not found.");
+    }
+
+    await this.updateOneById({
+      id: insight.id,
+      data: {
+        status: AIInsightStatus.Resolved,
+        ...(insight.humanVerdict
+          ? {}
+          : {
+              humanVerdict: AIInsightHumanVerdict.Confirmed,
+              humanVerdictAt: OneUptimeDate.getCurrentDate(),
+              humanVerdictByUserId: data.byUserId,
+            }),
+      },
+      props: { isRoot: true },
+    });
+
+    return { insightId: insight.id, status: AIInsightStatus.Resolved };
+  }
+
+  /*
+   * Data for the dashboard's live triage panel: the insight's triage AIRun
+   * plus its ordered event trail. Triage runs are system-authored
+   * (userId = null) and therefore hidden by the per-user privacy pin on the
+   * generic AIRun / AIRunEvent CRUD — so, mirroring AIInvestigationAPI's
+   * sendLatestInvestigation, the run and events are read as root AFTER the
+   * caller has access-checked the insight under the USER's permissions.
+   *
+   * A missing insight (raced deletion) or an insight without a triage run
+   * (no LLM provider / no budget headroom at scan time) returns the empty
+   * shape — the panel renders an empty state, not an error.
+   */
+  @CaptureSpan()
+  public async getLatestTriageRunWithEvents(data: {
+    insightId: ObjectID;
+  }): Promise<{ run: AIRun | null; events: Array<AIRunEvent> }> {
+    const insight: AIInsight | null = await this.findOneById({
+      id: data.insightId,
+      select: { _id: true, triageAiRunId: true },
+      props: { isRoot: true },
+    });
+
+    if (!insight || !insight.triageAiRunId) {
+      return { run: null, events: [] };
+    }
+
+    const run: AIRun | null = await AIRunService.findOneById({
+      id: insight.triageAiRunId,
+      select: {
+        _id: true,
+        status: true,
+        startedAt: true,
+        completedAt: true,
+        errorMessage: true,
+        llmCallCount: true,
+        toolCallCount: true,
+        totalTokens: true,
+        createdAt: true,
+      },
+      props: { isRoot: true },
+    });
+
+    if (!run || !run.id) {
+      return { run: null, events: [] };
+    }
+
+    const events: Array<AIRunEvent> = await AIRunEventService.findBy({
+      query: { aiRunId: run.id },
+      select: {
+        _id: true,
+        sequence: true,
+        eventType: true,
+        toolName: true,
+        resultSummary: true,
+        createdAt: true,
+      },
+      sort: { sequence: SortOrder.Ascending },
+      limit: MAX_TRIAGE_RUN_EVENTS,
+      skip: 0,
+      props: { isRoot: true },
+    });
+
+    return { run, events };
+  }
+}
+
+export default new Service();