@oneuptime/common 12.0.3 → 12.0.5

package/Server/Services/AIService.ts

@@ -155,19 +155,32 @@ export const AI_REMEDIATION_EXECUTION_FEATURE: string =
  * (SQL `= ANY`, and an `.includes()` gate on the write path that no writer can
  * trip because no writer emits these strings any more).
  */
+const LEGACY_SENTINEL_INCIDENT_INVESTIGATION_FEATURE: string =
+  "Sentinel Incident Investigation";
+const LEGACY_SENTINEL_ALERT_INVESTIGATION_FEATURE: string =
+  "Sentinel Alert Investigation";
+const LEGACY_SENTINEL_INVESTIGATION_GRADING_FEATURE: string =
+  "Sentinel Investigation Grading";
+const LEGACY_SENTINEL_CONFIDENCE_CLASSIFICATION_FEATURE: string =
+  "Sentinel Confidence Classification";
+const LEGACY_SENTINEL_CODE_FIX_FEATURE: string = "Sentinel Code Fix";
+const LEGACY_SENTINEL_INSIGHT_TRIAGE_FEATURE: string =
+  "Sentinel Insight Triage";
+
 export const LEGACY_AUTONOMOUS_AI_FEATURES: Array<string> = [
-  "Sentinel Incident Investigation",
-  "Sentinel Alert Investigation",
-  "Sentinel Investigation Grading",
-  "Sentinel Confidence Classification",
-  "Sentinel Code Fix",
-  "Sentinel Insight Triage",
+  LEGACY_SENTINEL_INCIDENT_INVESTIGATION_FEATURE,
+  LEGACY_SENTINEL_ALERT_INVESTIGATION_FEATURE,
+  LEGACY_SENTINEL_INVESTIGATION_GRADING_FEATURE,
+  LEGACY_SENTINEL_CONFIDENCE_CLASSIFICATION_FEATURE,
+  LEGACY_SENTINEL_CODE_FIX_FEATURE,
+  LEGACY_SENTINEL_INSIGHT_TRIAGE_FEATURE,
 ];
 
 /*
- * Features that run WITHOUT a human in the loop. The per-project daily token
- * budget (Project.aiDailyAutonomousTokenLimit, G4) applies only to these —
- * interactive chat and explicitly user-triggered AI are never budget-blocked.
+ * Features that run WITHOUT a human in the loop. Incident-linked and
+ * alert-linked calls use their respective daily token limits; subjectless
+ * calls use Project.aiDailyAutonomousTokenLimit. Interactive chat and
+ * explicitly user-triggered AI are never budget-blocked.
  * Auto-postmortem is deliberately excluded for now: it is one call per
  * resolved incident, not storm-shaped; include it when it moves to the queue.
  *
@@ -188,7 +201,7 @@ export const AUTONOMOUS_AI_FEATURES: Array<string> = [
    * user-triggered, but the tool loop then runs unattended for up to ~40
    * calls — storm-shaped enough that the daily budget must cover it. The
    * per-run loop budgets (CodeFixAgentCompletion) cap a single run; this
-   * daily pool caps all of them together.
+   * daily subject lane caps all of them together.
    */
   AI_CODE_FIX_FEATURE,
   /*
@@ -215,7 +228,7 @@ export const AUTONOMOUS_AI_FEATURES: Array<string> = [
   /*
    * Auto-remediation command composition/execution
    * (RemediationExecutionRunner). Same trigger shape as planning, with a
-   * larger per-run tool budget — the daily pool must cover it.
+   * larger per-run tool budget — the daily subject lane must cover it.
    */
   AI_REMEDIATION_EXECUTION_FEATURE,
   /*
@@ -226,6 +239,23 @@ export const AUTONOMOUS_AI_FEATURES: Array<string> = [
   ...LEGACY_AUTONOMOUS_AI_FEATURES,
 ];
 
+/*
+ * Before subject-lane accounting shipped, the main investigation calls did
+ * not persist incidentId/alertId (or aiRunId). Their feature label is the only
+ * durable lane signal left on those rows. Keep these lists so a deployment
+ * does not reset today's incident/alert spend or charge it to background work.
+ */
+const LEGACY_INCIDENT_LANE_FEATURES: Array<string> = [
+  AI_INCIDENT_INVESTIGATION_FEATURE,
+  AI_INVESTIGATION_GRADING_FEATURE,
+  LEGACY_SENTINEL_INCIDENT_INVESTIGATION_FEATURE,
+  LEGACY_SENTINEL_INVESTIGATION_GRADING_FEATURE,
+];
+const LEGACY_ALERT_LANE_FEATURES: Array<string> = [
+  AI_ALERT_INVESTIGATION_FEATURE,
+  LEGACY_SENTINEL_ALERT_INVESTIGATION_FEATURE,
+];
+
 export interface AutonomousBudgetStatus {
   exhausted: boolean;
   // null when the project has no limit configured.
@@ -237,10 +267,10 @@ export interface AILogRequest {
   projectId: ObjectID;
   userId?: ObjectID | undefined;
   feature: string; // e.g., "IncidentPostmortem", "IncidentNote"
-  incidentId?: ObjectID;
-  alertId?: ObjectID;
-  scheduledMaintenanceId?: ObjectID;
-  aiRunId?: ObjectID;
+  incidentId?: ObjectID | undefined;
+  alertId?: ObjectID | undefined;
+  scheduledMaintenanceId?: ObjectID | undefined;
+  aiRunId?: ObjectID | undefined;
   /*
    * When set, use this specific provider (validated against the project) rather
    * than the project default. Powers the in-chat provider/model switcher.
@@ -337,23 +367,40 @@ export class Service extends BaseService {
   }
 
   /*
-   * G4 daily budget: has this project consumed its daily autonomous-token
-   * allowance (UTC day)? Counts only AUTONOMOUS_AI_FEATURES tokens, so chat
-   * usage neither eats the autonomous budget nor is blocked by it.
+   * G4 daily budget: has this project consumed the selected subject lane's
+   * daily autonomous-token allowance (UTC day)? Counts only that lane's
+   * AUTONOMOUS_AI_FEATURES tokens, so chat usage neither eats the autonomous
+   * budget nor is blocked by it.
    */
   @CaptureSpan()
   public async getAutonomousDailyBudgetStatus(
     projectId: ObjectID,
+    subject?: {
+      incidentId?: ObjectID | undefined;
+      alertId?: ObjectID | undefined;
+    },
   ): Promise<AutonomousBudgetStatus> {
+    this.assertSingleSubject(subject);
+
     const project: Project | null = await ProjectService.findOneById({
       id: projectId,
-      select: { aiDailyAutonomousTokenLimit: true },
+      select: {
+        aiDailyAutonomousTokenLimit: true,
+        incidentAiDailyAutonomousTokenLimit: true,
+        alertAiDailyAutonomousTokenLimit: true,
+      },
       props: { isRoot: true },
     });
 
-    const limitInTokens: number | null =
+    let limitInTokens: number | null =
       project?.aiDailyAutonomousTokenLimit ?? null;
 
+    if (subject?.incidentId) {
+      limitInTokens = project?.incidentAiDailyAutonomousTokenLimit ?? null;
+    } else if (subject?.alertId) {
+      limitInTokens = project?.alertAiDailyAutonomousTokenLimit ?? null;
+    }
+
     if (limitInTokens === null) {
       return { exhausted: false, limitInTokens: null, usedTokensToday: 0 };
     }
@@ -374,6 +421,10 @@ export class Service extends BaseService {
           "UTC",
         ),
         features: AUTONOMOUS_AI_FEATURES,
+        incidentId: subject?.incidentId,
+        alertId: subject?.alertId,
+        legacyIncidentFeatures: LEGACY_INCIDENT_LANE_FEATURES,
+        legacyAlertFeatures: LEGACY_ALERT_LANE_FEATURES,
       },
     );
 
@@ -388,6 +439,8 @@ export class Service extends BaseService {
   public async executeWithLogging(
     request: AILogRequest,
   ): Promise<AILogResponse> {
+    this.assertSingleSubject(request);
+
     const startTime: Date = new Date();
 
     // Get LLM provider for the project (honoring an explicit per-chat choice).
@@ -503,10 +556,18 @@ export class Service extends BaseService {
      */
     if (AUTONOMOUS_AI_FEATURES.includes(request.feature)) {
       const budget: AutonomousBudgetStatus =
-        await this.getAutonomousDailyBudgetStatus(request.projectId);
+        await this.getAutonomousDailyBudgetStatus(request.projectId, {
+          incidentId: request.incidentId,
+          alertId: request.alertId,
+        });
 
       if (budget.exhausted) {
-        const budgetMessage: string = `Daily autonomous AI token budget exhausted (${budget.usedTokensToday.toLocaleString()} of ${budget.limitInTokens?.toLocaleString()} tokens used today). Autonomous AI requests resume tomorrow (UTC) — raise or unset the limit in the AI settings pages.`;
+        const settingsLocation: string = request.incidentId
+          ? "Incidents > Settings > AI"
+          : request.alertId
+            ? "Alerts > Settings > AI"
+            : "Project Settings > AI > AI Guardrails";
+        const budgetMessage: string = `Daily autonomous AI token budget exhausted (${budget.usedTokensToday.toLocaleString()} of ${budget.limitInTokens?.toLocaleString()} tokens used today). Autonomous AI requests resume tomorrow (UTC) — raise or unset the limit under ${settingsLocation}.`;
 
         logEntry.status = LlmLogStatus.BudgetExceeded;
         logEntry.statusMessage = budgetMessage.substring(0, 490);
@@ -658,6 +719,17 @@ export class Service extends BaseService {
     }
   }
 
+  private assertSingleSubject(subject?: {
+    incidentId?: ObjectID | undefined;
+    alertId?: ObjectID | undefined;
+  }): void {
+    if (subject?.incidentId && subject.alertId) {
+      throw new BadDataException(
+        "An AI request cannot belong to both an incident and an alert.",
+      );
+    }
+  }
+
   /*
    * Set gen_ai.* attributes (OpenTelemetry GenAI semantic conventions) on the
    * currently-active span. The @CaptureSpan()-wrapped caller owns that span, so

package/Server/Services/AlertFeedService.ts

@@ -84,6 +84,7 @@ export class Service extends DatabaseService<Model> {
     displayColor?: Color | undefined;
     userId?: ObjectID | undefined;
     postedAt?: Date | undefined;
+    aiRunId?: ObjectID | undefined;
     workspaceNotification?:
       | {
           notifyUserId?: ObjectID | undefined; // this is oneuptime user id.
@@ -126,6 +127,10 @@ export class Service extends DatabaseService<Model> {
       alertFeed.alertFeedEventType = data.alertFeedEventType;
       alertFeed.projectId = data.projectId;
 
+      if (data.aiRunId) {
+        alertFeed.aiRunId = data.aiRunId;
+      }
+
       if (!data.postedAt) {
         alertFeed.postedAt = OneUptimeDate.getCurrentDate();
       }

package/Server/Services/AnalyticsDatabaseService.ts

@@ -34,6 +34,7 @@ import {
   TimeoutOverflowMode,
 } from "../Utils/AnalyticsDatabase/QuerySettingsHelper";
 import {
+  getDistributedDdlTaskTimeoutSeconds,
   getStorageTableName,
   onClusterClause,
 } from "../Utils/AnalyticsDatabase/ClusterConfig";
@@ -137,6 +138,34 @@ export const MigrationExecuteOptions: ClickhouseExecuteOptions = {
      * migration pool's 30-minute idle ceiling.
      */
     http_headers_progress_interval_ms: "10000",
+    /*
+     * ON CLUSTER DDL is queued in Keeper and executed by each host's DDLWorker
+     * sequentially, so on a busy or backlogged cluster a host can be healthy
+     * yet not reach the task within the wait window. The server default output
+     * mode (`throw`) turns that into TIMEOUT_EXCEEDED (code 159) and aborts
+     * the whole migrate run — even though the task stays queued and the hosts
+     * execute it in the background (until the DDL queue evicts it:
+     * task_max_lifetime, one week, or falling more than max_tasks_in_queue
+     * entries behind). `null_status_on_timeout` returns NULL for the hosts
+     * that haven't finished yet instead of throwing, while a real DDL failure
+     * on any host that did run it within the window still throws. Safe because
+     * every schema statement here is idempotent (IF NOT EXISTS / OR REPLACE)
+     * and, with the default `distributed_ddl.pool_size = 1`, later DDL queues
+     * strictly behind earlier DDL on each host. Migrate.ts additionally warns
+     * at end of run when the DDL queue still has unfinished tasks. Note these
+     * options are also reused by runtime ALTER ... DELETE mutations (session
+     * erasure / pin materialization); timeout-as-success is acceptable there
+     * too since a mutation is durable once enqueued. Requires ClickHouse >=
+     * 21.4 (older servers reject the setting as unknown).
+     */
+    distributed_ddl_output_mode: "null_status_on_timeout",
+    /*
+     * Int64 settings are typed as strings by @clickhouse/client. Read at
+     * module load; env is static for the life of the process. The migration
+     * pool's socket-idle ceiling scales with this value (ClickhouseConfig.ts)
+     * so a raised wait isn't killed client-side.
+     */
+    distributed_ddl_task_timeout: String(getDistributedDdlTaskTimeoutSeconds()),
   },
 };
 
@@ -1440,6 +1469,75 @@ export default class AnalyticsDatabaseService<
     return { statement, columns: select.columns };
   }
 
+  /**
+   * Append the unique `_id` as a final sort key so the ORDER BY is a TOTAL
+   * order.
+   *
+   * `ORDER BY time DESC` alone is not: rows tying on `time` may come back
+   * in any order, and ClickHouse is free to order them differently between
+   * two executions of the same query (part order, thread scheduling and
+   * `optimize_read_in_order` all feed into it). Paging with LIMIT/OFFSET on
+   * top of a non-total order silently drops rows at page boundaries and
+   * repeats others: a row that sat at offset 49 on page 1 can shift to
+   * offset 50 before page 2 is fetched, so nobody ever sees it.
+   *
+   * Ties are the normal case, not an edge case — a service emitting a burst
+   * of logs writes many rows with an identical timestamp.
+   *
+   * `_id` is stamped per row with `ObjectID.generateTimeOrdered()`, so it is
+   * unique and its ordering agrees with time order.
+   *
+   * Two cases must NOT get the tiebreaker:
+   * - GROUP BY queries, where `_id` is neither a grouping key nor an
+   *   aggregate, so referencing it is an error.
+   * - Models with no `_id` column: derived aggregate targets deliberately
+   *   omit it because the aggregation key is the row identity (see _findBy).
+   */
+  private toPaginationStableSort(
+    sort: Sort<TBaseModel>,
+    options: { hasGroupBy: boolean },
+  ): Sort<TBaseModel> {
+    if (options.hasGroupBy) {
+      return sort;
+    }
+
+    const sortKeys: Array<string> = Object.keys(sort || {});
+
+    // Already a total order; adding `_id` again would be a no-op term.
+    if (sortKeys.includes("_id")) {
+      return sort;
+    }
+
+    const hasIdColumn: boolean = this.model.tableColumns.some(
+      (column: AnalyticsTableColumn): boolean => {
+        return column.key === "_id";
+      },
+    );
+
+    if (!hasIdColumn) {
+      return sort;
+    }
+
+    /*
+     * Tie-break in the same direction as the last declared sort key, so a
+     * newest-first list stays newest-first within a single timestamp.
+     */
+    const lastSortKey: string | undefined = sortKeys[sortKeys.length - 1];
+
+    const direction: SortOrder =
+      (lastSortKey
+        ? ((sort as Record<string, SortOrder | undefined>)[lastSortKey] as
+            | SortOrder
+            | undefined)
+        : undefined) ?? SortOrder.Descending;
+
+    /*
+     * `_id` is absent from `sort` (checked above), so spreading appends it
+     * last and the caller's own keys keep their relative order.
+     */
+    return { ...sort, _id: direction } as Sort<TBaseModel>;
+  }
+
   public toFindStatement(findBy: FindBy<TBaseModel>): {
     statement: Statement;
     columns: Array<string>;
@@ -1470,7 +1568,9 @@ export default class AnalyticsDatabaseService<
     );
 
     const sortStatement: Statement = this.statementGenerator.toSortStatement(
-      findBy.sort!,
+      this.toPaginationStableSort(findBy.sort!, {
+        hasGroupBy: Boolean(groupByStatement),
+      }),
     );
 
     const statement: Statement = SQL``;

package/Server/Services/CodeRepositoryService.ts

@@ -14,6 +14,11 @@ import {
 import GitHubUtil, {
   GitHubRepository,
 } from "../Utils/CodeRepository/GitHub/GitHub";
+import GitHubInstallationBinding from "../Utils/CodeRepository/GitHub/GitHubInstallationBinding";
+import BadDataException from "../../Types/Exception/BadDataException";
+import CreateBy from "../Types/Database/CreateBy";
+import UpdateBy from "../Types/Database/UpdateBy";
+import { OnCreate, OnUpdate } from "../Types/Database/Hooks";
 
 export interface ImportReposFromInstallationResult {
   imported: number;
@@ -25,6 +30,93 @@ export class Service extends DatabaseService<Model> {
     super(Model);
   }
 
+  /*
+   * Last line of defence for the installation-to-project binding.
+   *
+   * gitHubAppInstallationId is already closed to API callers at the column
+   * level, but that only stops requests that go through permission checks —
+   * every write here is made with `props: { isRoot: true }`, which skips them
+   * entirely. This hook runs regardless of isRoot, so any route (including one
+   * added later) that writes a row carrying another tenant's installation ID
+   * is refused at the service boundary rather than at the token endpoint.
+   *
+   * The legitimate writer is importReposFromInstallation, which the install
+   * callback and the installation webhooks only ever call for a project the
+   * installation is already bound to — so this check passes for them.
+   */
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    const installationId: string | undefined =
+      createBy.data.gitHubAppInstallationId;
+
+    if (installationId) {
+      const projectId: ObjectID | undefined =
+        createBy.data.projectId || createBy.props.tenantId;
+
+      if (!projectId) {
+        throw new BadDataException(
+          "A project is required to connect a GitHub App installation to a repository.",
+        );
+      }
+
+      await GitHubInstallationBinding.assertInstallationBoundToProject({
+        projectId: projectId,
+        installationId: installationId,
+      });
+    }
+
+    return { createBy, carryForward: null };
+  }
+
+  /*
+   * Same rule on the update path. Clearing the installation ID is always
+   * allowed — that is how the `installation.deleted` webhook tears a
+   * disconnected installation out of the database.
+   */
+  @CaptureSpan()
+  protected override async onBeforeUpdate(
+    updateBy: UpdateBy<Model>,
+  ): Promise<OnUpdate<Model>> {
+    const installationId: string | undefined = updateBy.data
+      .gitHubAppInstallationId as string | undefined;
+
+    if (installationId) {
+      /*
+       * updateBy matches by query rather than by id, so check every row the
+       * write would touch: one unbound row is enough to leak a token.
+       */
+      const repositories: Array<Model> = await this.findBy({
+        query: updateBy.query,
+        select: {
+          _id: true,
+          projectId: true,
+        },
+        limit: LIMIT_MAX,
+        skip: 0,
+        props: {
+          isRoot: true,
+        },
+      });
+
+      for (const repository of repositories) {
+        if (!repository.projectId) {
+          throw new BadDataException(
+            "A project is required to connect a GitHub App installation to a repository.",
+          );
+        }
+
+        await GitHubInstallationBinding.assertInstallationBoundToProject({
+          projectId: repository.projectId,
+          installationId: installationId,
+        });
+      }
+    }
+
+    return { updateBy, carryForward: null };
+  }
+
   /*
    * Imports all repositories accessible to a GitHub App installation into a
    * project. Repositories that already exist for the project (matched on
@@ -174,13 +266,24 @@ export class Service extends DatabaseService<Model> {
       },
     });
 
-    // Only GitHub-App-connected repos can be cloned/pushed by the agent.
+    /*
+     * Only GitHub-App-connected repos can be cloned/pushed by the agent — and
+     * only ones whose installation this project actually owns. Resolution
+     * probes EVERY candidate's file tree, so an unbound row here would have
+     * the server read another tenant's private repository on its own
+     * initiative (GHSA-xx95-gmcf-7q86).
+     */
+    const boundInstallationId: string | null =
+      await GitHubInstallationBinding.getBoundInstallationId(data.projectId);
+
     const resolvable: Array<ResolvableRepository> = repositories
       .filter((repository: Model) => {
         return (
           repository.repositoryHostedAt === CodeRepositoryType.GitHub &&
           Boolean(repository.gitHubAppInstallationId) &&
-          Boolean(repository.id)
+          Boolean(repository.id) &&
+          Boolean(boundInstallationId) &&
+          repository.gitHubAppInstallationId === boundInstallationId
         );
       })
       .map((repository: Model) => {