@oneuptime/common 11.5.12 → 11.6.0

package/Server/Services/DatabaseService.ts

@@ -1743,10 +1743,67 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
     });
   }
 
+  /*
+   * Update `data` may arrive as a full model instance rather than a plain
+   * partial — the QueryDeepPartialEntity type structurally admits both, and
+   * callers do construct `new Model()` payloads. A model instance carries
+   * non-column own properties from DatabaseBaseModel (every column
+   * initializer plus `isPermissionIf = {}`), and _updateBy turns every data
+   * key into a select column for its internal find, so the extra keys made
+   * that find throw `TableColumnMetadata not found for isPermissionIf
+   * column` and fail the whole update. Strip a model instance down to its
+   * set table columns; plain objects pass through untouched so a typo'd
+   * column name in a literal still fails loudly instead of being silently
+   * dropped.
+   *
+   * `_id`, `createdAt`, `updatedAt` and `version` are dropped for model
+   * instances the same way BaseAPI drops them from client payloads: the row
+   * is located by the update query (spreading a foreign `_id` into the save
+   * payload would redirect the write), timestamps are database-managed, and
+   * `version` is TypeORM's optimistic-concurrency counter.
+   */
+  public sanitizeUpdateData(
+    data: UpdateBy<TBaseModel>["data"],
+  ): UpdateBy<TBaseModel>["data"] {
+    if (!(data instanceof BaseModel)) {
+      return data;
+    }
+
+    const plainData: JSONObject = {};
+
+    for (const key of Object.keys(data)) {
+      if (!this.model.isTableColumn(key)) {
+        continue;
+      }
+
+      if (
+        key === "_id" ||
+        key === "createdAt" ||
+        key === "updatedAt" ||
+        key === "version"
+      ) {
+        continue;
+      }
+
+      const value: JSONValue = (data as any)[key];
+
+      // Unset columns must not become writes (or select columns).
+      if (value === undefined) {
+        continue;
+      }
+
+      plainData[key] = value;
+    }
+
+    return plainData as UpdateBy<TBaseModel>["data"];
+  }
+
   private async _updateBy(updateBy: UpdateBy<TBaseModel>): Promise<number> {
     try {
       this.setTelemetryContextFromProps(updateBy.props);
 
+      updateBy.data = this.sanitizeUpdateData(updateBy.data);
+
       const onUpdate: OnUpdate<TBaseModel> = updateBy.props.ignoreHooks
         ? { updateBy, carryForward: [] }
         : await this.onBeforeUpdate(updateBy);
@@ -1830,9 +1887,17 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
       });
 
       for (const item of items) {
+        /*
+         * _id must be set AFTER the spread: update data can carry an
+         * explicit `_id: undefined` (sanitizeUpdateData strips it from model
+         * instances, but a plain object can still hold it), and spreading it
+         * after _id would clobber the located row's id — save() then sees no
+         * primary key, INSERTs instead of updating, and dies on the first
+         * NOT NULL column.
+         */
         const updatedItem: any = {
-          _id: item._id!,
           ...data,
+          _id: item._id!,
         } as any;
 
         logger.debug("Updated Item", {

package/Server/Services/TelemetryExceptionService.ts

@@ -28,6 +28,12 @@ import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import Query from "../Types/Database/Query";
 import Includes from "../../Types/BaseDatabase/Includes";
 import logger from "../Utils/Logger";
+import AIAgentTaskPullRequest from "../../Models/DatabaseModels/AIAgentTaskPullRequest";
+import AIAgentTaskPullRequestService from "./AIAgentTaskPullRequestService";
+import PullRequestState from "../../Types/CodeRepository/PullRequestState";
+import { normalizeExceptionText } from "../Utils/Telemetry/ExceptionSanitizer";
+import Semaphore, { SemaphoreMutex } from "../Infrastructure/Semaphore";
+import LIMIT_MAX from "../../Types/Database/LimitMax";
 
 /*
  * Hard cap on the fingerprint NOT IN list handed to the ClickHouse count
@@ -221,6 +227,11 @@ export class Service extends DatabaseService<Model> {
       select: {
         _id: true,
         projectId: true,
+        message: true,
+        exceptionType: true,
+        isResolved: true,
+        isArchived: true,
+        aiFixDeclinedAt: true,
       },
       props,
     });
@@ -229,6 +240,45 @@ export class Service extends DatabaseService<Model> {
       throw new BadDataException("Telemetry Exception not found");
     }
 
+    /*
+     * Server-side lifecycle gate (the dashboard hides the button for
+     * resolved exceptions, but the API must enforce it — the automatic
+     * insight lane and any direct API caller land here too).
+     */
+    if (telemetryException.isResolved) {
+      throw new BadDataException(
+        "This exception is marked as resolved. Unresolve it before starting an AI agent task for it.",
+      );
+    }
+
+    if (telemetryException.isArchived) {
+      throw new BadDataException(
+        "This exception is archived. Unarchive it before starting an AI agent task for it.",
+      );
+    }
+
+    /*
+     * Human "closed without merging" feedback: when an AI FIX PR for this
+     * exception was declined, the automatic lane must not keep re-opening
+     * fix PRs for it. Scoped to the FixException recipe — the stamp is only
+     * ever SET for closed fix PRs (SyncPullRequestStates), so a regression
+     * test or error-handling task is neither blocked by it nor allowed to
+     * clear it. A HUMAN clicking "Fix with AI" is the explicit override;
+     * the stamp is cleared only AFTER the retry run is actually created
+     * (see below), so a retry that fails a later gate leaves the human's
+     * decline in force.
+     */
+    const isDeclinedFixOverride: boolean = Boolean(
+      telemetryException.aiFixDeclinedAt &&
+        taskType === CodeFixTaskType.FixException,
+    );
+
+    if (isDeclinedFixOverride && !props.userId) {
+      throw new BadDataException(
+        "A previous AI fix pull request for this exception was closed without merging, so automatic fix attempts are paused for it. A user can retry from the exception page.",
+      );
+    }
+
     /*
      * G11 guardrail: the per-project daily fix-run budget. Checked before
      * the (more expensive) readiness probes — an over-budget project gets
@@ -261,38 +311,107 @@ export class Service extends DatabaseService<Model> {
     }
 
     /*
-     * Duplicate guard is per (exception, taskType): an active FixException
-     * run must not block queuing a WriteRegressionTest run and vice versa.
+     * The duplicate guards below are check-then-create: without
+     * serialization, two triage completions for splintered variants of the
+     * same root cause can both pass the guards and both create runs. A
+     * per-project Redis mutex closes the window; acquisition failure
+     * (Redis down) degrades to the unserialized behavior rather than
+     * blocking fix creation — the guards still run, they just race.
      */
-    await this.validateNoActiveCodeFixRunExists(telemetryExceptionId, taskType);
-
-    const run: AIRun = new AIRun();
-    run.projectId = telemetryException.projectId;
-    run.runType = AIRunType.CodeFix;
-    run.codeFixTaskType = taskType;
-    run.status = AIRunStatus.Queued;
-    run.triggeredByTelemetryExceptionId = telemetryExceptionId;
-
-    // Attribution: the user who clicked "Fix with AI Agent".
-    if (props.userId) {
-      run.userId = props.userId;
+    let mutex: SemaphoreMutex | null = null;
+
+    try {
+      mutex = await Semaphore.lock({
+        key: telemetryException.projectId.toString(),
+        namespace: "TelemetryExceptionService.createCodeFixRunForException",
+        lockTimeout: 15000,
+      });
+    } catch (err) {
+      logger.debug(
+        `Could not acquire the code-fix creation mutex for project ${telemetryException.projectId.toString()} — proceeding without serialization: ${err}`,
+      );
     }
 
-    /*
-     * Created as root: AIRun rows are server-written only (empty create
-     * ACL); the user's access was already checked by the exception read.
-     */
-    const createdRun: AIRun = await AIRunService.create({
-      data: run,
-      props: {
-        isRoot: true,
-      },
-    });
+    let createdRun: AIRun;
+
+    try {
+      /*
+       * Duplicate guard is per (exception, taskType): an active
+       * FixException run must not block queuing a WriteRegressionTest run
+       * and vice versa.
+       */
+      await this.validateNoActiveCodeFixRunExists(
+        telemetryExceptionId,
+        taskType,
+      );
+
+      /*
+       * Cross-group duplicate guard: interpolated dynamic values (garbage
+       * UUIDs, file paths, request payloads) splinter one root cause into
+       * many fingerprints, and the per-exception guard above cannot see
+       * that. Compare NORMALIZED messages against exceptions that already
+       * have an active run or an open AI pull request, so one root cause
+       * yields one PR — not one per interpolated variant.
+       */
+      await this.validateNoOpenFixForSimilarException({
+        projectId: telemetryException.projectId,
+        telemetryExceptionId: telemetryExceptionId,
+        taskType: taskType,
+        message: telemetryException.message || "",
+        exceptionType: telemetryException.exceptionType || "",
+      });
+
+      const run: AIRun = new AIRun();
+      run.projectId = telemetryException.projectId;
+      run.runType = AIRunType.CodeFix;
+      run.codeFixTaskType = taskType;
+      run.status = AIRunStatus.Queued;
+      run.triggeredByTelemetryExceptionId = telemetryExceptionId;
+
+      // Attribution: the user who clicked "Fix with AI Agent".
+      if (props.userId) {
+        run.userId = props.userId;
+      }
+
+      /*
+       * Created as root: AIRun rows are server-written only (empty create
+       * ACL); the user's access was already checked by the exception read.
+       */
+      createdRun = await AIRunService.create({
+        data: run,
+        props: {
+          isRoot: true,
+        },
+      });
+    } finally {
+      if (mutex) {
+        try {
+          await Semaphore.release(mutex);
+        } catch (err) {
+          logger.debug(`Failed to release the code-fix creation mutex: ${err}`);
+        }
+      }
+    }
 
     if (!createdRun.id) {
       throw new BadDataException("Failed to create the AI fix run");
     }
 
+    /*
+     * The human's retry actually went through — only now consume the
+     * decline override. A retry that failed any gate above leaves the
+     * suppression in force.
+     */
+    if (isDeclinedFixOverride) {
+      await this.updateOneById({
+        id: telemetryExceptionId,
+        data: {
+          aiFixDeclinedAt: null,
+        },
+        props: { isRoot: true },
+      });
+    }
+
     return createdRun;
   }
 
@@ -338,6 +457,183 @@ export class Service extends DatabaseService<Model> {
     }
   }
 
+  /*
+   * Cross-group duplicate guard. Two exception groups are "the same work"
+   * for the fix lane when their exceptionType and NORMALIZED message match
+   * (normalizeExceptionText replaces UUIDs, IDs, emails, paths' dynamic
+   * segments, timestamps...). Candidates are the exceptions behind
+   * (a) non-terminal CodeFix runs of the same recipe, and (b) OPEN
+   * AI-authored pull requests — a completed run whose PR is still open
+   * must keep blocking, or every interpolated variant re-fixes the same
+   * root cause (observed: 15 near-identical PRs for one invalid-uuid
+   * throw site).
+   */
+  @CaptureSpan()
+  private async validateNoOpenFixForSimilarException(data: {
+    projectId: ObjectID;
+    telemetryExceptionId: ObjectID;
+    taskType: CodeFixTaskType;
+    message: string;
+    exceptionType: string;
+  }): Promise<void> {
+    // (a) Exceptions with a non-terminal run of the same recipe.
+    const activeRuns: Array<AIRun> = await AIRunService.findBy({
+      query: {
+        projectId: data.projectId,
+        runType: AIRunType.CodeFix,
+        /*
+         * A null codeFixTaskType means FixException (rows created before
+         * task recipes existed) — same legacy-null matching as
+         * validateNoActiveCodeFixRunExists.
+         */
+        codeFixTaskType:
+          data.taskType === CodeFixTaskType.FixException
+            ? QueryHelper.equalToOrNull(CodeFixTaskType.FixException)
+            : data.taskType,
+        status: QueryHelper.notIn([
+          AIRunStatus.Completed,
+          AIRunStatus.NoFixFound,
+          AIRunStatus.Error,
+          AIRunStatus.Cancelled,
+          AIRunStatus.Stale,
+        ]),
+      },
+      select: {
+        triggeredByTelemetryExceptionId: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: { isRoot: true },
+    });
+
+    // (b) Exceptions behind still-open AI pull requests.
+    const openPullRequests: Array<AIAgentTaskPullRequest> =
+      await AIAgentTaskPullRequestService.findBy({
+        query: {
+          projectId: data.projectId,
+          pullRequestState: PullRequestState.Open,
+        },
+        select: {
+          aiRunId: true,
+        },
+        limit: LIMIT_MAX,
+        skip: 0,
+        props: { isRoot: true },
+      });
+
+    /*
+     * A guard that silently stops guarding is worse than none — surface
+     * candidate-scan truncation instead of quietly missing duplicates.
+     */
+    if (
+      activeRuns.length >= LIMIT_MAX ||
+      openPullRequests.length >= LIMIT_MAX
+    ) {
+      logger.warn(
+        `Cross-group AI fix dedupe guard: candidate scan truncated at ${LIMIT_MAX} rows for project ${data.projectId.toString()} — the duplicate-PR guard may miss matches.`,
+      );
+    }
+
+    const openPrRunIds: Array<ObjectID> = openPullRequests
+      .map((pullRequest: AIAgentTaskPullRequest) => {
+        return pullRequest.aiRunId;
+      })
+      .filter((runId: ObjectID | undefined): runId is ObjectID => {
+        return Boolean(runId);
+      });
+
+    const openPrRuns: Array<AIRun> =
+      openPrRunIds.length > 0
+        ? await AIRunService.findBy({
+            query: {
+              _id: new Includes(openPrRunIds),
+              projectId: data.projectId,
+              runType: AIRunType.CodeFix,
+              codeFixTaskType:
+                data.taskType === CodeFixTaskType.FixException
+                  ? QueryHelper.equalToOrNull(CodeFixTaskType.FixException)
+                  : data.taskType,
+            },
+            select: {
+              triggeredByTelemetryExceptionId: true,
+            },
+            limit: LIMIT_MAX,
+            skip: 0,
+            props: { isRoot: true },
+          })
+        : [];
+
+    /*
+     * The target exception is deliberately NOT excluded: its own COMPLETED
+     * run with a still-open PR must keep blocking a re-fix (the
+     * non-terminal guard above cannot see completed runs). Its own ACTIVE
+     * runs were already rejected by validateNoActiveCodeFixRunExists.
+     */
+    const candidateExceptionIds: Array<ObjectID> = [];
+    const seenIds: Set<string> = new Set<string>();
+
+    for (const run of [...activeRuns, ...openPrRuns]) {
+      const exceptionId: ObjectID | undefined =
+        run.triggeredByTelemetryExceptionId;
+
+      if (!exceptionId) {
+        continue;
+      }
+
+      const idString: string = exceptionId.toString();
+
+      if (seenIds.has(idString)) {
+        continue;
+      }
+
+      seenIds.add(idString);
+      candidateExceptionIds.push(exceptionId);
+    }
+
+    if (candidateExceptionIds.length === 0) {
+      return;
+    }
+
+    const targetSignature: string = `${data.exceptionType}|${normalizeExceptionText(
+      data.message,
+    )}`;
+
+    const candidates: Array<Model> = await this.findBy({
+      query: {
+        projectId: data.projectId,
+        _id: new Includes(candidateExceptionIds),
+      },
+      select: {
+        _id: true,
+        message: true,
+        exceptionType: true,
+      },
+      limit: candidateExceptionIds.length,
+      skip: 0,
+      props: { isRoot: true },
+    });
+
+    for (const candidate of candidates) {
+      const candidateSignature: string = `${candidate.exceptionType || ""}|${normalizeExceptionText(
+        candidate.message || "",
+      )}`;
+
+      if (candidateSignature !== targetSignature) {
+        continue;
+      }
+
+      if (candidate._id?.toString() === data.telemetryExceptionId.toString()) {
+        throw new BadDataException(
+          "This exception already has an open AI pull request for this task type. Review that pull request instead of opening another one.",
+        );
+      }
+
+      throw new BadDataException(
+        `A similar exception (same type and normalized message, exception ${candidate._id?.toString()}) already has an AI agent task in progress or an open AI pull request. Review that pull request instead of opening another one for the same root cause.`,
+      );
+    }
+  }
+
   @CaptureSpan()
   public async getDashboardSummary(
     props: DatabaseCommonInteractionProps,

package/Server/Services/WorkflowService.ts

@@ -49,6 +49,40 @@ export class Service extends DatabaseService<Model> {
       createdItem.webhookSecretKey = secretKey;
     }
 
+    /*
+     * A workflow that arrives with its graph already in place - imported from
+     * a JSON export, or duplicated from another workflow - never passes
+     * through onUpdateSuccess, so nothing has denormalized its trigger onto
+     * the row. The runner looks workflows up by triggerId, so without this the
+     * workflow is saved and looks correct in the builder but silently never
+     * fires.
+     */
+    if (createdItem.id && createdItem.graph) {
+      await this.saveTriggerFromGraph({
+        workflowId: createdItem.id,
+        graph: createdItem.graph,
+      });
+
+      /*
+       * Registers schedule triggers with the runner. Best effort: the row and
+       * its trigger are already persisted, so a workflow service outage must
+       * not fail the create. The trigger is already on the row, so the next
+       * save - or the runner's own startup scan in Schedule.init, which
+       * queries by triggerId - picks the workflow up.
+       */
+      try {
+        await this.notifyWorkflowService(createdItem.id);
+      } catch (error) {
+        logger.error(
+          `Error notifying workflow service of created workflow: ${error}`,
+          {
+            projectId: createdItem.projectId?.toString(),
+            workflowId: createdItem.id?.toString(),
+          } as LogAttributes,
+        );
+      }
+    }
+
     if (createdItem.projectId && createdItem.id) {
       /*
        * Run label rule first so rule-added labels are persisted before
@@ -87,39 +121,13 @@ export class Service extends DatabaseService<Model> {
   ): Promise<OnUpdate<Model>> {
     /// save trigger and trigger args.
 
-    if (
-      onUpdate.updateBy.data &&
-      (onUpdate.updateBy.data as any).graph &&
-      (((onUpdate.updateBy.data as any).graph as any)[
-        "nodes"
-      ] as Array<JSONObject>)
-    ) {
-      let trigger: NodeDataProp | null = null;
-
-      // check if it has a trigger node.
-      for (const node of ((onUpdate.updateBy.data as any).graph as any)[
-        "nodes"
-      ] as Array<JSONObject>) {
-        const nodeData: NodeDataProp = node["data"] as any;
-        if (
-          nodeData.componentType === ComponentType.Trigger &&
-          nodeData.nodeType === NodeType.Node
-        ) {
-          // found the trigger;
-          trigger = nodeData;
-        }
-      }
+    const updatedGraph: JSONObject | undefined = (onUpdate.updateBy.data as any)
+      ?.graph as JSONObject | undefined;
 
-      await this.updateOneById({
-        id: new ObjectID(onUpdate.updateBy.query._id! as any),
-        data: {
-          triggerId: trigger?.metadataId || null,
-          triggerArguments: trigger?.arguments || {},
-        } as any,
-        props: {
-          isRoot: true,
-          ignoreHooks: true,
-        },
+    if (updatedGraph) {
+      await this.saveTriggerFromGraph({
+        workflowId: new ObjectID(onUpdate.updateBy.query._id! as any),
+        graph: updatedGraph,
       });
     }
 
@@ -127,23 +135,76 @@ export class Service extends DatabaseService<Model> {
       workflowId: onUpdate.updateBy.query._id?.toString(),
     } as LogAttributes);
 
+    await this.notifyWorkflowService(
+      new ObjectID(onUpdate.updateBy.query._id! as any),
+    );
+
+    logger.debug("Updated workflow on the workflow service", {
+      workflowId: onUpdate.updateBy.query._id?.toString(),
+    } as LogAttributes);
+
+    return onUpdate;
+  }
+
+  /*
+   * The trigger node is denormalized out of the graph onto triggerId /
+   * triggerArguments because the runner queries workflows by trigger and
+   * cannot parse every graph to do it. A graph with no trigger node clears
+   * both columns, which is how a workflow stops firing when its trigger is
+   * removed in the builder.
+   */
+  private async saveTriggerFromGraph(data: {
+    workflowId: ObjectID;
+    graph: JSONObject;
+  }): Promise<void> {
+    const nodes: Array<JSONObject> | undefined = data.graph["nodes"] as
+      | Array<JSONObject>
+      | undefined;
+
+    if (!nodes || !Array.isArray(nodes)) {
+      return;
+    }
+
+    let trigger: NodeDataProp | null = null;
+
+    // check if it has a trigger node.
+    for (const node of nodes) {
+      const nodeData: NodeDataProp = node["data"] as any;
+
+      if (
+        nodeData?.componentType === ComponentType.Trigger &&
+        nodeData?.nodeType === NodeType.Node
+      ) {
+        // found the trigger;
+        trigger = nodeData;
+      }
+    }
+
+    await this.updateOneById({
+      id: data.workflowId,
+      data: {
+        triggerId: trigger?.metadataId || null,
+        triggerArguments: trigger?.arguments || {},
+      } as any,
+      props: {
+        isRoot: true,
+        ignoreHooks: true,
+      },
+    });
+  }
+
+  private async notifyWorkflowService(workflowId: ObjectID): Promise<void> {
     await API.post<EmptyResponseData>({
       url: new URL(
         Protocol.HTTP,
         WorkflowHostname,
-        new Route("/workflow/update/" + onUpdate.updateBy.query._id!),
+        new Route("/workflow/update/" + workflowId.toString()),
       ),
       data: {},
       headers: {
         ...ClusterKeyAuthorization.getClusterKeyHeaders(),
       },
     });
-
-    logger.debug("Updated workflow on the workflow service", {
-      workflowId: onUpdate.updateBy.query._id?.toString(),
-    } as LogAttributes);
-
-    return onUpdate;
   }
 }
 export default new Service();