@oneuptime/common 11.7.4 → 12.0.1

package/Server/Services/{RunbookAgentJobService.ts → RunnerJobService.ts}

@@ -1,9 +1,22 @@
 import DatabaseService from "./DatabaseService";
 import ObjectID from "../../Types/ObjectID";
 import BadDataException from "../../Types/Exception/BadDataException";
-import Model from "../../Models/DatabaseModels/RunbookAgentJob";
-import RunbookAgentJobStatus from "../../Types/Runbook/RunbookAgentJobStatus";
-import RunbookStepType from "../../Types/Runbook/RunbookStepType";
+import Model from "../../Models/DatabaseModels/RunnerJob";
+import RunnerJobStatus from "../../Types/Runbook/RunnerJobStatus";
+import RunnerJobOrigin from "../../Types/Runbook/RunnerJobOrigin";
+import RunbookStepType, {
+  isRunnerExecutedStepType,
+} from "../../Types/Runbook/RunbookStepType";
+import { AI_COMMAND_STEP_TYPES } from "../../Types/AutoRemediation/AiRemediationCommandPlan";
+import CommandPolicy from "../../Utils/AiRemediation/CommandPolicy";
+import QueryHelper from "../Types/Database/QueryHelper";
+
+/*
+ * Project-wide hourly ceiling on AI-composed command jobs, counted on the
+ * RunnerJob rows themselves so every path (FullAuto inline execution,
+ * approved plans, rollbacks) shares one brake.
+ */
+export const MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR: number = 30;
 import SortOrder from "../../Types/BaseDatabase/SortOrder";
 import OneUptimeDate from "../../Types/Date";
 import { JSONObject } from "../../Types/JSON";
@@ -32,15 +45,15 @@ const POLL_INTERVAL_MS: number = 500;
  * durable record of what the step did. Everything else is still in flight and
  * can be waited on.
  */
-const TERMINAL_STATUSES: Array<RunbookAgentJobStatus> = [
-  RunbookAgentJobStatus.Succeeded,
-  RunbookAgentJobStatus.Failed,
-  RunbookAgentJobStatus.TimedOut,
-  RunbookAgentJobStatus.Cancelled,
+const TERMINAL_STATUSES: Array<RunnerJobStatus> = [
+  RunnerJobStatus.Succeeded,
+  RunnerJobStatus.Failed,
+  RunnerJobStatus.TimedOut,
+  RunnerJobStatus.Cancelled,
 ];
 
 export function isTerminalAgentJobStatus(
-  status: RunbookAgentJobStatus | undefined,
+  status: RunnerJobStatus | undefined,
 ): boolean {
   return Boolean(status && TERMINAL_STATUSES.includes(status));
 }
@@ -79,19 +92,17 @@ export class Service extends DatabaseService<Model> {
     script: string;
     timeoutInMs: number;
     claimTimeoutInMs?: number | undefined;
+    payload?: JSONObject | undefined;
   }): Promise<Model> {
     if (!data.targetAgentId) {
       throw new BadDataException(
-        "targetAgentId is required to dispatch a step to a Runbook Agent.",
+        "targetAgentId is required to dispatch a step to a Runner.",
       );
     }
 
-    if (
-      data.stepType !== RunbookStepType.Bash &&
-      data.stepType !== RunbookStepType.JavaScript
-    ) {
+    if (!isRunnerExecutedStepType(data.stepType)) {
       throw new BadDataException(
-        `RunbookAgent does not execute step type "${data.stepType}".`,
+        `Runner does not execute step type "${data.stepType}".`,
       );
     }
 
@@ -103,12 +114,123 @@ export class Service extends DatabaseService<Model> {
     const row: Model = new Model();
     row.projectId = data.projectId;
     row.runbookExecutionId = data.runbookExecutionId;
+    row.origin = RunnerJobOrigin.Runbook;
     row.stepId = data.stepId;
     row.stepType = data.stepType;
     row.targetAgentId = data.targetAgentId;
     row.script = data.script;
+    if (data.payload) {
+      row.payload = data.payload;
+    }
     row.timeoutInMs = data.timeoutInMs;
-    row.status = RunbookAgentJobStatus.Pending;
+    row.status = RunnerJobStatus.Pending;
+    row.claimDeadlineAt = claimDeadlineAt;
+
+    return this.create({ data: row, props: { isRoot: true } });
+  }
+
+  /*
+   * Enqueue an ad-hoc AI-composed command as an AiRemediation-origin job.
+   * This is a server-side chokepoint on the path an LLM's output takes to a
+   * shell, so it re-validates what the tool layer already checked: step
+   * type and the hard command denylist. The claim path then only serves
+   * these jobs to Runners whose canRunAiCommands capability is on.
+   */
+  @CaptureSpan()
+  public async enqueueAiCommand(data: {
+    projectId: ObjectID;
+    aiRunId: ObjectID;
+    autoRemediationSuggestionId: ObjectID;
+    stepId: string;
+    stepType: RunbookStepType;
+    targetAgentId: ObjectID;
+    command: string;
+    credentialId?: string | undefined;
+    timeoutInMs: number;
+    claimTimeoutInMs?: number | undefined;
+  }): Promise<Model> {
+    if (!data.targetAgentId) {
+      throw new BadDataException(
+        "targetAgentId is required to dispatch a command to a Runner.",
+      );
+    }
+
+    if (!AI_COMMAND_STEP_TYPES.includes(data.stepType)) {
+      throw new BadDataException(
+        `AI remediation does not execute step type "${data.stepType}".`,
+      );
+    }
+
+    const command: string = (data.command || "").trim();
+    if (!command) {
+      throw new BadDataException("An AI command job needs a command.");
+    }
+
+    const denyReason: string | null = CommandPolicy.getDenyReason(command);
+    if (denyReason) {
+      throw new BadDataException(
+        `Denied by the remediation command policy: ${denyReason}.`,
+      );
+    }
+
+    if (data.stepType === RunbookStepType.SSH && !data.credentialId) {
+      throw new BadDataException("An SSH command job needs a credentialId.");
+    }
+
+    /*
+     * Project-wide hourly storm brake for the whole AI-command lane. It
+     * lives HERE, at the single enqueue chokepoint, so it also bounds the
+     * approved-plan and rollback paths — not just the FullAuto inline tool
+     * that has its own pre-check for a friendlier message to the model.
+     * Check-then-act, so a burst of concurrent runs can overshoot slightly;
+     * the cap is a storm brake, not a quota.
+     */
+    const jobsInLastHour: number = (
+      await this.countBy({
+        query: {
+          projectId: data.projectId,
+          origin: RunnerJobOrigin.AiRemediation,
+          createdAt: QueryHelper.greaterThan(OneUptimeDate.getSomeHoursAgo(1)),
+        },
+        props: { isRoot: true },
+      })
+    ).toNumber();
+
+    if (jobsInLastHour >= MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR) {
+      throw new BadDataException(
+        `This project has run ${jobsInLastHour} AI remediation commands in the last hour, which is its limit (${MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR}). No further AI commands can run this hour.`,
+      );
+    }
+
+    const claimDeadlineAt: Date = OneUptimeDate.addRemoveSeconds(
+      OneUptimeDate.getCurrentDate(),
+      Math.ceil((data.claimTimeoutInMs ?? DEFAULT_CLAIM_TIMEOUT_MS) / 1000),
+    );
+
+    const row: Model = new Model();
+    row.projectId = data.projectId;
+    row.origin = RunnerJobOrigin.AiRemediation;
+    row.aiRunId = data.aiRunId;
+    row.autoRemediationSuggestionId = data.autoRemediationSuggestionId;
+    row.stepId = data.stepId;
+    row.stepType = data.stepType;
+    row.targetAgentId = data.targetAgentId;
+    /*
+     * Same layout the runbook step executors use: Bash carries the command
+     * as the script; SSH carries an empty script plus structured payload,
+     * and the credential is resolved at claim time — never stored here.
+     */
+    if (data.stepType === RunbookStepType.Bash) {
+      row.script = command;
+    } else {
+      row.script = "";
+      row.payload = {
+        credentialId: data.credentialId as string,
+        command: command,
+      };
+    }
+    row.timeoutInMs = data.timeoutInMs;
+    row.status = RunnerJobStatus.Pending;
     row.claimDeadlineAt = claimDeadlineAt;
 
     return this.create({ data: row, props: { isRoot: true } });
@@ -165,6 +287,12 @@ export class Service extends DatabaseService<Model> {
     agentId: ObjectID;
     projectId: ObjectID;
     leaseMs?: number | undefined;
+    /*
+     * Which job origins this Runner's capabilities entitle it to claim.
+     * Defaults to runbook jobs only — the AiRemediation origin must be
+     * asked for explicitly by the ingress after checking canRunAiCommands.
+     */
+    allowedOrigins?: Array<RunnerJobOrigin> | undefined;
   }): Promise<Model | null> {
     const dataSource: ReturnType<typeof PostgresAppInstance.getDataSource> =
       PostgresAppInstance.getDataSource();
@@ -175,19 +303,25 @@ export class Service extends DatabaseService<Model> {
 
     const leaseMs: number = data.leaseMs ?? DEFAULT_LEASE_MS;
 
+    const allowedOrigins: Array<RunnerJobOrigin> =
+      data.allowedOrigins && data.allowedOrigins.length > 0
+        ? data.allowedOrigins
+        : [RunnerJobOrigin.Runbook];
+
     const sql: string = `
       WITH claimed AS (
-        SELECT "_id" FROM "RunbookAgentJob"
+        SELECT "_id" FROM "RunnerJob"
         WHERE "projectId" = $1::uuid
           AND "status" = $2
           AND "targetAgentId" = $3::uuid
+          AND "origin" = ANY($6::text[])
           AND "claimDeadlineAt" > NOW()
           AND "deletedAt" IS NULL
         ORDER BY "createdAt" ASC
         LIMIT 1
         FOR UPDATE SKIP LOCKED
       )
-      UPDATE "RunbookAgentJob" j
+      UPDATE "RunnerJob" j
       SET "status" = $4,
           "assignedAgentId" = $3::uuid,
           "claimedAt" = NOW(),
@@ -196,18 +330,21 @@ export class Service extends DatabaseService<Model> {
           "version" = j."version" + 1
       FROM claimed
       WHERE j."_id" = claimed."_id"
-      RETURNING j."_id", j."projectId", j."runbookExecutionId",
+      RETURNING j."_id", j."projectId", j."runbookExecutionId", j."origin",
+                j."aiRunId", j."autoRemediationSuggestionId",
                 j."stepId", j."stepType", j."targetAgentId", j."script",
-                j."timeoutInMs", j."status", j."claimedAt", j."leaseExpiresAt";
+                j."payload", j."timeoutInMs", j."status", j."claimedAt",
+                j."leaseExpiresAt";
     `;
 
     const rows: Array<JSONObject> = unwrapRows(
       await dataSource.query(sql, [
         data.projectId.toString(),
-        RunbookAgentJobStatus.Pending,
+        RunnerJobStatus.Pending,
         data.agentId.toString(),
-        RunbookAgentJobStatus.Claimed,
+        RunnerJobStatus.Claimed,
         leaseMs.toString(),
+        allowedOrigins,
       ]),
     );
 
@@ -219,13 +356,27 @@ export class Service extends DatabaseService<Model> {
     const job: Model = new Model();
     job._id = String(r["_id"]);
     job.projectId = new ObjectID(String(r["projectId"]));
-    job.runbookExecutionId = new ObjectID(String(r["runbookExecutionId"]));
+    if (r["runbookExecutionId"]) {
+      job.runbookExecutionId = new ObjectID(String(r["runbookExecutionId"]));
+    }
+    job.origin = (r["origin"] as RunnerJobOrigin) || RunnerJobOrigin.Runbook;
+    if (r["aiRunId"]) {
+      job.aiRunId = new ObjectID(String(r["aiRunId"]));
+    }
+    if (r["autoRemediationSuggestionId"]) {
+      job.autoRemediationSuggestionId = new ObjectID(
+        String(r["autoRemediationSuggestionId"]),
+      );
+    }
     job.stepId = String(r["stepId"]);
     job.stepType = r["stepType"] as RunbookStepType;
     job.targetAgentId = new ObjectID(String(r["targetAgentId"]));
     job.script = String(r["script"]);
+    if (r["payload"] && typeof r["payload"] === "object") {
+      job.payload = r["payload"] as JSONObject;
+    }
     job.timeoutInMs = Number(r["timeoutInMs"]);
-    job.status = r["status"] as RunbookAgentJobStatus;
+    job.status = r["status"] as RunnerJobStatus;
     if (r["claimedAt"]) {
       job.claimedAt = new Date(String(r["claimedAt"]));
     }
@@ -256,7 +407,7 @@ export class Service extends DatabaseService<Model> {
      * should stop executing.
      */
     const sql: string = `
-      UPDATE "RunbookAgentJob"
+      UPDATE "RunnerJob"
       SET "status" = CASE WHEN "status" = $4 THEN $5 ELSE "status" END,
           "startedAt" = COALESCE("startedAt", NOW()),
           "leaseExpiresAt" = NOW() + ($1 || ' milliseconds')::interval,
@@ -272,8 +423,8 @@ export class Service extends DatabaseService<Model> {
         leaseMs.toString(),
         data.jobId.toString(),
         data.agentId.toString(),
-        RunbookAgentJobStatus.Claimed,
-        RunbookAgentJobStatus.Running,
+        RunnerJobStatus.Claimed,
+        RunnerJobStatus.Running,
       ]),
     );
 
@@ -295,9 +446,9 @@ export class Service extends DatabaseService<Model> {
       throw new BadDataException("Database is not connected");
     }
 
-    const status: RunbookAgentJobStatus = data.success
-      ? RunbookAgentJobStatus.Succeeded
-      : RunbookAgentJobStatus.Failed;
+    const status: RunnerJobStatus = data.success
+      ? RunnerJobStatus.Succeeded
+      : RunnerJobStatus.Failed;
 
     /*
      * Only the agent that holds the lease is allowed to write the terminal
@@ -305,7 +456,7 @@ export class Service extends DatabaseService<Model> {
      * reclaimed), this is a no-op.
      */
     const sql: string = `
-      UPDATE "RunbookAgentJob"
+      UPDATE "RunnerJob"
       SET "status" = $1,
           "output" = $2,
           "exitCode" = $3,
@@ -326,8 +477,8 @@ export class Service extends DatabaseService<Model> {
         data.errorMessage ?? null,
         data.jobId.toString(),
         data.agentId.toString(),
-        RunbookAgentJobStatus.Claimed,
-        RunbookAgentJobStatus.Running,
+        RunnerJobStatus.Claimed,
+        RunnerJobStatus.Running,
       ]),
     );
 
@@ -380,7 +531,7 @@ export class Service extends DatabaseService<Model> {
 
       if (!job) {
         throw new BadDataException(
-          `RunbookAgentJob ${data.jobId.toString()} disappeared while waiting.`,
+          `RunnerJob ${data.jobId.toString()} disappeared while waiting.`,
         );
       }
 
@@ -392,7 +543,7 @@ export class Service extends DatabaseService<Model> {
 
       // Pending with claim deadline elapsed -> no agent picked it up.
       if (
-        job.status === RunbookAgentJobStatus.Pending &&
+        job.status === RunnerJobStatus.Pending &&
         job.claimDeadlineAt &&
         now > job.claimDeadlineAt
       ) {
@@ -405,8 +556,8 @@ export class Service extends DatabaseService<Model> {
 
       // Claimed/Running with lease elapsed -> agent went silent.
       if (
-        (job.status === RunbookAgentJobStatus.Claimed ||
-          job.status === RunbookAgentJobStatus.Running) &&
+        (job.status === RunnerJobStatus.Claimed ||
+          job.status === RunnerJobStatus.Running) &&
         job.leaseExpiresAt &&
         now > job.leaseExpiresAt
       ) {
@@ -445,7 +596,7 @@ export class Service extends DatabaseService<Model> {
      * state. The same agent's late-arriving result must lose to this race.
      */
     const sql: string = `
-      UPDATE "RunbookAgentJob"
+      UPDATE "RunnerJob"
       SET "status" = $1,
           "errorMessage" = $2,
           "completedAt" = NOW(),
@@ -456,13 +607,13 @@ export class Service extends DatabaseService<Model> {
       RETURNING "_id";
     `;
     await dataSource.query(sql, [
-      RunbookAgentJobStatus.TimedOut,
+      RunnerJobStatus.TimedOut,
       data.reason,
       data.jobId.toString(),
-      RunbookAgentJobStatus.Succeeded,
-      RunbookAgentJobStatus.Failed,
-      RunbookAgentJobStatus.TimedOut,
-      RunbookAgentJobStatus.Cancelled,
+      RunnerJobStatus.Succeeded,
+      RunnerJobStatus.Failed,
+      RunnerJobStatus.TimedOut,
+      RunnerJobStatus.Cancelled,
     ]);
 
     const updated: Model | null = await this.findOneById({
@@ -480,7 +631,7 @@ export class Service extends DatabaseService<Model> {
 
     if (!updated) {
       throw new BadDataException(
-        `RunbookAgentJob ${data.jobId.toString()} not found while timing out.`,
+        `RunnerJob ${data.jobId.toString()} not found while timing out.`,
       );
     }
 
@@ -498,21 +649,21 @@ export class Service extends DatabaseService<Model> {
     }
     try {
       await dataSource.query(
-        `UPDATE "RunbookAgentJob"
+        `UPDATE "RunnerJob"
          SET "status" = $1, "completedAt" = NOW(), "updatedAt" = NOW(), "version" = "version" + 1
          WHERE "runbookExecutionId" = $2::uuid
            AND "status" NOT IN ($3, $4, $5, $6)`,
         [
-          RunbookAgentJobStatus.Cancelled,
+          RunnerJobStatus.Cancelled,
           data.runbookExecutionId.toString(),
-          RunbookAgentJobStatus.Succeeded,
-          RunbookAgentJobStatus.Failed,
-          RunbookAgentJobStatus.TimedOut,
-          RunbookAgentJobStatus.Cancelled,
+          RunnerJobStatus.Succeeded,
+          RunnerJobStatus.Failed,
+          RunnerJobStatus.TimedOut,
+          RunnerJobStatus.Cancelled,
         ],
       );
     } catch (err) {
-      logger.error("Failed to cancel RunbookAgentJobs for execution");
+      logger.error("Failed to cancel RunnerJobs for execution");
       logger.error(err);
     }
   }

package/Server/Services/{RunbookAgentOwnerUserService.ts → RunnerOwnerTeamService.ts}

@@ -1,5 +1,5 @@
 import DatabaseService from "./DatabaseService";
-import Model from "../../Models/DatabaseModels/RunbookAgentOwnerUser";
+import Model from "../../Models/DatabaseModels/RunnerOwnerTeam";
 
 export class Service extends DatabaseService<Model> {
   public constructor() {

package/Server/Services/RunnerOwnerUserService.ts

@@ -0,0 +1,10 @@
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/RunnerOwnerUser";
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+}
+
+export default new Service();

package/Server/Services/RunnerService.ts

@@ -0,0 +1,166 @@
+import CreateBy from "../Types/Database/CreateBy";
+import { OnCreate } from "../Types/Database/Hooks";
+import DatabaseService from "./DatabaseService";
+import ObjectID from "../../Types/ObjectID";
+import Version from "../../Types/Version";
+import BadDataException from "../../Types/Exception/BadDataException";
+import Model, {
+  RunnerConnectionStatus,
+} from "../../Models/DatabaseModels/Runner";
+import OneUptimeDate from "../../Types/Date";
+import { JSONObject } from "../../Types/JSON";
+import QueryHelper from "../Types/Database/QueryHelper";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import { RUNNER_ALIVE_WINDOW_IN_MINUTES } from "../../Types/Runner/RunnerLiveStatus";
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    if (!createBy.data.key) {
+      createBy.data.key = ObjectID.generate().toString();
+    }
+
+    if (!createBy.data.agentVersion) {
+      createBy.data.agentVersion = new Version("1.0.0");
+    }
+
+    if (!createBy.data.connectionStatus) {
+      createBy.data.connectionStatus = RunnerConnectionStatus.Disconnected;
+    }
+
+    return { createBy, carryForward: [] };
+  }
+
+  @CaptureSpan()
+  public async findByIdAndKey(data: {
+    agentId: ObjectID;
+    agentKey: string;
+  }): Promise<Model | null> {
+    if (!data.agentId || !data.agentKey) {
+      return null;
+    }
+
+    return this.findOneBy({
+      query: {
+        _id: data.agentId.toString(),
+        key: data.agentKey,
+      },
+      select: {
+        _id: true,
+        projectId: true,
+        name: true,
+        canRunRunbooks: true,
+        canRunCodeFixTasks: true,
+        canRunAiCommands: true,
+      },
+      props: { isRoot: true },
+    });
+  }
+
+  /*
+   * Runners in this project that have opted into AI-composed commands and
+   * heartbeated recently. Same liveness semantics as the code-fix lookup:
+   * lastAlive as a query predicate, never a sort over possibly-NULL rows.
+   */
+  @CaptureSpan()
+  public async getOnlineAiCommandRunnersForProject(data: {
+    projectId: ObjectID;
+    limit?: number | undefined;
+  }): Promise<Array<Model>> {
+    return this.findBy({
+      query: {
+        projectId: data.projectId,
+        canRunAiCommands: true,
+        lastAlive: QueryHelper.greaterThan(
+          OneUptimeDate.getSomeMinutesAgo(RUNNER_ALIVE_WINDOW_IN_MINUTES),
+        ),
+      },
+      select: {
+        _id: true,
+        name: true,
+        description: true,
+        lastAlive: true,
+        hostInfo: true,
+      },
+      limit: data.limit ?? 25,
+      skip: 0,
+      props: { isRoot: true },
+    });
+  }
+
+  /*
+   * A Runner in this project that is allowed to take code-fix work and has
+   * heartbeated recently. Liveness is judged on lastAlive alone: unlike
+   * AIAgent, nothing ever flips a Runner's connectionStatus back to
+   * Disconnected, so a row that once connected would look Connected forever.
+   */
+  @CaptureSpan()
+  public async getOnlineCodeFixRunnerForProject(
+    projectId: ObjectID,
+  ): Promise<Model | null> {
+    /*
+     * The recency window is a QUERY predicate, not a post-filter on a sorted
+     * row: `ORDER BY "lastAlive" DESC` in Postgres puts NULLs FIRST, so a
+     * Runner row that was created but never started would be picked ahead of
+     * one that is heartbeating right now, and the project would read as
+     * having no agent.
+     */
+    const runners: Array<Model> = await this.findBy({
+      query: {
+        projectId: projectId,
+        canRunCodeFixTasks: true,
+        lastAlive: QueryHelper.greaterThan(
+          OneUptimeDate.getSomeMinutesAgo(RUNNER_ALIVE_WINDOW_IN_MINUTES),
+        ),
+      },
+      select: {
+        _id: true,
+        name: true,
+        lastAlive: true,
+      },
+      limit: 1,
+      skip: 0,
+      props: { isRoot: true },
+    });
+
+    return runners[0] || null;
+  }
+
+  @CaptureSpan()
+  public async heartbeat(data: {
+    agentId: ObjectID;
+    agentVersion?: Version | undefined;
+    hostInfo?: JSONObject | undefined;
+  }): Promise<void> {
+    if (!data.agentId) {
+      throw new BadDataException("agentId is required");
+    }
+
+    const update: JSONObject = {
+      lastAlive: OneUptimeDate.getCurrentDate(),
+      connectionStatus: RunnerConnectionStatus.Connected,
+    };
+
+    if (data.agentVersion) {
+      update["agentVersion"] = data.agentVersion;
+    }
+
+    if (data.hostInfo) {
+      update["hostInfo"] = data.hostInfo;
+    }
+
+    await this.updateOneById({
+      id: data.agentId,
+      data: update as never,
+      props: { isRoot: true },
+    });
+  }
+}
+
+export default new Service();

package/Server/Services/ScheduledMaintenanceStateService.ts

@@ -1,3 +1,4 @@
+import logger from "../Utils/Logger";
 import CreateBy from "../Types/Database/CreateBy";
 import DeleteBy from "../Types/Database/DeleteBy";
 import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
@@ -153,17 +154,30 @@ export class Service extends DatabaseService<ScheduledMaintenanceState> {
         newOrder = scheduledMaintenanceState.order! - 1;
       }
 
-      await this.updateOneBy({
-        query: {
-          _id: scheduledMaintenanceState._id!,
-        },
-        data: {
-          order: newOrder,
-        },
-        props: {
-          isRoot: true,
-        },
-      });
+      /*
+       * Concurrent deletes (e.g. Terraform destroying several items in
+       * parallel) can soft-delete a row between the findBy above and
+       * this update; save() would then INSERT with a null projectId and
+       * fail the whole delete with a 500. A row that vanished
+       * mid-rearrange needs no repositioning - skip it.
+       */
+      try {
+        await this.updateOneBy({
+          query: {
+            _id: scheduledMaintenanceState._id!,
+          },
+          data: {
+            order: newOrder,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+      } catch (err) {
+        logger.warn(
+          `rearrange: skipping row (likely deleted concurrently): ${err}`,
+        );
+      }
     }
   }