@kb-labs/workflow-engine 2.118.2 → 2.119.0-canary.0b1ec4151

package/dist/index.js

@@ -2,8 +2,8 @@ import { readFile, access, mkdir, writeFile, readdir } from 'fs/promises';
 import { resolve, join, basename } from 'path';
 import { parse, stringify } from 'yaml';
 import { WorkflowSpecSchema, resolveExpression, evaluateExpression } from '@kb-labs/workflow-contracts';
+import { assertTransition, WORKFLOW_REDIS_CHANNEL, EVENT_NAMES, IllegalStateTransitionError, isTerminal, IDEMPOTENCY_TTL_ENV, CONCURRENCY_TTL_ENV } from '@kb-labs/workflow-constants';
 import { randomUUID } from 'crypto';
-import { WORKFLOW_REDIS_CHANNEL, EVENT_NAMES, IDEMPOTENCY_TTL_ENV, CONCURRENCY_TTL_ENV } from '@kb-labs/workflow-constants';
 import { classifyFailure, decideRetry } from '@kb-labs/core-retry';
 import { createFileSystemArtifactClient } from '@kb-labs/workflow-artifacts';
 import { existsSync } from 'fs';
@@ -77,6 +77,47 @@ ${issues}`);
     return { spec, source };
   }
 };
+var LOCK_TTL_MS = 5e3;
+var LOCK_ACQUIRE_TIMEOUT_MS = 15e3;
+var LOCK_RETRY_BASE_MS = 20;
+var LOCK_RETRY_MAX_MS = 250;
+var LockAcquireTimeoutError = class extends Error {
+  constructor(lockKey) {
+    super(`Timed out waiting for lock: ${lockKey}`);
+    this.lockKey = lockKey;
+    this.name = "LockAcquireTimeoutError";
+  }
+  lockKey;
+};
+function sleep(ms) {
+  return new Promise((resolve4) => {
+    setTimeout(resolve4, ms);
+  });
+}
+async function withLock(cache, lockKey, fn) {
+  const token = randomUUID();
+  const deadline = Date.now() + LOCK_ACQUIRE_TIMEOUT_MS;
+  let delay = LOCK_RETRY_BASE_MS;
+  for (; ; ) {
+    const acquired = await cache.setIfNotExists(lockKey, token, LOCK_TTL_MS);
+    if (acquired) {
+      break;
+    }
+    if (Date.now() >= deadline) {
+      throw new LockAcquireTimeoutError(lockKey);
+    }
+    await sleep(delay + Math.random() * delay);
+    delay = Math.min(delay * 2, LOCK_RETRY_MAX_MS);
+  }
+  try {
+    return await fn();
+  } finally {
+    const current = await cache.get(lockKey);
+    if (current === token) {
+      await cache.delete(lockKey);
+    }
+  }
+}
 
 // src/state-store.ts
 var RUN_TTL_MS = 24 * 60 * 60 * 1e3;
@@ -118,16 +159,30 @@ var StateStore = class {
     const runIds = await this.cache.zrangebyscore("workflow:runs:index", -Infinity, Infinity);
     return runIds ?? [];
   }
+  /**
+   * Holds an exclusive per-run lock (see `withLock`) for the whole
+   * read-modify-write, so `mutator` runs exactly once per call — no other
+   * writer can observe or clobber the run in between. This is what actually
+   * fixes the original bug class (a job marked `failed` while its step was
+   * mid-write to `waiting_approval`): the two writes can no longer interleave.
+   *
+   * `mutator` must still not `await` — it runs synchronously against a
+   * single in-memory draft while the lock is held; async work belongs after
+   * `updateRun` resolves (and holding the lock across an `await` would just
+   * make every other writer to this run block on it needlessly).
+   */
   async updateRun(runId, mutator) {
-    const run = await this.getRun(runId);
-    if (!run) {
-      return null;
-    }
-    const draft = clone(run);
-    const result = mutator(draft);
-    const next = result ?? draft;
-    await this.saveRun(next);
-    return next;
+    return withLock(this.cache, `kb:lock:run:${runId}`, async () => {
+      const run = await this.getRun(runId);
+      if (!run) {
+        return null;
+      }
+      const draft = clone(run);
+      const result = mutator(draft);
+      const next = result ?? draft;
+      await this.saveRun(next);
+      return next;
+    });
   }
   async updateJob(runId, jobId, mutator) {
     let updatedJob = null;
@@ -177,9 +232,44 @@ var StateStore = class {
     });
     return updatedStep;
   }
+  /**
+   * Like `updateRun`, but validates the status transition against the
+   * workflow state machine before applying it — throws
+   * `IllegalStateTransitionError` (from `@kb-labs/workflow-constants`) if
+   * `to` is not reachable from the run's current status. `mutate` sets any
+   * *other* fields; it must not itself assign `.status` (this method owns
+   * that assignment, after the check).
+   */
+  async transitionRun(runId, to, mutate = () => {
+  }, options) {
+    return this.updateRun(runId, (draft) => {
+      assertTransition("run", draft.status, to, options);
+      draft.status = to;
+      mutate(draft);
+    });
+  }
+  /** Job-level counterpart of `transitionRun` — see its docblock. */
+  async transitionJob(runId, jobId, to, mutate = () => {
+  }, options) {
+    return this.updateJob(runId, jobId, (draft) => {
+      assertTransition("job", draft.status, to, options);
+      draft.status = to;
+      mutate(draft);
+    });
+  }
+  /** Step-level counterpart of `transitionRun` — see its docblock. */
+  async transitionStep(runId, jobId, stepId, to, mutate = () => {
+  }, options) {
+    return this.updateStep(runId, jobId, stepId, (draft) => {
+      assertTransition("step", draft.status, to, options);
+      draft.status = to;
+      mutate(draft);
+    });
+  }
   async releaseBlockedJobs(runId, completedJobName) {
-    const released = [];
+    let released = [];
     await this.updateRun(runId, (run) => {
+      released = [];
       for (const job of run.jobs) {
         if (job.status !== "queued" || !job.blocked) {
           continue;
@@ -520,29 +610,41 @@ var Scheduler = class {
       priority: entry.priority
     });
   }
+  /**
+   * The read (`zrangebyscore`) and the remove (`zrem`) below are two
+   * separate cache round-trips, not one atomic op — without the lock, two
+   * daemon instances racing this method could both read the same top entry
+   * before either removes it, and both would go on to execute the same job
+   * (the daemon runs multiple instances in production, so this is a live
+   * bug, not a theoretical one). `withLock` serializes dequeues against this
+   * one priority queue across every process sharing the same cache backend,
+   * the same way `StateStore.updateRun` serializes writes to one run.
+   */
   async dequeueFromPriority(priority) {
-    const now = Date.now();
     const key = `kb:jobqueue:${priority}`;
-    const results = await this.cache.zrangebyscore(
-      key,
-      0,
-      now + this.lookAheadMs
-    );
-    if (results.length === 0) {
-      return null;
-    }
-    const raw = results[0];
-    if (typeof raw !== "string") {
-      return null;
-    }
-    try {
-      const entry = JSON.parse(raw);
-      await this.cache.zrem(key, raw);
-      return entry;
-    } catch (error) {
-      this.logger.error("Failed to parse job queue entry", error instanceof Error ? error : void 0);
-      return null;
-    }
+    return withLock(this.cache, `kb:lock:queue:${priority}`, async () => {
+      const now = Date.now();
+      const results = await this.cache.zrangebyscore(
+        key,
+        0,
+        now + this.lookAheadMs
+      );
+      if (results.length === 0) {
+        return null;
+      }
+      const raw = results[0];
+      if (typeof raw !== "string") {
+        return null;
+      }
+      try {
+        const entry = JSON.parse(raw);
+        await this.cache.zrem(key, raw);
+        return entry;
+      } catch (error) {
+        this.logger.error("Failed to parse job queue entry", error instanceof Error ? error : void 0);
+        return null;
+      }
+    });
   }
   getDefaultPriority() {
     return this.defaultPriority;
@@ -777,14 +879,19 @@ var WorkflowEngine = class {
   }
   async cancelRun(runId) {
     const run = await this.getRun(runId);
-    if (!run || ["success", "failed", "cancelled", "skipped", "dlq"].includes(run.status)) {
+    if (!run) {
       return;
     }
-    await this.stateStore.updateRun(runId, (draft) => {
-      draft.status = "cancelled";
-      draft.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
-      return draft;
-    });
+    try {
+      await this.stateStore.transitionRun(runId, "cancelled", (draft) => {
+        draft.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
+      });
+    } catch (error) {
+      if (error instanceof IllegalStateTransitionError) {
+        return;
+      }
+      throw error;
+    }
     this.analytics?.track("workflow.run.cancelled", {
       runId,
       name: run?.name,
@@ -918,8 +1025,7 @@ var WorkflowEngine = class {
       this.logger.warn("Cannot mark job as failed: job not found", { runId, jobId });
       return;
     }
-    await this.stateStore.updateJob(runId, jobId, (draft) => {
-      draft.status = "failed";
+    await this.stateStore.transitionJob(runId, jobId, "failed", (draft) => {
       draft.error = {
         message: error.message,
         stack: error.stack,
@@ -970,12 +1076,19 @@ var WorkflowEngine = class {
         backoffMs
       });
       setTimeout(async () => {
-        await this.stateStore.updateJob(runId, jobId, (draft) => {
-          draft.status = "queued";
-          draft.error = void 0;
-          draft.startedAt = void 0;
-          draft.finishedAt = void 0;
-        });
+        try {
+          await this.stateStore.transitionJob(runId, jobId, "queued", (draft) => {
+            draft.error = void 0;
+            draft.startedAt = void 0;
+            draft.finishedAt = void 0;
+          });
+        } catch (transitionError) {
+          if (transitionError instanceof IllegalStateTransitionError) {
+            this.logger.info("Dropping stale job retry: job is no longer failed", { runId, jobId });
+            return;
+          }
+          throw transitionError;
+        }
         const updatedRun = await this.stateStore.getRun(runId);
         const updatedJob = updatedRun?.jobs.find((j) => j.id === jobId);
         if (updatedJob) {
@@ -998,8 +1111,7 @@ var WorkflowEngine = class {
     const released = await this.stateStore.releaseBlockedJobs(runId, failedJobName);
     for (const downstreamJob of released) {
       const now = (/* @__PURE__ */ new Date()).toISOString();
-      await this.stateStore.updateJob(runId, downstreamJob.id, (draft) => {
-        draft.status = "cancelled";
+      await this.stateStore.transitionJob(runId, downstreamJob.id, "cancelled", (draft) => {
         draft.blocked = false;
         draft.finishedAt = now;
         draft.error = { message: reason, timestamp: now };
@@ -1023,8 +1135,7 @@ var WorkflowEngine = class {
    * Interrupted jobs will be retried on next daemon startup.
    */
   async markJobInterrupted(runId, jobId) {
-    await this.stateStore.updateJob(runId, jobId, (draft) => {
-      draft.status = "interrupted";
+    await this.stateStore.transitionJob(runId, jobId, "interrupted", (draft) => {
       draft.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
     });
     this.logger.warn("Job interrupted", { runId, jobId });
@@ -1033,8 +1144,7 @@ var WorkflowEngine = class {
    * Mark job as started (running).
    */
   async markJobStarted(runId, jobId) {
-    await this.stateStore.updateJob(runId, jobId, (draft) => {
-      draft.status = "running";
+    await this.stateStore.transitionJob(runId, jobId, "running", (draft) => {
       draft.startedAt = (/* @__PURE__ */ new Date()).toISOString();
     });
     await this.stateStore.updateRun(runId, (draft) => {
@@ -1070,8 +1180,7 @@ var WorkflowEngine = class {
     const job = run?.jobs.find((j) => j.id === jobId);
     const startTime = job?.startedAt ? new Date(job.startedAt).getTime() : Date.now();
     const duration = Date.now() - startTime;
-    await this.stateStore.updateJob(runId, jobId, (draft) => {
-      draft.status = "success";
+    await this.stateStore.transitionJob(runId, jobId, "success", (draft) => {
       draft.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
     });
     this.logger.info("Job completed successfully", { runId, jobId });
@@ -1131,8 +1240,7 @@ var WorkflowEngine = class {
   /** Mark a job as skipped (success) and release any jobs blocked on it. */
   async skipJob(runId, job) {
     const now = (/* @__PURE__ */ new Date()).toISOString();
-    await this.stateStore.updateJob(runId, job.id, (draft) => {
-      draft.status = "success";
+    await this.stateStore.transitionJob(runId, job.id, "success", (draft) => {
       draft.startedAt = now;
       draft.finishedAt = now;
     });
@@ -1159,13 +1267,12 @@ var WorkflowEngine = class {
     }
     const allSuccess = run.jobs.every((j) => j.status === "success");
     const anyFailed = run.jobs.some((j) => j.status === "failed");
-    const anyRunning = run.jobs.some((j) => j.status === "running" || j.status === "queued");
-    if (anyRunning) {
+    const anyNonTerminal = run.jobs.some((j) => !isTerminal(j.status, "job"));
+    if (anyNonTerminal) {
       return;
     }
     if (allSuccess) {
-      const updated = await this.stateStore.updateRun(runId, (draft) => {
-        draft.status = "success";
+      const updated = await this.stateStore.transitionRun(runId, "success", (draft) => {
         draft.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
         draft.result = {
           ...draft.result ?? { status: "success" },
@@ -1193,8 +1300,7 @@ var WorkflowEngine = class {
       await this.reconcileChildInvocation(runId);
     } else if (anyFailed) {
       const failedJob = run.jobs.find((j) => j.status === "failed");
-      const updated = await this.stateStore.updateRun(runId, (draft) => {
-        draft.status = "failed";
+      const updated = await this.stateStore.transitionRun(runId, "failed", (draft) => {
         draft.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
         if (failedJob?.error) {
           draft.result = {
@@ -1231,8 +1337,7 @@ var WorkflowEngine = class {
    * Mark step as started (running).
    */
   async markStepStarted(runId, jobId, stepId) {
-    await this.stateStore.updateStep(runId, jobId, stepId, (draft) => {
-      draft.status = "running";
+    await this.stateStore.transitionStep(runId, jobId, stepId, "running", (draft) => {
       draft.startedAt = (/* @__PURE__ */ new Date()).toISOString();
     });
     this.logger.debug("Step started", { runId, jobId, stepId });
@@ -1247,8 +1352,7 @@ var WorkflowEngine = class {
    * Mark step as completed successfully with output.
    */
   async markStepCompleted(runId, jobId, stepId, output) {
-    await this.stateStore.updateStep(runId, jobId, stepId, (draft) => {
-      draft.status = "success";
+    await this.stateStore.transitionStep(runId, jobId, stepId, "success", (draft) => {
       draft.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
       if (output !== void 0) {
         draft.outputs = output;
@@ -1267,8 +1371,7 @@ var WorkflowEngine = class {
    * Mark step as failed with error.
    */
   async markStepFailed(runId, jobId, stepId, error, outputs) {
-    await this.stateStore.updateStep(runId, jobId, stepId, (draft) => {
-      draft.status = "failed";
+    await this.stateStore.transitionStep(runId, jobId, stepId, "failed", (draft) => {
       draft.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
       draft.error = {
         message: error.message,
@@ -1290,10 +1393,29 @@ var WorkflowEngine = class {
   /**
    * Mark step as waiting for human approval.
    */
+  /**
+   * Park a step waiting for human approval — and park its parent job with
+   * it, in the SAME atomic write (one `transitionJob` call touching both the
+   * job's own status and its nested step). Two separate writes (step then
+   * job) would leave a window where a reader could observe step=waiting but
+   * job=running; going through one call closes that window entirely, not
+   * just narrows it.
+   *
+   * The job-level `waiting_approval` status is what makes the daemon-restart
+   * exemption in `cleanupStaleRuns` structural: that force-fail loop only
+   * ever touches `running`/`queued` jobs, so a parked job is never in its
+   * blast radius — no bespoke "is this job actually abandoned or just
+   * waiting on a human" check needed there.
+   */
   async markStepWaitingApproval(runId, jobId, stepId) {
-    await this.stateStore.updateStep(runId, jobId, stepId, (draft) => {
-      draft.status = "waiting_approval";
-      draft.startedAt = draft.startedAt ?? (/* @__PURE__ */ new Date()).toISOString();
+    await this.stateStore.transitionJob(runId, jobId, "waiting_approval", (jobDraft) => {
+      const step = jobDraft.steps.find((s) => s.id === stepId);
+      if (!step) {
+        return;
+      }
+      assertTransition("step", step.status, "waiting_approval");
+      step.status = "waiting_approval";
+      step.startedAt = step.startedAt ?? (/* @__PURE__ */ new Date()).toISOString();
     });
     await this.events.publish({
       type: EVENT_NAMES.step.waitingApproval,
@@ -1303,14 +1425,21 @@ var WorkflowEngine = class {
     this.logger.info("Step waiting for approval", { runId, jobId, stepId });
   }
   /**
-   * Park a step while its child workflow runs. The worker returns after this
-   * transition, so parent workflows never consume the pool needed by children.
+   * Park a step (and its parent job — see `markStepWaitingApproval`'s
+   * docblock for why job+step move together in one write) while its child
+   * workflow runs. The worker returns after this transition, so parent
+   * workflows never consume the pool needed by children.
    */
   async markStepWaitingChild(runId, jobId, stepId, childRunId) {
-    await this.stateStore.updateStep(runId, jobId, stepId, (draft) => {
-      draft.status = "waiting_child";
-      draft.startedAt ??= (/* @__PURE__ */ new Date()).toISOString();
-      draft.metadata = { ...draft.metadata ?? {}, childRunId };
+    await this.stateStore.transitionJob(runId, jobId, "waiting_child", (jobDraft) => {
+      const step = jobDraft.steps.find((s) => s.id === stepId);
+      if (!step) {
+        return;
+      }
+      assertTransition("step", step.status, "waiting_child");
+      step.status = "waiting_child";
+      step.startedAt ??= (/* @__PURE__ */ new Date()).toISOString();
+      step.metadata = { ...step.metadata ?? {}, childRunId };
     });
     await this.events.publish({
       type: EVENT_NAMES.step.waitingChild,
@@ -1322,8 +1451,7 @@ var WorkflowEngine = class {
   }
   /** Re-queue a parked parent job after its child workflow reaches a terminal state. */
   async resumeJob(runId, jobId) {
-    const job = await this.stateStore.updateJob(runId, jobId, (draft) => {
-      draft.status = "queued";
+    const job = await this.stateStore.transitionJob(runId, jobId, "queued", (draft) => {
       draft.finishedAt = void 0;
     });
     if (job) {
@@ -1334,42 +1462,54 @@ var WorkflowEngine = class {
    * Resolve a pending approval — approve or reject.
    * On approve: marks step as success with approval outputs.
    * On reject: marks step as failed with rejection error.
+   *
+   * Resolves the step AND un-parks the job (`waiting_approval` → `queued`)
+   * in one atomic write — same reasoning as `markStepWaitingApproval` — then
+   * re-enqueues the job. This re-enqueue is what actually resumes execution:
+   * with approval no longer polled in-process (the worker parks and returns
+   * instead of waiting), nothing else will ever pick this job back up.
    */
   async resolveApproval(runId, jobId, stepId, action, data, comment) {
-    if (action === "approve") {
-      await this.stateStore.updateStep(runId, jobId, stepId, (draft) => {
-        draft.status = "success";
-        draft.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
-        draft.outputs = {
-          approved: true,
-          action,
-          ...comment ? { comment } : {},
-          ...data ?? {}
-        };
-      });
-      this.logger.info("Approval granted", { runId, jobId, stepId, comment });
-    } else {
-      await this.stateStore.updateStep(runId, jobId, stepId, (draft) => {
-        draft.status = "failed";
-        draft.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
-        draft.error = {
-          message: comment || "Approval rejected",
-          code: "APPROVAL_REJECTED"
-        };
-        draft.outputs = {
-          approved: false,
-          action,
-          ...comment ? { comment } : {},
-          ...data ?? {}
-        };
+    const run = await this.stateStore.getRun(runId);
+    const existingJob = run?.jobs.find((j) => j.id === jobId);
+    if (!run || !existingJob) {
+      throw new Error(`Cannot resolve approval: run or job not found (runId=${runId}, jobId=${jobId})`);
+    }
+    if (isTerminal(run.status, "run") || isTerminal(existingJob.status, "job")) {
+      throw new IllegalStateTransitionError("job", existingJob.status, "queued", {
+        reason: `run.status=${run.status}`
       });
-      this.logger.info("Approval rejected", { runId, jobId, stepId, comment });
     }
+    const stepStatus = action === "approve" ? "success" : "failed";
+    const outputs = {
+      approved: action === "approve",
+      action,
+      ...comment ? { comment } : {},
+      ...data ?? {}
+    };
+    const job = await this.stateStore.transitionJob(runId, jobId, "queued", (jobDraft) => {
+      const step = jobDraft.steps.find((s) => s.id === stepId);
+      if (!step) {
+        return;
+      }
+      assertTransition("step", step.status, stepStatus);
+      step.status = stepStatus;
+      step.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
+      step.outputs = outputs;
+      if (action === "reject") {
+        step.error = { message: comment || "Approval rejected", code: "APPROVAL_REJECTED" };
+      }
+      jobDraft.finishedAt = void 0;
+    });
+    this.logger.info(action === "approve" ? "Approval granted" : "Approval rejected", { runId, jobId, stepId, comment });
     await this.events.publish({
       type: EVENT_NAMES.step.updated,
       runId,
       payload: { jobId, stepId, action }
     });
+    if (job) {
+      await this.scheduler.enqueueJob(runId, job, job.priority ?? "normal");
+    }
   }
   /**
    * Get the state store for direct access (used by worker for gate restart-from).
@@ -1384,9 +1524,12 @@ var WorkflowEngine = class {
     return this.scheduler;
   }
   /**
-   * Mark stale running/queued runs as failed on daemon startup.
-   * Runs that were in-flight when the daemon crashed are unrecoverable —
-   * their executor process is gone, so we mark them failed immediately.
+   * Mark stale running/queued jobs as failed on daemon startup — their
+   * executor process is gone, so they're unrecoverable. The run itself is
+   * only finalized as 'failed' if nothing else could still complete it;
+   * a run with one abandoned job and one job legitimately parked on a human
+   * approval or a child workflow stays 'running' (only the abandoned job is
+   * failed) until the parked one resolves.
    */
   async cleanupStaleRuns() {
     const runIds = await this.stateStore.getAllRunIds();
@@ -1405,9 +1548,6 @@ var WorkflowEngine = class {
         if (run.status !== "running" && run.status !== "queued") {
           return;
         }
-        if (run.jobs.some((job) => job.steps.some((step) => step.status === "waiting_child"))) {
-          return;
-        }
         if (protectedChildRunIds.has(run.id)) {
           const hasRunningJob = run.jobs.some((job) => job.status === "running");
           if (hasRunningJob) {
@@ -1429,14 +1569,31 @@ var WorkflowEngine = class {
           }
           return;
         }
-        await this.stateStore.updateRun(runId, (draft) => {
-          draft.status = "failed";
+        const hasParkedJob = run.jobs.some((job) => job.status === "waiting_approval" || job.status === "waiting_child");
+        const abandonedJobIds = run.jobs.filter((job) => job.status === "running" || job.status === "queued").map((job) => job.id);
+        if (abandonedJobIds.length === 0) {
+          return;
+        }
+        if (hasParkedJob) {
+          await this.stateStore.updateRun(runId, (draft) => {
+            for (const job of draft.jobs) {
+              if (abandonedJobIds.includes(job.id)) {
+                job.status = "failed";
+                job.error = { message: "Daemon restarted \u2014 run was abandoned" };
+                job.finishedAt = now;
+              }
+            }
+          });
+          count++;
+          return;
+        }
+        await this.stateStore.transitionRun(runId, "failed", (draft) => {
           draft.finishedAt = now;
           if (draft.startedAt) {
             draft.durationMs = new Date(now).getTime() - new Date(draft.startedAt).getTime();
           }
           for (const job of draft.jobs) {
-            if (job.status === "running" || job.status === "queued") {
+            if (abandonedJobIds.includes(job.id)) {
               job.status = "failed";
               job.error = { message: "Daemon restarted \u2014 run was abandoned" };
               job.finishedAt = now;
@@ -1467,8 +1624,7 @@ var WorkflowEngine = class {
         await Promise.all(
           interruptedJobs.map(async (job) => {
             this.logger.info("Resuming interrupted job", { runId, jobId: job.id });
-            await this.stateStore.updateJob(runId, job.id, (draft) => {
-              draft.status = "queued";
+            await this.stateStore.transitionJob(runId, job.id, "queued", (draft) => {
               draft.startedAt = void 0;
               draft.finishedAt = void 0;
             });
@@ -1523,9 +1679,8 @@ var WorkflowEngine = class {
     return updated;
   }
   async finalizeRun(runId, status, context = {}) {
-    const updated = await this.stateStore.updateRun(runId, (run) => {
+    const updated = await this.stateStore.transitionRun(runId, status, (run) => {
       const now = (/* @__PURE__ */ new Date()).toISOString();
-      run.status = status;
       run.finishedAt = now;
       run.durationMs = computeDurationMs(run.startedAt ?? run.queuedAt, now);
       if (context.jobs) {