akm-cli 0.9.17-alpha.3 → 0.9.17-alpha.5

package/dist/workflows/exec/run-workflow.js

@@ -2,28 +2,27 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 /**
- * Engine-driven workflow execution — the `akm workflow run`
- * start/resume/execute path, and the single execution surface for a run: akm
- * walks the frozen plan and dispatches every unit itself. Every step
- * advances through `completeWorkflowStep` (never a direct step-row write),
- * the plan is read from its frozen `plan_json` row rather than live source,
- * a run lease enforces one driving engine invocation at a time, gate loops
- * are bounded, and the SDK dispatch registry is drained in a `finally` on
- * every exit path so no child process keeps the event loop open.
- *
- * See docs/architecture/decisions/0011-engine-run-loop-invariants.md for the
- * full design history behind each of these invariants.
+ * Engine-driven workflow execution — the `akm workflow run` start/resume/
+ * execute path. akm walks the frozen plan (read from `plan_json`, never live
+ * source) and dispatches every unit itself; every step advances through
+ * `completeWorkflowStep`; one O_EXCL lock file per run keeps a second driver
+ * off (a dead holder's lock is reclaimed); gate loops are bounded; and the
+ * SDK dispatch registry is drained on every exit so the process can exit.
+ * See docs/architecture/decisions/0011-engine-run-loop-invariants.md.
  */
-import { randomUUID } from "node:crypto";
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
 import { TransientError, UsageError } from "../../core/errors.js";
-import { withMaintenanceStartBarrierAsync } from "../../core/maintenance-barrier.js";
+import { releaseLock } from "../../core/file-lock.js";
+import { formatLockHolderPid, tryAcquireRunLock } from "../../core/run-lock.js";
 import { disposeDispatchResources } from "../../integrations/agent/runner-dispatch.js";
+import { resolveStorageLocations } from "../../storage/locations.js";
 import { withWorkflowRunsConnection, withWorkflowRunsRepo } from "../../storage/repositories/workflow-runs-repository.js";
-import { assertRunParamsSatisfyPlan } from "../ir/params.js";
 import { computePlanHash } from "../ir/plan-hash.js";
-import { decodeWorkflowPlanV4 } from "../ir/schema-v4.js";
-import { requireExecutableWorkflowPlan } from "../runtime/plan-classifier.js";
-import { completeWorkflowStep, getNextWorkflowStep, resumeWorkflowRun } from "../runtime/runs.js";
+import { decodeWorkflowPlan, readRunPlan } from "../runtime/run-plan.js";
+import { abandonWorkflowRun, completeWorkflowStep, getNextWorkflowStep, resumeWorkflowRun, } from "../runtime/runs.js";
+import { loadWorkflowAsset } from "../runtime/workflow-asset-loader.js";
 import { frozenSummaryJudge } from "./frozen-judge.js";
 import { mergeLoweringNotices } from "./lowering-notices.js";
 import { defaultUnitDispatcher, executeStepPlan, } from "./native-executor.js";
@@ -62,7 +61,13 @@ export async function runWorkflowSteps(options) {
         stepsProcessed += result.stepsProcessed;
         const notices = mergeLoweringNotices(...executed.map((step) => step.notices));
         const aggregate = { ...result, executed, stepsProcessed, ...(notices ? { notices } : {}) };
-        if (result.run.status !== "failed" || result.aborted || result.gateRejection || remainingRetries <= 0) {
+        // An attempt that executed nothing (an abandoned, undecodable plan) has no
+        // failed step for a retry to re-open.
+        if (result.run.status !== "failed" ||
+            result.aborted ||
+            result.gateRejection ||
+            result.executed.length === 0 ||
+            remainingRetries <= 0) {
             return aggregate;
         }
         if (remainingSteps !== undefined) {
@@ -86,16 +91,6 @@ async function runWorkflowAttempt(options, liveEvidence) {
         parameterFlags: options.parameterFlags,
         newRun: options.newRun,
     });
-    // Version/canonical/hash validation precedes every executable mutation,
-    // including lease acquisition. Historical rows remain inspectable/abandonable.
-    if (!next.done) {
-        await withWorkflowRunsRepo((repo) => {
-            const row = repo.getRunById(next.run.id);
-            if (!row)
-                throw new UsageError(`Workflow run ${next.run.id} was not found.`);
-            requireExecutableWorkflowPlan(row);
-        });
-    }
     // Refuse non-active runs BEFORE any dispatch — completeWorkflowStep would
     // reject the completion anyway, but only after the units already ran (and
     // cost money). Mirror its preflight up front.
@@ -103,70 +98,68 @@ async function runWorkflowAttempt(options, liveEvidence) {
         throw new UsageError(`Workflow run ${next.run.id} is ${next.run.status} and cannot be executed. ` +
             `Use \`akm workflow resume ${next.run.id}\` to reopen it first.`);
     }
-    // Run lease (R2 single-driver enforcement): claim the run BEFORE any
-    // dispatch — a second `akm workflow run` on a live-leased run refuses up
-    // front instead of racing the first engine's spine. An expired lease is
-    // claimable (crash recovery). Released in the finally below; renewed
-    // between steps inside the loop. A done run takes no lease: nothing will
-    // dispatch, and the status re-read below must stay a pure no-op.
+    // One driver per run: the per-run lock file is taken BEFORE the plan is
+    // read or anything dispatches, so a second `akm workflow run` on a run
+    // another process is driving refuses up front (exit 75) instead of racing
+    // its spine. A done run takes no lock: nothing will dispatch.
     const runId = next.run.id;
-    const leaseHolder = randomUUID();
-    const leased = !next.done;
-    if (leased) {
-        await acquireRunLease(runId, leaseHolder);
-    }
-    // Lease heartbeat (P1 fix): the lease TTL is renewed BETWEEN steps, but a
-    // single unit's dispatch can outlive the TTL (the default unit timeout is 10
-    // minutes, > the 90s lease). An unheartbeated lease would silently expire
-    // mid-dispatch, letting a second `akm workflow run` claim the run and
-    // re-dispatch the same units — the two engines clobber each other's journal
-    // rows and double-run side effects. A timer INSIDE this invocation renews the
-    // lease while dispatch is in flight; it is cleared in the `finally`, so it
-    // dies with the process — exactly when the lease SHOULD become claimable
-    // after TTL. A renewal that fails (the lease was genuinely stolen after an
-    // expiry, e.g. the process was suspended) aborts dispatch and fails the run
-    // loudly rather than keep double-driving.
-    const heartbeat = leased
-        ? new LeaseHeartbeat(runId, leaseHolder, options.heartbeatScheduler, options.signal)
-        : undefined;
-    heartbeat?.start();
+    const lock = next.done ? undefined : acquireWorkflowRunLock(runId);
     try {
-        // Run-wide state.db connection scope: `executeStepPlan` already opens one
-        // per STEP, so widening it to the whole drive loop additionally folds the
-        // spine writes (`completeWorkflowStep`), the per-step lease renewals, the
-        // journal reads, and `finalizeExecutedStep`'s gate-row journaling onto that
-        // one handle — `openStateDatabase` costs a maintenance-activity lockfile
-        // plus a read-only ledger preflight on EVERY call. Nesting is an idempotent
-        // join (`core/state-db-scope.ts`): the inner per-step scope reuses this
-        // handle and does not close it, and this scope's own `finally` closes on
-        // every exit path (return, throw, abort), after which escaped async work
-        // transparently falls back to opening its own connection.
-        const result = await withWorkflowRunsConnection(() => driveRun(options, next, leaseHolder, heartbeat, liveEvidence));
-        // Creation-time notices reach the caller only here: the run row has no
-        // warnings column, and a later invocation of the same run must stay silent
-        // about a decision it did not make. `driveRun` never sets `warnings` or
-        // `resumed` — both are properties of THIS resolution of `target`, not of
-        // the run row (#919).
+        let plan;
+        const warnings = [...(next.startWarnings ?? [])];
+        if (!next.done) {
+            const row = await withWorkflowRunsRepo((repo) => repo.getRunById(runId));
+            if (!row)
+                throw new UsageError(`Workflow run ${runId} was not found.`);
+            const read = readRunPlan(row);
+            if (!read.ok) {
+                // A newer akm's run is left untouched for that akm.
+                if (read.newer)
+                    throw new UsageError(read.problem);
+                // Otherwise a frozen plan this akm cannot decode is a status change,
+                // not an exception: the run is abandoned and the message names how to
+                // start afresh.
+                const abandoned = await abandonWorkflowRun(runId);
+                const message = `${read.problem} The run was abandoned; start a new run with 'akm workflow run ${row.workflow_ref}'.`;
+                return {
+                    run: abandoned.run,
+                    executed: [],
+                    stepsProcessed: 0,
+                    warnings: [...warnings, message],
+                    ...(next.resumed ? { resumed: true } : {}),
+                };
+            }
+            plan = read.plan;
+            if (!next.autoStarted) {
+                const drift = await workflowSourceDriftWarning(runId, row.workflow_ref, plan);
+                if (drift)
+                    warnings.push(drift);
+            }
+        }
+        // One state.db connection scope for the whole drive loop (the per-step
+        // scopes nest into it); each `openStateDatabase` is not free.
+        const result = await withWorkflowRunsConnection(() => driveRun(options, next, plan, liveEvidence));
+        // Creation/resume-time notices reach the caller only here: the run row
+        // has no warnings column, and a later invocation of the same run must
+        // stay silent about a decision it did not make. `driveRun` never sets
+        // `warnings` or `resumed` — both are properties of THIS resolution of
+        // `target`, not of the run row (#919).
         return {
             ...result,
             ...(next.resumed ? { resumed: true } : {}),
-            ...(next.startWarnings?.length ? { warnings: next.startWarnings } : {}),
+            ...(warnings.length > 0 ? { warnings } : {}),
         };
     }
     finally {
-        heartbeat?.stop();
         try {
-            if (leased) {
-                await withWorkflowRunsRepo((repo) => {
-                    repo.releaseEngineLease(runId, leaseHolder);
-                });
-            }
+            if (lock)
+                releaseLock(lock);
         }
         finally {
             // Process-lifecycle drain (owner finding 4): release any cached SDK server
             // child processes so a one-shot CLI invocation exits cleanly instead of
-            // hanging on the leaked handle. Runs even if lease release itself fails;
-            // a teardown-time repository error must not skip dispatch cleanup.
+            // hanging on the leaked handle. Runs even if the lock release itself
+            // fails; a teardown-time error must not skip dispatch cleanup.
             try {
                 await (options.disposeDispatchResources ?? disposeDispatchResources)();
             }
@@ -176,150 +169,54 @@ async function runWorkflowAttempt(options, liveEvidence) {
         }
     }
 }
-/** Lease lifetime: long enough to survive slow steps between renewals, short
- * enough that a crashed engine frees the run quickly. Renewed per step. */
-const RUN_LEASE_TTL_MS = 90_000;
-function leaseExpiry() {
-    return new Date(Date.now() + RUN_LEASE_TTL_MS).toISOString();
-}
-/**
- * Atomically claim the run lease or refuse with a TransientError naming the
- * current holder + expiry (#948 addendum — moved off UsageError, exit 75).
- * The single-UPDATE claim in the repository is the arbiter — two racing
- * invocations cannot both win.
- */
-async function acquireRunLease(runId, holder) {
-    await withMaintenanceStartBarrierAsync(() => withWorkflowRunsRepo((repo) => {
-        if (repo.acquireEngineLease(runId, holder, leaseExpiry(), new Date().toISOString()))
-            return;
-        const row = repo.getRunById(runId);
-        throw new TransientError(`Workflow run ${runId} is already being driven by engine ${row?.engine_lease_holder ?? "(unknown)"} ` +
-            `(run lease expires ${row?.engine_lease_until ?? "(unknown)"}). A second \`akm workflow run\` would race it — ` +
-            `wait for that invocation to finish or for the lease to expire.`, "RUN_LEASE_HELD");
-    }));
+/** The per-run lock file: `<data dir>/workflow-run-locks/<run id>.lock`, next to `state.db`. */
+export function workflowRunLockPath(runId) {
+    return path.join(path.dirname(resolveStorageLocations().stateDb), "workflow-run-locks", `${runId}.lock`);
 }
 /**
- * Renew the lease between steps. Losing the lease mid-run (it expired during
- * a long step and another engine claimed it) is a hard stop: the new owner
- * drives the spine now, and continuing would race it.
+ * Take the run's O_EXCL lock file, or refuse with `RUN_LEASE_HELD` (exit 75)
+ * naming the live holder. A lock whose holder pid is dead is reclaimed by
+ * `tryAcquireRunLock` before this refuses, so a crashed engine never wedges
+ * a run; nothing here expires by age.
  */
-async function renewRunLease(runId, holder) {
-    await withWorkflowRunsRepo((repo) => {
-        if (repo.renewEngineLease(runId, holder, leaseExpiry()))
-            return;
-        const row = repo.getRunById(runId);
-        throw new UsageError(`Workflow run ${runId} lost its run lease (now held by ${row?.engine_lease_holder ?? "(nobody)"}). ` +
-            `Another engine invocation claimed the run after this one's lease expired — stopping to avoid racing it.`);
-    });
-}
-/** Renew mid-dispatch this often. Well under the TTL so a slow/skipped tick
- * still leaves ample margin before the lease would expire. */
-const HEARTBEAT_INTERVAL_MS = RUN_LEASE_TTL_MS / 3;
-/** Real timer: an unref'd interval so a live heartbeat never keeps the process alive. */
-function defaultHeartbeatScheduler(tick) {
-    const id = setInterval(() => void tick(), HEARTBEAT_INTERVAL_MS);
-    id.unref?.();
-    return () => clearInterval(id);
+function acquireWorkflowRunLock(runId) {
+    const result = tryAcquireRunLock(workflowRunLockPath(runId), { label: `workflow run ${runId}` });
+    if (result.state === "acquired")
+        return result.ownership;
+    const since = result.holder.startedAt ? `, since ${result.holder.startedAt}` : "";
+    throw new TransientError(`Workflow run ${runId} is already being driven by another akm process ` +
+        `(pid ${formatLockHolderPid(result.holder)}${since}). A second \`akm workflow run\` would race it — ` +
+        "wait for that invocation to finish.", "RUN_LEASE_HELD");
 }
 /**
- * Keeps the run lease alive while a step dispatches (P1 fix — the between-step
- * renewal cannot cover a unit that runs longer than the TTL). A timer inside
- * the engine invocation renews the lease through the holder-guarded
- * {@link renewEngineLease}; the heartbeat owns an {@link AbortController}
- * (chained onto the caller's signal) that becomes the effective DISPATCH
- * signal, so a lost lease aborts in-flight dispatch PROMPTLY. After the abort,
- * {@link assertAlive} throws a loud UsageError, so the engine stops instead of
- * continuing to drive a run another engine now owns. No background daemon: the
- * timer is cleared in the caller's `finally` and dies with the process.
+ * Resume re-reads the authored workflow source and says so, once, when it no
+ * longer matches the bytes the plan was frozen from. The run keeps executing
+ * the frozen plan either way — that is the whole safety the freeze buys — and
+ * the warning tells the operator a fresh run is what picks up the edit.
  */
-class LeaseHeartbeat {
-    runId;
-    holder;
-    controller = new AbortController();
-    detachUpstream;
-    schedule;
-    cancel;
-    renewing = false;
-    /** Set once a renewal failed — the lease was stolen after a genuine expiry. */
-    lost = false;
-    /** The holder that stole the lease, captured for the loud error. */
-    stolenBy = null;
-    constructor(runId, holder, scheduler, upstream) {
-        this.runId = runId;
-        this.holder = holder;
-        this.schedule = scheduler ?? defaultHeartbeatScheduler;
-        // A caller abort (Ctrl-C, budget) must abort dispatch too; chain it into
-        // the effective signal. Distinct from a lost lease: a caller abort does
-        // NOT set `lost`, so `assertAlive` stays quiet and the existing graceful
-        // break on `options.signal` handles it.
-        if (upstream) {
-            if (upstream.aborted) {
-                this.controller.abort();
-            }
-            else {
-                const onAbort = () => this.controller.abort();
-                upstream.addEventListener("abort", onAbort, { once: true });
-                this.detachUpstream = () => upstream.removeEventListener("abort", onAbort);
-            }
-        }
-    }
-    /** The effective dispatch signal: aborts on a lost lease OR a caller abort. */
-    get signal() {
-        return this.controller.signal;
-    }
-    start() {
-        this.cancel ??= this.schedule(() => this.tick());
-    }
-    /** One renewal attempt. A failure marks the lease lost and aborts dispatch. */
-    async tick() {
-        if (this.lost || this.renewing || this.controller.signal.aborted)
-            return;
-        this.renewing = true;
-        try {
-            const renewed = await withWorkflowRunsRepo((repo) => repo.renewEngineLease(this.runId, this.holder, leaseExpiry()));
-            if (!renewed) {
-                this.stolenBy = await withWorkflowRunsRepo((repo) => repo.getRunById(this.runId)?.engine_lease_holder ?? null);
-                this.loseLease();
-            }
-        }
-        catch {
-            // A renewal that THREW (a DB error / connection failure, or the follow-up
-            // getRunById itself throwing) is treated exactly like a stolen lease: we
-            // can no longer PROVE we still hold it, so abort in-flight dispatch and let
-            // `assertAlive` stop the engine loudly. Swallowing the error here is what
-            // keeps the fire-and-forget `void tick()` in the default scheduler from
-            // leaking an unhandled promise rejection.
-            this.loseLease();
-        }
-        finally {
-            this.renewing = false;
-        }
+async function workflowSourceDriftWarning(runId, workflowRef, plan) {
+    if (!plan.sourceHash)
+        return undefined;
+    let sourcePath;
+    try {
+        sourcePath = (await loadWorkflowAsset(workflowRef)).path;
     }
-    /** Mark the lease lost, stop the timer, and abort in-flight dispatch — the new
-     * owner drives the spine now (or, on a renewal error, we can no longer prove we
-     * do). Idempotent: repeated calls are harmless. */
-    loseLease() {
-        this.lost = true;
-        this.stop();
-        this.controller.abort();
+    catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        return (`Workflow run ${runId}: the authored source ${workflowRef} could not be re-read (${detail}); ` +
+            "continuing with the frozen plan.");
     }
-    /**
-     * Throw loudly if a heartbeat renewal failed. Called at dispatch boundaries:
-     * a lost lease means another engine claimed the run mid-step, so continuing
-     * (completing steps, dispatching more units) would double-drive it.
-     */
-    assertAlive() {
-        if (!this.lost)
-            return;
-        throw new UsageError(`Workflow run ${this.runId} lost its run lease mid-dispatch (heartbeat renewal failed; lease now held by ` +
-            `${this.stolenBy ?? "(nobody)"}). Another engine invocation claimed the run after this one's lease expired — ` +
-            `aborting to avoid double-driving it.`);
+    let current;
+    try {
+        current = createHash("sha256").update(fs.readFileSync(sourcePath)).digest("hex");
     }
-    stop() {
-        this.cancel?.();
-        this.cancel = undefined;
-        this.detachUpstream?.();
+    catch {
+        return `Workflow run ${runId}: ${sourcePath} is no longer readable; continuing with the frozen plan.`;
     }
+    if (current === plan.sourceHash)
+        return undefined;
+    return (`Workflow run ${runId}: ${workflowRef} (${sourcePath}) has changed since this run was frozen; ` +
+        `continuing with the frozen plan. Start a new run with 'akm workflow run ${workflowRef} --new' to pick up the edit.`);
 }
 /**
  * A terminal run is a pure no-op: do not load or integrity-check its frozen
@@ -337,31 +234,14 @@ async function completedRunResult(runId) {
 function workflowSummaryJudge(options, stepPlan, signal, owner) {
     if (options.summaryJudge !== undefined)
         return options.summaryJudge;
-    // The judge dispatches under the REAL run/step identity; the per-loop gate row
-    // identity is threaded in per call by the completion path that journals it.
-    // eventSource (gap closed, code review): the judge's dispatch is a "command"
-    // request like any other exec/agent/sdk unit, so it goes through the same
-    // provenance thread `executeStepSubgraph` uses below — undefined for every
-    // non-task caller, byte-identical.
+    // The judge dispatches under the real run/step identity and the same event source as units.
     return frozenSummaryJudge(stepPlan.gate.frozenJudge, signal, options.dispatcher ?? defaultUnitDispatcher, owner, options.eventSource);
 }
 /**
- * Seed the declared budget ceilings from the journal so they are truly
- * per-RUN: a resumed or re-invoked run must not restart a declared `budget`
- * at zero. The append-only attempt journal is authoritative: dispatch
- * attempts count against `budget.max_units`, and their known tokens count
- * against `budget.max_tokens`. Durable result reuse is free.
- *
- * Gate-evaluation rows (`phase = "gate"`, journaled by the completion-gate
- * judge) are EXCLUDED from the seed: the live path never consumes
- * DispatchBudget for a judge call, so counting its journal row on resume
- * would make an interrupted run hit `max_units` (and the lifetime cap)
- * earlier than the identical uninterrupted run — a spurious hard failure
- * that `on_error` cannot soften. The seed must reproduce exactly what live
- * accounting would have accumulated.
- *
- * Attempt rows are append-only, so retries cannot collapse or erase prior
- * dispatch accounting.
+ * Seed the run's budget from the append-only attempt journal so a resumed run
+ * does not restart it at zero: dispatch attempts count against `max_units`,
+ * their tokens against `max_tokens`. Gate-evaluation rows are excluded, as the
+ * live path never charges a judge call.
  */
 async function seedRunAccountingFromJournal(runId) {
     const accounting = await withWorkflowRunsRepo((repo) => repo.getAttemptAccounting(runId));
@@ -371,22 +251,15 @@ async function seedRunAccountingFromJournal(runId) {
     };
 }
 /**
- * The decoded/hash-verified row plan is the sole execution authority. The
- * loader seam may assert an expected plan in tests, but can never replace it.
- *
- * Reviewer #12: the journaled params row must still satisfy the frozen param
- * schemas before the engine resolves any unit prompt from it, so
- * schema-violating params — post-start corruption — fail loudly BEFORE any
- * unit is dispatched (start already validated the params it stored).
+ * The row plan is the sole execution authority. The loader seam may assert an
+ * expected plan in tests, but can never replace it.
  */
-async function loadAuthoritativeRunPlan(options, next) {
-    const stored = await loadStoredPlan(next.run.id);
+async function loadAuthoritativeRunPlan(options, next, stored) {
     if (options.loadPlan) {
-        const expected = decodeWorkflowPlanV4(await options.loadPlan(next.run.workflowRef));
+        const expected = decodeWorkflowPlan(await options.loadPlan(next.run.workflowRef));
         if (computePlanHash(expected) !== computePlanHash(stored))
             throw new UsageError(`Injected workflow plan for run ${next.run.id} differs from its frozen plan.`);
     }
-    assertRunParamsSatisfyPlan(next.run.id, stored, next.run.params ?? {});
     return stored;
 }
 /**
@@ -395,8 +268,8 @@ async function loadAuthoritativeRunPlan(options, next) {
  * Returns the re-read spine state so the caller can continue its walk.
  */
 async function skipUnselectedRouteTarget(input) {
-    const { runId, stepId, stepPlan, skipInfo, routeUnselected, executed, leaseHolder } = input;
-    // Cascade (peer review R1): a skipped step that is ITSELF a router
+    const { runId, stepId, stepPlan, skipInfo, routeUnselected, executed } = input;
+    // Cascade: a skipped step that is ITSELF a router
     // never evaluates its route, so none of its declared targets were
     // selected — mark them all skip-on-reach too (a target another
     // completed router selects stays protected via routeSelected). Without
@@ -408,27 +281,13 @@ async function skipUnselectedRouteTarget(input) {
         ? `Skipped by route: step "${skipInfo.router}" was itself skipped, so none of its branch targets run.`
         : `Skipped by route: step "${skipInfo.router}" selected "${skipInfo.selected}".`;
     executed.push({ stepId, ok: true, unitCount: 0, failedUnits: 0, summary: notes });
-    await completeWorkflowStep({ runId, stepId, status: "skipped", notes, leaseHolder });
+    await completeWorkflowStep({ runId, stepId, status: "skipped", notes });
     return getNextWorkflowStep(runId);
 }
 /**
- * Crash-resume gate state (Codex P1): SEED the starting gate loop from the
- * journal through the SAME shared helpers the first pass used — no fork.
- * A run interrupted after a rejected gate was journaled
- * (`<step>.gate:l<n>`, complete:false) must resume at loop n+1 with the
- * stored corrective feedback threaded into the unit prompts; without this
- * the engine restarts at loop 1, reuses the rejected loop-1 rows, overwrites
- * `<step>.gate:l1`, and re-judges the stale artifact — breaking journaled
- * replay and making the resumed run diverge from the interrupted one. The rows
- * are re-read per step (NOT the once-at-start budget seed) so a step reached
- * later within THIS same invocation still starts fresh at loop 1.
- *
- * Only the STEP's rows are read (index-backed on `(run_id, step_id)`): both
- * helpers already discard every row carrying a different `step_id`, and gate
- * rows are journaled under the step's own id, so the narrow query returns a
- * superset of what they read. Re-reading the whole run journal here would
- * re-materialize every earlier step's `result_json` — synchronously, blocking
- * the event loop the lease heartbeat and abort handling share — once per step.
+ * Seed a step's starting gate loop and feedback from its journaled gate rows,
+ * so a run interrupted after a rejection resumes at the next loop with the
+ * stored feedback instead of re-judging loop 1. Reads only this step's rows.
  */
 async function recoverGateLoopState(runId, stepPlan) {
     // A step with no effective completion criteria never reaches a judge
@@ -443,13 +302,8 @@ async function recoverGateLoopState(runId, stepPlan) {
     return { startLoop, seededFeedback: recoverGateFeedback(stepJournal, stepId, startLoop) };
 }
 /**
- * The kinds that FINISHED the step (completed / failed / gate-exhausted) — the
- * ONE `maxSteps` consumption for its whole gate loop. An abort and a judge
- * outage leave the step unfinished and consume nothing: the next invocation
- * still owes the work. A blocked child is the SAME shape as a judge outage
- * (P3b §3.4) — a gate is a gate for a child workflow too, so it is likewise
- * NOT in this set: `akm workflow resume` is what clears it, never an
- * automatic in-step re-dispatch.
+ * The kinds that finished the step — its one `maxSteps` consumption. An abort,
+ * a judge outage, and a blocked child consume nothing: the work is still owed.
  */
 const STEP_FINISHED_KINDS = new Set(["advanced", "failed", "gate-exhausted"]);
 /**
@@ -459,7 +313,7 @@ const STEP_FINISHED_KINDS = new Set(["advanced", "failed", "gate-exhausted"]);
  * native executor.
  */
 async function executeStepSubgraph(ctx, loop) {
-    const { options, next, plan, stepPlan, step, evidence, leaseHolder, dispatchSignal } = ctx;
+    const { options, next, plan, stepPlan, step, evidence } = ctx;
     const { gateLoop, gateFeedback, unitsDispatched, tokensUsed } = loop;
     return !stepPlan.root && stepPlan.route
         ? {
@@ -471,42 +325,32 @@ async function executeStepSubgraph(ctx, loop) {
         }
         : await executeStepPlan(stepPlan, {
             runId: next.run.id,
-            leaseHolder,
             workflowRef: next.run.workflowRef,
             params: next.run.params ?? {},
             evidence,
             unitsDispatched,
             tokensUsed,
-            // Budget ceilings ride the FROZEN plan (addendum R2): a mid-run
+            // Budget ceilings ride the FROZEN plan: a mid-run
             // asset edit can never loosen or tighten a run's budget.
             ...(plan.budget ? { budget: plan.budget } : {}),
             gateLoop,
             ...(gateFeedback ? { gateFeedback } : {}),
-            // F-1 (spec §5.2 point 2): threaded to an exec unit's child env;
+            // F-1: threaded to an exec unit's child env;
             // undefined for every non-task caller (byte-identical, RunWorkflowOptions doc).
             ...(options.eventSource !== undefined ? { eventSource: options.eventSource } : {}),
-            // The heartbeat's signal is the effective dispatch signal: a lost
-            // lease (or a caller abort) aborts in-flight units promptly.
-            ...(dispatchSignal ? { signal: dispatchSignal } : {}),
+            ...(options.signal ? { signal: options.signal } : {}),
             ...(options.dispatcher ? { dispatcher: options.dispatcher } : {}),
             maxConcurrency: Math.min(options.maxConcurrency ?? Number.POSITIVE_INFINITY, plan.execution?.maxConcurrency ?? 1),
         });
 }
 /**
- * Drive ONE step's bounded gate loop (addendum R2, `gate.max_loops`): loop 1 is
- * the normal execution; a gate rejection with attempts left re-executes the
- * subgraph with the judge's feedback threaded into unit prompts.
- *
- * The engine owns only the loop control the shared completion path
- * (`finalizeExecutedStep`) maps onto — retry re-executes; advanced moves on;
- * failure/judge-failure/exhaustion stops this invocation — and returns that
- * decision plus the running budget totals to {@link driveRun}. `ctx.executed`
- * is appended in place (one report per iteration); everything else the caller
- * must observe travels back through {@link StepGateLoopOutcome}.
+ * Drive one step's bounded gate loop: a rejection with loops left re-executes
+ * the subgraph with the judge's feedback in the unit prompts. Returns the loop
+ * decision and running budget totals; `ctx.executed` is appended in place.
  */
 async function runStepGateLoop(ctx, gate, totals) {
     const { options, next, stepPlan, step, evidence, executed, routeSelected, routeUnselected } = ctx;
-    const { summaryJudge, leaseHolder, heartbeat } = ctx;
+    const { summaryJudge } = ctx;
     const { startLoop, maxLoops } = gate;
     let { unitsDispatched, tokensUsed } = totals;
     let gateFeedback = gate.seededFeedback;
@@ -518,15 +362,7 @@ async function runStepGateLoop(ctx, gate, totals) {
         tokensUsed,
     });
     for (let gateLoop = startLoop; gateLoop <= maxLoops; gateLoop++) {
-        // A loop re-execution dispatches a fresh round of units — renew the
-        // lease so a long evaluator-optimizer cycle cannot outlive the TTL.
-        if (gateLoop > 1)
-            await renewRunLease(next.run.id, leaseHolder);
         const result = await executeStepSubgraph(ctx, { gateLoop, gateFeedback, unitsDispatched, tokensUsed });
-        // If the heartbeat lost the lease WHILE this step dispatched, another
-        // engine now owns the run — stop loudly BEFORE finalizing the step
-        // (completeWorkflowStep would race the new owner's spine).
-        heartbeat?.assertAlive();
         unitsDispatched = result.unitsDispatched;
         if (result.tokensUsed !== undefined)
             tokensUsed = result.tokensUsed;
@@ -540,12 +376,7 @@ async function runStepGateLoop(ctx, gate, totals) {
             summary: result.summary,
             ...(result.notices ? { notices: result.notices } : {}),
         });
-        // Route evaluation + artifact-judged completion gate + gate-row
-        // journaling + the bounded-loop rejection contract are the SHARED
-        // completion path (`finalizeExecutedStep`): every step advances through
-        // that one sequence, whether its units were just dispatched or rehydrated
-        // from the journal on resume, so the same frozen plan always promotes the
-        // same artifact and advances (or rejects) the spine identically.
+        // Route, gate, and advance: the shared completion path, live or resumed.
         let finalize;
         try {
             finalize = await finalizeExecutedStep({
@@ -563,20 +394,13 @@ async function runStepGateLoop(ctx, gate, totals) {
                 routeUnselected,
                 summaryJudge,
                 signal: options.signal,
-                // The judge runs under the DISPATCH signal, so the completion path must
-                // see it too: an abort delivered there (a lost lease, a caller Ctrl-C)
-                // is an interruption, not a verifier outage.
-                ...(ctx.dispatchSignal ? { dispatchSignal: ctx.dispatchSignal } : {}),
-                leaseHolder,
             });
         }
         catch (error) {
-            heartbeat?.assertAlive();
             if (options.signal?.aborted)
                 return outcome({ kind: "aborted" });
             throw error;
         }
-        heartbeat?.assertAlive();
         if (finalize.kind === "retry") {
             // Re-execute the subgraph with the judge/validation feedback threaded
             // into unit prompts — the changed prompt changes each unit's input
@@ -585,13 +409,8 @@ async function runStepGateLoop(ctx, gate, totals) {
             continue;
         }
         if (finalize.kind === "advanced") {
-            // Hand the rest of this invocation the COMPLETE artifact — but only when
-            // some LATER step's frozen references can actually read it (set-time
-            // retention, see `referencedStepIds`). `finalize` has already journaled
-            // the step (and stamped any route decision onto `result.evidence`), and
-            // the persisted row may carry a truncation envelope in place of an
-            // over-cap value — the row bound must not change what the very next step
-            // reads.
+            // Keep the complete artifact for later steps of this invocation, but only
+            // when some later reference can read it (`referencedStepIds`).
             if (ctx.liveEvidenceConsumers.has(step.id))
                 ctx.liveEvidence.set(step.id, result.evidence);
             // A route-only step's summary IS its decision (finalize surfaces it).
@@ -609,7 +428,7 @@ async function runStepGateLoop(ctx, gate, totals) {
             return outcome({ kind: "judge-failed", judgeFailure: { stepId: step.id, message: finalize.summary } });
         }
         if (finalize.kind === "child-blocked") {
-            // P3b §3.4: a composed child workflow is blocked. Like judge-failed,
+            // a composed child workflow is blocked. Like judge-failed,
             // NO gate loop was consumed and the step does not count against
             // maxSteps. `result.childBlocked` (set by reduceStepOutcomes off the
             // failed unit's live-only childRun field) carries the identity
@@ -648,28 +467,18 @@ async function runStepGateLoop(ctx, gate, totals) {
     // here would mean those two bounds disagree — a bug, not a run outcome.
     throw new Error(`Workflow run ${next.run.id} step "${step.id}" left its gate loop with no terminal outcome (loop bounds disagree).`);
 }
-/** The engine loop proper — runs under the lease held by `runWorkflowSteps`. */
-async function driveRun(options, initial, leaseHolder, heartbeat, 
+/** The engine loop proper — runs under the per-run lock `runWorkflowAttempt` holds. */
+async function driveRun(options, initial, 
+/** The run row's decoded frozen plan; undefined only for a done run, which drives nothing. */
+storedPlan, 
 /**
- * The COMPLETE in-memory evidence of every step THIS call has completed,
- * keyed by step id, preferred over the re-read row when the downstream scope
- * is rebuilt below — avoiding a re-parse of a row this same invocation just
- * wrote (step artifacts are persisted whole, so the two values agree; this
- * is purely an avoided round trip, not a correctness dependency). A LATER
- * `akm workflow run` starts with an empty map and reads the rows directly.
- *
- * Only steps some OTHER step's references NAME are stored (`referencedStepIds`
- * — the set-time filter): a step nothing downstream reads has no consumer to
- * keep it complete for, so retaining it would buy nothing and cost its bytes
- * for the rest of the invocation.
+ * In-memory evidence of steps this call completed that a later step reads,
+ * preferred over re-parsing their rows (the values agree).
  */
 liveEvidence) {
     let next = initial;
-    if (initial.done)
+    if (initial.done || !storedPlan)
         return completedRunResult(initial.run.id);
-    // The effective dispatch signal: the heartbeat's controller (a lost lease or
-    // a caller abort aborts it) while leased, else the raw caller signal.
-    const dispatchSignal = heartbeat?.signal ?? options.signal;
     const executed = [];
     let gateRejection;
     let judgeFailure;
@@ -682,7 +491,7 @@ liveEvidence) {
     // a route-skipped step consumes NOTHING (no work was dispatched for it).
     let stepsProcessed = 0;
     let { unitsDispatched, tokensUsed } = await seedRunAccountingFromJournal(next.run.id);
-    const plan = await loadAuthoritativeRunPlan(options, next);
+    const plan = await loadAuthoritativeRunPlan(options, next, storedPlan);
     // Live-evidence retention is decided at SET time, from the frozen plan alone:
     // a completed step's complete artifact is held only while some other step's
     // references can still read it. An exec unit's promoted stdout can be 8 MiB,
@@ -694,31 +503,17 @@ liveEvidence) {
     // (two routers may share a target).
     const routeSelected = new Set();
     const routeUnselected = new Map();
-    // Resume contract: route decisions are journaled in the route step's
-    // evidence (`evidence.route.selected`) and must be REPLAYED into the
-    // bookkeeping before the spine advances — a re-invoked run (crash, Ctrl-C,
-    // maxSteps, gate rejection after the route completed) would otherwise reach
-    // the unselected targets with empty in-memory state and execute the wrong
-    // branch. Decisions stay pure functions of (frozen plan, params, journaled
-    // results) — the addendum determinism bar. A done run skips the seeding:
-    // nothing will dispatch, so an unrecoverable prior decision must not
-    // block the no-op status return below.
+    // Replay journaled route decisions before the spine advances, so a
+    // re-invoked run skips the unselected branches. A done run skips this.
     if (!next.done) {
         seedJournaledRouteDecisions(plan, next, routeSelected, routeUnselected);
     }
     while (!next.done && next.step && next.run.status === "active" && stepsProcessed < maxSteps) {
-        // A LOST lease (the heartbeat's renewal failed mid-step) is a loud stop —
-        // another engine owns the spine now. A caller abort (options.signal) is a
-        // graceful break, distinct from a lost lease.
-        heartbeat?.assertAlive();
+        // A caller abort (options.signal) is a graceful break.
         if (options.signal?.aborted) {
             aborted = true;
             break;
         }
-        // Renew the run lease between steps (a fresh 90s window per iteration).
-        // Losing it (expired mid-step + claimed by another engine) throws — the
-        // new owner drives the spine now.
-        await renewRunLease(next.run.id, leaseHolder);
         const step = next.step;
         const stepPlan = plan.steps.find((s) => s.stepId === step.id);
         if (!stepPlan) {
@@ -735,26 +530,21 @@ liveEvidence) {
                 skipInfo,
                 routeUnselected,
                 executed,
-                leaseHolder,
             });
             continue;
         }
         const evidence = {};
         for (const s of next.workflow.steps)
             evidence[s.id] = liveEvidence.get(s.id) ?? s.evidence;
-        // Bounded gate loop (addendum R2, `gate.max_loops`): loop 1 is the normal
+        // Bounded gate loop: loop 1 is the normal
         // execution; a gate rejection with attempts left re-executes the subgraph
         // with the judge's feedback threaded into unit prompts. The bound comes
         // from the shared derivation, which holds an exec step to a single
         // execution — its argv cannot answer feedback (see effectiveGateMaxLoops).
         const maxLoops = effectiveGateMaxLoops(stepPlan);
         const { startLoop, seededFeedback } = await recoverGateLoopState(next.run.id, stepPlan);
-        // Resume AFTER the FINAL rejection (`startLoop` past the loop bound): the
-        // gate was already exhausted before the crash, so there is NO fresh loop to
-        // run — reproduce the documented gateRejection outcome from the stored
-        // final-loop feedback instead of re-dispatching a spurious extra loop. The
-        // l1..l<maxLoops> rows stay untouched and the step stays active, exactly as
-        // when the engine first exhausted the gate.
+        // Resumed after the final rejection: reproduce the gate-exhausted outcome
+        // from the stored feedback instead of running an extra loop.
         if (startLoop > maxLoops) {
             gateRejection = {
                 stepId: step.id,
@@ -770,7 +560,7 @@ liveEvidence) {
         // verify. No gate loop is consumed and nothing is dispatched.
         let summaryJudge;
         try {
-            summaryJudge = workflowSummaryJudge(options, stepPlan, dispatchSignal, {
+            summaryJudge = workflowSummaryJudge(options, stepPlan, options.signal, {
                 runId: next.run.id,
                 stepId: step.id,
             });
@@ -784,7 +574,6 @@ liveEvidence) {
                 runId: next.run.id,
                 stepId: step.id,
                 cause: `the verification judge could not be resolved from the frozen plan${detail}`,
-                leaseHolder,
             });
             executed.push({ stepId: step.id, ok: false, unitCount: 0, failedUnits: 0, summary: notes });
             judgeFailure = { stepId: step.id, message: notes };
@@ -803,9 +592,6 @@ liveEvidence) {
             routeSelected,
             routeUnselected,
             summaryJudge,
-            leaseHolder,
-            heartbeat,
-            dispatchSignal,
         }, { startLoop, maxLoops, seededFeedback }, { unitsDispatched, tokensUsed });
         unitsDispatched = outcome.unitsDispatched;
         tokensUsed = outcome.tokensUsed;
@@ -840,22 +626,3 @@ liveEvidence) {
         ...(aborted ? { aborted: true } : {}),
     };
 }
-/**
- * Load the plan a run executes (frozen-plan contract, migration 006):
- *
- *   - `plan_json` present → parse it and verify `plan_hash` (sha256 of the
- *     canonical JSON). A mismatch means the journaled plan was tampered with
- *     or corrupted — fail loudly, never silently recompile. The workflow
- *     asset file is NEVER touched on this path.
- * Missing and non-current plans fail validation and are never rebuilt from a
- * mutable source asset.
- */
-async function loadStoredPlan(runId) {
-    const row = await withWorkflowRunsRepo((repo) => {
-        const run = repo.getRunById(runId);
-        return run;
-    });
-    if (!row)
-        throw new UsageError(`Workflow run ${runId} was not found.`);
-    return requireExecutableWorkflowPlan(row);
-}