clinkx 0.2.3 → 1.0.0

package/clinkx-workflows/dist/state.js

@@ -1,15 +1,27 @@
-import { createHash } from "node:crypto";
+import { createHash, randomBytes } from "node:crypto";
 import { execFileSync } from "node:child_process";
-import { readFileSync } from "node:fs";
-import { mkdir, open, readFile, readdir, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
+import { constants as fsConstants, readFileSync } from "node:fs";
+import { lstat, mkdir, open, readFile, readdir, realpath, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
 import { dirname, basename, isAbsolute, join, relative, resolve } from "node:path";
 import { hostname } from "node:os";
 import { fileURLToPath } from "node:url";
 import { WorkflowError } from "./errors.js";
 import { buildFidelityMetadata, } from "./fidelity.js";
 import { fingerprintsMatch, generateFingerprint, generateWorkflowFingerprint, } from "./fingerprint.js";
-import { DEFAULT_RETAIN_CANCELLED_DAYS, DEFAULT_RETAIN_DEBUG_DAYS, DEFAULT_RETAIN_FAILED_DAYS, deriveAllowedRoots, getArtifactsDir, getChildConfigPath, getChildMaxConcurrent, getChildMaxOutputBytes, getChildMaxPromptBytes, getChildMaxResponseChars, getLogLevel, getRetentionConfig, getRunsDir, getRunStateDir, getStateDir, resolveWorkspaceRoot, } from "./config.js";
+import { parseWorkflowContext, } from "./context-preflight.js";
+import { isLegalContextBuilderLifecycleTransition, parseContextBuilderLifecycleStateV1, } from "./context-builder-lifecycle-contract.js";
+import { PlanReviewRunStateSchema, } from "./plan-review/run-state.js";
+import { CouncilPlanReviewErrorSchema, createPlanReviewPublicationError, serializePlanReviewResult, writeSerializedPlanReviewResultSidecar, } from "./plan-review/publication.js";
+import { isPlanReviewPublicationEligible, projectPlanReviewRun, } from "./plan-review/projector.js";
+import { isPlanReviewRunState } from "./plan-review/admission.js";
+import { restorePlanReviewRun } from "./plan-review/resume.js";
+import { PlanReviewStageError } from "./plan-review/stage-error.js";
+import { DEFAULT_RETAIN_ASYNC_DAYS, DEFAULT_RETAIN_CANCELLED_DAYS, DEFAULT_RETAIN_DEBUG_DAYS, DEFAULT_RETAIN_FAILED_DAYS, DEFAULT_RETAIN_SUCCEEDED_DAYS, deriveAllowedRoots, getArtifactsDir, getChildConfigPath, getChildMaxConcurrent, getChildMaxOutputBytes, getChildMaxPromptBytes, getChildMaxResponseChars, getLogLevel, getRetentionConfig, getRunsDir, getRunStateDir, getStateDir, resolveWorkspaceRoot, } from "./config.js";
 import { logger } from "./logger.js";
+import { resolveOpenedDescriptorPath } from "./contained-file.js";
+import { deriveWorkflowRunGeneration } from "./dashboard/generation.js";
+import { removeSessionLogGeneration } from "./dashboard/session-log-retention.js";
+import { createSessionLogGeneration, readSessionLogGeneration, } from "./dashboard/session-log-identity.js";
 const STATE_FILENAME = "state.json";
 const STATE_TMP_FILENAME = "state.json.tmp";
 const WORKFLOW_SNAPSHOT_FILENAME = "workflow.snapshot.json";
@@ -17,8 +29,162 @@ const INPUTS_SNAPSHOT_FILENAME = "inputs.snapshot.json";
 const CHILD_CONFIG_DIRNAME = "child-config";
 const CHILD_CONFIG_SNAPSHOT_FILENAME = "snapshot.json";
 const RUN_LOCK_FILENAME = "run.lock";
+/**
+ * Producer-side refresh cadence for a held `run.lock`'s mtime. Mirrors
+ * `dashboard/auth/credential-store.ts`'s LOCK_HEARTBEAT_MS.
+ */
+const RUN_LOCK_HEARTBEAT_MS = 10_000;
+/**
+ * Reader-side staleness floor the shipped cadence is designed against; the
+ * dashboard observer derives its own threshold from the declared interval.
+ * Re-exported for tests and cross-module documentation only — nothing in this
+ * module consults it, because run-lock CONTROL decisions
+ * (`classifyRunLockSync`, `reapRunCancelled`, pruning) must never depend on
+ * mtime freshness.
+ */
+export { RUN_LOCK_HEARTBEAT_STALE_FLOOR_MS } from "./run-lock-heartbeat.js";
+/** Same cap as the dashboard observer's contained-read `maxBytes`. */
+const RUN_LOCK_MAX_BYTES = 4_096;
 const OUTPUT_INLINE_THRESHOLD_BYTES = 10 * 1024;
+/**
+ * Cap for error strings persisted inline in state.json (run-level `error` and
+ * per-call `errorMessage`). Unlike call outputs (which spill to sidecar files
+ * via persistOutput), errors always live inline, and provider stderr/exception
+ * chains can be arbitrarily large — keep a bounded head plus an explicit
+ * truncation marker recording how many bytes were cut.
+ */
+const MAX_PERSISTED_ERROR_BYTES = 16 * 1024;
 const DAY_MS = 24 * 60 * 60 * 1000;
+const STALE_RUNNING_RECOVERY_ERROR = "Recovered stale running run during resume";
+/**
+ * Run-level error written by reconcileStaleRunDirectories. Deliberately
+ * distinct from STALE_RUNNING_RECOVERY_ERROR (the resume-path recovery string)
+ * so the two reconcile paths remain distinguishable in persisted state.
+ */
+export const STALE_RUN_RECONCILE_ERROR = "Reconciled stale running run: no live worker process";
+const STALE_RUN_RECONCILE_CALL_ERROR = "Reconciled: worker terminated before call completion";
+function resolvePlanReviewChairmanCallId(definition) {
+    const stage = definition.stages.find((entry) => entry.id === "chairman");
+    const callId = stage?.calls[0]?.id;
+    if (callId === undefined || stage?.calls.length !== 1) {
+        throw new Error("Persisted plan-review workflow definition must have exactly one chairman call");
+    }
+    return callId;
+}
+async function readPersistedPlanReviewChairmanCallId(runDir, state) {
+    const workflowSnapshotPath = join(runDir, state.workflowSnapshotPath ?? WORKFLOW_SNAPSHOT_FILENAME);
+    await verifySnapshotHash(workflowSnapshotPath, state.workflowSnapshotHash, "workflow snapshot");
+    const definition = await readJsonFile(workflowSnapshotPath);
+    return resolvePlanReviewChairmanCallId(definition);
+}
+function withPlanReviewPublication(state, publication) {
+    const nextState = structuredClone(state);
+    nextState.plan_review_state = PlanReviewRunStateSchema.parse({
+        ...nextState.plan_review_state,
+        publication,
+    });
+    return nextState;
+}
+/**
+ * C-9 chain-internal finalizer. It never calls a public store writer or
+ * enqueueWrite. The committed marker is persisted on the pre-terminal state,
+ * after the sidecar and before the caller commits the terminal state.
+ */
+const finalizePlanReviewTerminalState = async ({ runDir, previousState, intendedState, intent, chairmanCallId, serializedResult, }) => {
+    const existing = previousState.plan_review_state?.publication;
+    if (existing?.committed === true) {
+        return { state: withPlanReviewPublication(intendedState, existing) };
+    }
+    if (!isPlanReviewPublicationEligible(previousState)) {
+        return { state: intendedState };
+    }
+    let restored;
+    try {
+        const resolvedChairmanCallId = chairmanCallId ??
+            await readPersistedPlanReviewChairmanCallId(runDir, previousState);
+        restored = await restorePlanReviewRun(runDir, previousState.plan_review_state, resolvedChairmanCallId);
+    }
+    catch (err) {
+        const detail = err instanceof Error ? err.message : String(err);
+        const stageError = err instanceof PlanReviewStageError
+            ? err
+            : new PlanReviewStageError("execution_contract", detail, "internal_error");
+        const finalStatus = intent.status === "succeeded" ? "failed" : intent.status;
+        const failed = structuredClone(intendedState);
+        failed.status = finalStatus;
+        if (intent.status === "succeeded") {
+            failed.error = `Plan-review persisted-state verification failed: ${detail}`;
+        }
+        failed.plan_review_error = intent.planReviewError ??
+            CouncilPlanReviewErrorSchema.parse({
+                schema_version: "council-plan-review-error-v1",
+                run_id: failed.runId,
+                execution_status: finalStatus,
+                stage: stageError.stage,
+                error: {
+                    reason: stageError.reason,
+                    detail: detail.slice(0, 2_048),
+                },
+            });
+        return {
+            state: failed,
+            ...(intent.status === "succeeded"
+                ? { throwAfterCommit: stageError }
+                : {}),
+        };
+    }
+    try {
+        const projected = serializedResult ?? serializePlanReviewResult(await projectPlanReviewRun({
+            runDir,
+            state: previousState,
+            contract: restored.contract,
+            execution_contract_digest: restored.execution_contract_digest,
+        }));
+        const marker = await writeSerializedPlanReviewResultSidecar(runDir, projected);
+        // Durable crash boundary: a process death after this write still leaves a
+        // contained, marker-addressable result even though terminal intent has not
+        // yet been committed.
+        const intermediate = withPlanReviewPublication(previousState, marker);
+        validateStrictStateSubtrees(intermediate, join(runDir, STATE_FILENAME));
+        await writeJsonAtomic(join(runDir, STATE_FILENAME), intermediate);
+        return { state: withPlanReviewPublication(intendedState, marker) };
+    }
+    catch (err) {
+        const detail = err instanceof Error ? err.message : String(err);
+        const finalStatus = intent.status === "succeeded" ? "failed" : intent.status;
+        const failed = withPlanReviewPublication(intendedState, { committed: false });
+        failed.status = finalStatus;
+        if (intent.status === "succeeded") {
+            failed.error = `Plan-review publication failed: ${detail}`;
+        }
+        failed.plan_review_error = createPlanReviewPublicationError(failed.runId, finalStatus, err instanceof PlanReviewStageError ? err.reason : "internal_error", detail);
+        return {
+            state: failed,
+            ...(intent.status === "succeeded"
+                ? {
+                    throwAfterCommit: new PlanReviewStageError("publication", failed.error, failed.plan_review_error.error.reason),
+                }
+                : {}),
+        };
+    }
+};
+class RunLockOwnershipUnknownError extends Error {
+    constructor(lockPath, detail) {
+        super(`Run lock ownership is unknown at ${lockPath}; refusing to reclaim it: ${detail}`);
+        this.name = "RunLockOwnershipUnknownError";
+    }
+}
+/**
+ * Constructed ONLY after a complete, valid lock record has been read and its
+ * token proves the lock is no longer ours. Never for open/stat/read/parse/type/
+ * size errors — those stay "ordinary" and must not revoke unlink authority.
+ */
+class RunLockOwnershipLostError extends Error {
+    constructor(lockPath, detail) {
+        super(`Run lock ownership was lost at ${lockPath}: ${detail}`);
+        this.name = "RunLockOwnershipLostError";
+    }
+}
 export function classifyRunDirectory(runDirPath) {
     try {
         const state = readRunStateSync(runDirPath);
@@ -34,8 +200,15 @@ export function classifyRunDirectory(runDirPath) {
                 return "failed";
             case "cancelled":
                 return "cancelled";
-            case "succeeded":
-                return state.debugKeepArtifacts === true ? "debug_kept" : "unknown";
+            case "succeeded": {
+                if ((state.debugKeepArtifacts !== undefined && typeof state.debugKeepArtifacts !== "boolean") ||
+                    (state.async !== undefined && typeof state.async !== "boolean")) {
+                    return "unknown";
+                }
+                return state.debugKeepArtifacts === true
+                    ? "debug_kept"
+                    : (state.async === true ? "succeeded_async" : "succeeded_sync");
+            }
             case "running":
             case "pending":
                 return "running_or_pending";
@@ -91,9 +264,40 @@ export async function pruneRunDirectories(runsRootDir, options) {
                     result.skipped += 1;
                     continue;
                 }
+                const identity = await readSessionLogGeneration(runDirPath);
+                let sessionLogGeneration;
+                if (identity.kind === "available") {
+                    sessionLogGeneration = identity.sessionLogGeneration;
+                }
+                else if (identity.kind === "legacy") {
+                    try {
+                        sessionLogGeneration = await admitWorkflowRunGeneration(runsRootDir, runDirPath, entry.name);
+                    }
+                    catch {
+                        logger.warn({ run_id: entry.name }, "legacy workflow session-log generation could not be admitted before parent pruning");
+                    }
+                }
+                else {
+                    logger.warn({ run_id: entry.name }, "workflow session-log identity was unsafe before parent pruning");
+                }
                 await rm(runDirPath, { recursive: true, force: true });
                 pruned = true;
                 result.pruned += 1;
+                if (sessionLogGeneration !== undefined) {
+                    try {
+                        const cleanup = await removeSessionLogGeneration({
+                            rootDir: join(dirname(resolve(runsRootDir)), "session-logs"),
+                            family: "workflow",
+                            generation: sessionLogGeneration,
+                        });
+                        if (cleanup.kind === "unsafe" || cleanup.kind === "unavailable") {
+                            logger.warn({ run_id: entry.name, cleanup: cleanup.kind }, "workflow parent pruned but exact session-log cleanup was unavailable");
+                        }
+                    }
+                    catch {
+                        logger.warn({ run_id: entry.name }, "workflow parent pruned but exact session-log cleanup failed");
+                    }
+                }
                 logger.info({
                     run_dir: runDirPath,
                     classification,
@@ -115,8 +319,64 @@ export async function pruneRunDirectories(runsRootDir, options) {
     logger.info({ runs_root_dir: runsRootDir, ...result }, "workflow run directory pruning complete");
     return result;
 }
+function isTerminalRunStatus(status) {
+    return status === "succeeded" || status === "failed" || status === "cancelled";
+}
+function isPlanReviewRun(state) {
+    // Presence-based typed detection (see isPlanReviewRunState): legacy runs
+    // persisted under the pre-rename name "council-plan-review" keep routing
+    // through the plan-review terminal-intent protocol, while runs of the
+    // GENERIC council-plan-review workflow never do.
+    return isPlanReviewRunState(state);
+}
+function isStaleRunningRecoveryCheckpoint(state) {
+    return state.status === "failed" &&
+        state.failureClass === "transient" &&
+        state.error === STALE_RUNNING_RECOVERY_ERROR &&
+        state.plan_review_error === undefined &&
+        state.stages.every((stage) => stage.calls.every((call) => call.state !== "running"));
+}
+function canReplaceStaleRunningRecovery(state, intent) {
+    return isStaleRunningRecoveryCheckpoint(state) &&
+        intent.status === "failed" &&
+        intent.planReviewError?.stage === "execution_contract";
+}
+function applyPlanReviewTerminalIntent(state, intent) {
+    const replacingRecovery = canReplaceStaleRunningRecovery(state, intent);
+    if (isTerminalRunStatus(state.status) && !replacingRecovery) {
+        return undefined;
+    }
+    const nextState = structuredClone(state);
+    nextState.status = intent.status;
+    nextState.completedAt = intent.completedAt;
+    nextState.error = intent.error;
+    if (intent.replaceFailureClass || replacingRecovery) {
+        nextState.failureClass = intent.failureClass;
+    }
+    if (intent.planReviewError === undefined) {
+        nextState.plan_review_error = undefined;
+    }
+    else {
+        const planReviewError = CouncilPlanReviewErrorSchema.parse(intent.planReviewError);
+        if (planReviewError.run_id !== state.runId) {
+            throw new Error(`plan_review_error.run_id ${planReviewError.run_id} does not match run ${state.runId}`);
+        }
+        if (planReviewError.execution_status !== intent.status) {
+            throw new Error(`plan_review_error.execution_status ${planReviewError.execution_status} does not match terminal intent ${intent.status}`);
+        }
+        nextState.plan_review_error = planReviewError;
+    }
+    return nextState;
+}
 export class WorkflowStateStore {
     runDir;
+    /** Portable journal identity; absent only for a legacy or failed-open run. */
+    sessionLogGeneration;
+    /**
+     * Warn-only duplicate-replay signal detected at create(); undefined on
+     * resume and when no permanently-failed identical sibling was found.
+     */
+    duplicateOfFailedRun;
     definition;
     variables;
     hooks;
@@ -126,22 +386,49 @@ export class WorkflowStateStore {
     debugKeepArtifacts;
     writeChain = Promise.resolve();
     lockHeld = true;
+    planReviewTerminalResult;
+    lockToken;
+    heartbeat;
+    heartbeatInFlight;
+    heartbeatError;
     constructor(options) {
         this.runDir = options.runDir;
+        this.sessionLogGeneration = options.sessionLogGeneration;
+        this.duplicateOfFailedRun = options.duplicateOfFailedRun;
         this.definition = options.definition;
         this.variables = options.variables;
         this.state = options.state;
         this.resume = options.resume;
         this.debugKeepArtifacts = options.debugKeepArtifacts;
         this.lockPath = join(options.runDir, RUN_LOCK_FILENAME);
+        this.lockToken = options.lockToken;
+        this.startHeartbeat();
         this.hooks = {
             onRunStateChange: async (event) => {
                 await this.enqueueWrite(async () => {
+                    const cappedError = capPersistedError(event.error);
+                    if (isPlanReviewRun(this.state) && isTerminalRunStatus(event.state)) {
+                        await this.finalizePlanReviewTerminalIntent({
+                            status: event.state,
+                            completedAt: event.completedAt,
+                            error: cappedError,
+                            failureClass: event.failureClass,
+                            replaceFailureClass: true,
+                            planReviewError: event.planReviewError,
+                        });
+                        return;
+                    }
+                    if (isPlanReviewRun(this.state) &&
+                        isTerminalRunStatus(this.state.status) &&
+                        !(event.state === "running" &&
+                            isStaleRunningRecoveryCheckpoint(this.state))) {
+                        return;
+                    }
                     const nextState = structuredClone(this.state);
                     nextState.status = event.state;
                     nextState.startedAt = event.startedAt ?? nextState.startedAt;
                     nextState.completedAt = event.completedAt;
-                    nextState.error = event.error;
+                    nextState.error = cappedError;
                     nextState.failureClass = event.failureClass;
                     this.state = nextState;
                     await this.writeState();
@@ -166,10 +453,13 @@ export class WorkflowStateStore {
                     const callEntry = getCallEntry(nextState, event.stageId, event.callId);
                     callEntry.state = event.state;
                     callEntry.retryCount = event.retryCount;
+                    if (event.state === "running") {
+                        callEntry.attemptsConsumed = Math.max(callEntry.attemptsConsumed ?? 0, event.attempt);
+                    }
                     callEntry.startedAt = event.startedAt;
                     callEntry.completedAt = event.completedAt;
                     callEntry.durationMs = event.durationMs;
-                    callEntry.errorMessage = event.error;
+                    callEntry.errorMessage = capPersistedError(event.error);
                     callEntry.failureClass = event.failureClass;
                     callEntry.childDebugMetadata = event.childDebugMetadata;
                     callEntry.contextProducing = callEntry.contextProducing ?? false;
@@ -205,10 +495,19 @@ export class WorkflowStateStore {
         const recycled = await prepareRunDirForCreate(runDir, options.runId);
         let lockHeld = false;
         try {
-            await acquireRunLock(runDir);
+            const lock = await acquireRunLock(runDir, { declareHeartbeat: true });
             lockHeld = true;
+            const sessionLogGeneration = await createSessionLogGeneration(runDir);
             const workflowSnapshotHash = await writeJsonSnapshot(join(runDir, WORKFLOW_SNAPSHOT_FILENAME), options.definition);
             const inputsSnapshotHash = await writeJsonSnapshot(join(runDir, INPUTS_SNAPSHOT_FILENAME), options.variables);
+            const duplicateOfFailedRun = await detectDuplicateReplay(runDir, workflowSnapshotHash, inputsSnapshotHash);
+            if (duplicateOfFailedRun !== undefined) {
+                logger.warn({
+                    run_id: options.runId,
+                    duplicate_of_run_id: duplicateOfFailedRun.runId,
+                    duplicate_completed_at: duplicateOfFailedRun.completedAt,
+                }, "run replays workflow and inputs identical to a permanently-failed prior run");
+            }
             const childConfigPath = getChildConfigPath();
             const resumable = childConfigPath != null;
             const fingerprintData = resumable
@@ -242,15 +541,21 @@ export class WorkflowStateStore {
                         callId: call.id,
                         state: "pending",
                         retryCount: 0,
+                        attemptsConsumed: 0,
                         contextProducing: isContextProducing(options.definition, stage.id, call.id),
                     })),
                 })),
                 variables: options.variables,
+                initialContextPolicy: options.initialContextPolicy,
                 debugKeepArtifacts,
+                async: options.async ?? false,
             };
             await writeJsonAtomic(join(runDir, STATE_FILENAME), state);
-            if (recycled) {
+            if (recycled.recycled) {
                 await rm(`${runDir}.recycled`, { recursive: true, force: true }).catch(() => { });
+                if (recycled.retiredSessionLogGeneration !== undefined) {
+                    await cleanupWorkflowSessionLogGeneration(dirname(runDir), options.runId, recycled.retiredSessionLogGeneration);
+                }
             }
             return new WorkflowStateStore({
                 runDir,
@@ -259,13 +564,16 @@ export class WorkflowStateStore {
                 state,
                 resume: {},
                 debugKeepArtifacts,
+                lockToken: lock.token,
+                sessionLogGeneration,
+                duplicateOfFailedRun,
             });
         }
         catch (error) {
             if (lockHeld) {
                 await safeUnlink(join(runDir, RUN_LOCK_FILENAME));
             }
-            if (recycled) {
+            if (recycled.recycled) {
                 try {
                     await rm(runDir, { recursive: true, force: true });
                     await rename(`${runDir}.recycled`, runDir);
@@ -283,9 +591,19 @@ export class WorkflowStateStore {
     static async resume(options) {
         const runDir = options.runDir ?? getRunStateDir(options.runId);
         const debugKeepArtifacts = options.debugKeepArtifacts ?? false;
-        await acquireRunLock(runDir);
+        const lock = await acquireRunLock(runDir, { declareHeartbeat: true });
         try {
             const state = await readRunState(runDir);
+            const identity = await readSessionLogGeneration(runDir);
+            if (identity.kind === "unavailable") {
+                throw new Error(`Run ${options.runId} has an unsafe session-log identity`);
+            }
+            if (isPlanReviewRun(state) &&
+                (state.status === "failed" ||
+                    state.status === "cancelled" ||
+                    state.status === "succeeded")) {
+                throw new Error(`Run ${options.runId} is terminal (${state.status}) and cannot be resumed; start a new run instead.`);
+            }
             if (!state.resumable) {
                 throw new Error(`Run ${options.runId} is non-resumable because it was started from ambient child config`);
             }
@@ -361,6 +679,10 @@ export class WorkflowStateStore {
                 state: recovered,
                 resume,
                 debugKeepArtifacts,
+                lockToken: lock.token,
+                ...(identity.kind === "available"
+                    ? { sessionLogGeneration: identity.sessionLogGeneration }
+                    : {}),
             });
         }
         catch (error) {
@@ -371,8 +693,54 @@ export class WorkflowStateStore {
     get runState() {
         return structuredClone(this.state);
     }
+    /**
+     * Open a new generic resume attempt before any context preflight or provider
+     * setup. Provider-call attempt counters remain untouched; only the run-level
+     * terminal envelope is cleared so a pre-dispatch failure/cancellation can be
+     * persisted as the outcome of this resume attempt.
+     *
+     * Plan-review stale-run recovery has its own terminal-intent protocol and is
+     * deliberately left unchanged here.
+     */
+    async beginResumeAttempt() {
+        await this.enqueueWrite(async () => {
+            if (isPlanReviewRun(this.state))
+                return;
+            const nextState = structuredClone(this.state);
+            if (isTerminalRunStatus(this.state.status)) {
+                nextState.previousTerminal = {
+                    status: this.state.status,
+                    ...(this.state.completedAt === undefined
+                        ? {}
+                        : { completedAt: this.state.completedAt }),
+                    ...(this.state.error === undefined ? {} : { error: this.state.error }),
+                    ...(this.state.failureClass === undefined
+                        ? {}
+                        : { failureClass: this.state.failureClass }),
+                };
+            }
+            nextState.status = "pending";
+            nextState.completedAt = undefined;
+            nextState.error = undefined;
+            nextState.failureClass = undefined;
+            this.state = nextState;
+            await this.writeState();
+        });
+    }
     async markCancelled(reason = "Workflow cancelled") {
         await this.enqueueWrite(async () => {
+            if (isPlanReviewRun(this.state)) {
+                await this.finalizePlanReviewTerminalIntent({
+                    status: "cancelled",
+                    completedAt: new Date().toISOString(),
+                    error: reason,
+                    replaceFailureClass: false,
+                });
+                return;
+            }
+            if (this.state.status !== "pending" && this.state.status !== "running") {
+                return;
+            }
             const nextState = structuredClone(this.state);
             nextState.status = "cancelled";
             nextState.completedAt = new Date().toISOString();
@@ -381,14 +749,308 @@ export class WorkflowStateStore {
             await this.writeState();
         });
     }
+    /**
+     * Idempotently mark the run failed. Only applies when the current status is
+     * non-terminal (pending/running) so it never clobbers a richer engine-emitted
+     * terminal status (succeeded/failed/cancelled).
+     */
+    async markFailed(error, failureClass, planReviewError) {
+        const cappedError = capPersistedError(error);
+        await this.enqueueWrite(async () => {
+            if (isPlanReviewRun(this.state)) {
+                await this.finalizePlanReviewTerminalIntent({
+                    status: "failed",
+                    completedAt: new Date().toISOString(),
+                    error: cappedError,
+                    failureClass,
+                    replaceFailureClass: failureClass !== undefined,
+                    planReviewError,
+                });
+                return;
+            }
+            if (this.state.status !== "pending" && this.state.status !== "running") {
+                return;
+            }
+            const nextState = structuredClone(this.state);
+            nextState.status = "failed";
+            nextState.completedAt = new Date().toISOString();
+            nextState.error = cappedError;
+            if (failureClass !== undefined) {
+                nextState.failureClass = failureClass;
+            }
+            this.state = nextState;
+            await this.writeState();
+        });
+    }
     async flush() {
         await this.writeChain;
     }
+    async commitPreflightContext({ variables, workflow_context, }) {
+        await this.enqueueWrite(async () => {
+            const lifecycle = this.state.context_builder_lifecycle;
+            if (lifecycle !== undefined) {
+                if ((lifecycle.status !== "succeeded" && lifecycle.status !== "partial") ||
+                    lifecycle.phase !== "complete" ||
+                    lifecycle.artifact_run_id !== workflow_context.context_builder.source_run_id) {
+                    throw new Error("workflow_context cannot commit before a matching successful lifecycle completes");
+                }
+            }
+            const committedWorkflowContext = lifecycle === undefined
+                ? workflow_context
+                : {
+                    ...workflow_context,
+                    context_builder: {
+                        ...workflow_context.context_builder,
+                        lifecycle: structuredClone(lifecycle),
+                    },
+                };
+            const inputsSnapshotPath = this.state.inputsSnapshotPath ?? INPUTS_SNAPSHOT_FILENAME;
+            const inputsSnapshotHash = await writeJsonSnapshot(join(this.runDir, inputsSnapshotPath), variables);
+            const nextState = structuredClone(this.state);
+            nextState.inputsSnapshotPath = inputsSnapshotPath;
+            nextState.inputsSnapshotHash = inputsSnapshotHash;
+            nextState.variables = variables;
+            nextState.workflow_context = structuredClone(committedWorkflowContext);
+            this.variables = variables;
+            this.state = nextState;
+            await this.writeState();
+            try {
+                await writeJsonAtomic(join(getStateDir(), "workflow-contexts", `${this.state.runId}.json`), committedWorkflowContext);
+            }
+            catch (error) {
+                logger.warn({ runId: this.state.runId, err: error }, "failed to write workflow context provenance");
+            }
+        });
+    }
+    async observeContextBuilderLifecycleTransition(transition) {
+        const previous = transition.previous === null
+            ? null
+            : parseContextBuilderLifecycleStateV1(transition.previous);
+        const snapshot = parseContextBuilderLifecycleStateV1(transition.snapshot);
+        if (snapshot.operation_kind !== "workflow_preflight") {
+            throw new Error("workflow state accepts only workflow_preflight lifecycle snapshots");
+        }
+        if (previous !== null && !isLegalContextBuilderLifecycleTransition(previous, snapshot)) {
+            throw new Error("illegal workflow context-builder lifecycle transition");
+        }
+        await this.enqueueWrite(async () => {
+            const current = this.state.context_builder_lifecycle;
+            if (previous !== null) {
+                if (current === undefined || !lifecycleSnapshotsEqual(current, previous)) {
+                    throw new Error("workflow context-builder lifecycle predecessor mismatch");
+                }
+            }
+            const nextState = structuredClone(this.state);
+            nextState.context_builder_lifecycle = structuredClone(snapshot);
+            this.state = nextState;
+            await this.writeState();
+        });
+    }
+    /**
+     * Persist the plan-review aux subtree (A-5).
+     *
+     * Called immediately after `create()` and BEFORE any seat can be dispatched.
+     * It cannot be a `create()` argument in the runtime path: the subtree binds
+     * the canonical snapshot's hash, and the snapshot can only be materialized
+     * into a run directory that `create()` itself makes — and the plan requires a
+     * snapshot/digest failure to be a POST-state `failed` run, not a pre-state
+     * reject. `create()` still validates+persists the subtree when a caller
+     * already has one (e.g. a rebuild), so both entry points are strict.
+     */
+    async commitPlanReviewState(state) {
+        const validated = PlanReviewRunStateSchema.parse(state);
+        await this.enqueueWrite(async () => {
+            const nextState = structuredClone(this.state);
+            nextState.plan_review_state = validated;
+            this.state = nextState;
+            await this.writeState();
+        });
+    }
+    /**
+     * Record that the plan-review context policy was actually enforced at the
+     * preflight seam, together with the post-filtered copied paths (A-7).
+     *
+     * Committed before stage dispatch so a resumed run replays the same enforced
+     * exclusion instead of re-deriving it from caller variables.
+     */
+    async commitPlanReviewContextProvenance(provenance) {
+        await this.enqueueWrite(async () => {
+            const current = this.state.plan_review_state;
+            if (current === undefined) {
+                return;
+            }
+            const nextState = structuredClone(this.state);
+            nextState.plan_review_state = PlanReviewRunStateSchema.parse({
+                ...current,
+                context_provenance: provenance,
+            });
+            this.state = nextState;
+            await this.writeState();
+        });
+    }
+    /** Persisted-state terminal projection used only by the engine's opt-in policy. */
+    async projectPlanReviewTerminalResult() {
+        if (this.planReviewTerminalResult !== undefined) {
+            return this.planReviewTerminalResult;
+        }
+        const state = this.runState;
+        const restored = await restorePlanReviewRun(this.runDir, state.plan_review_state, resolvePlanReviewChairmanCallId(this.definition));
+        const envelope = await projectPlanReviewRun({
+            runDir: this.runDir,
+            state,
+            contract: restored.contract,
+            execution_contract_digest: restored.execution_contract_digest,
+        });
+        this.planReviewTerminalResult = serializePlanReviewResult(envelope);
+        return this.planReviewTerminalResult;
+    }
+    /**
+     * Refresh this store's own `run.lock` mtime so a reader that cannot verify our
+     * PID namespace (a containerized or bind-mounted dashboard) has portable
+     * liveness evidence. Observation only — nothing in this module reads the
+     * mtime, and no control path consults it.
+     */
+    startHeartbeat() {
+        this.heartbeat = setInterval(() => {
+            // Single-flight: never overlap beats. Skip this tick if we no longer hold
+            // the lock, a prior beat latched an error, or a beat is still running. The
+            // callback must NOT await — a slow fstat/read must not stall the interval.
+            if (!this.lockHeld ||
+                this.heartbeatError != null ||
+                this.heartbeatInFlight != null) {
+                return;
+            }
+            const beat = this.beatRunLock()
+                .catch((error) => {
+                // Losing liveness REPORTING must never lose the run: latch and log, and
+                // never let this reach enqueueWrite. This deliberately diverges from
+                // credential-store's assertHealthy(), where the lock protects a shared
+                // file; here it only decorates a dashboard observation.
+                if (error instanceof RunLockOwnershipLostError) {
+                    // PROVEN ownership loss (a complete valid record whose token is not
+                    // ours): revoke unlink authority BEFORE latching/logging so close()
+                    // cannot delete a lock a different owner now holds.
+                    this.lockHeld = false;
+                }
+                // ENOENT / EACCES / EIO / ELOOP / non-regular-file / oversized /
+                // malformed JSON / invalid record / short read stay "ordinary" and DO
+                // NOT clear lockHeld. That is safe ONLY under the confirmed
+                // SINGLE-PRODUCER topology: no other writer holds this exact lock, so a
+                // transient read failure is not evidence of a foreign owner and ENOENT
+                // is our own recycled/removed path, not a takeover. A future
+                // multi-producer change MUST revisit this asymmetry.
+                //
+                // Latching is PERMANENT for this run: a single ordinary transient
+                // failure (e.g. a rejected utimes / EIO) halts THIS run's heartbeat
+                // with NO retry — retry was deliberately omitted as over-engineering.
+                // A foreign-host dashboard may therefore age a live-but-transiently-
+                // erroring run to `orphaned` and never self-correct until the run ends.
+                // That is fail-safe: it only stops a liveness DECORATION, never mutates
+                // run state and never fails the run.
+                this.heartbeatError = error instanceof Error ? error : new Error(String(error));
+                logger.warn({ run_dir: this.runDir, err: this.heartbeatError.message }, "workflow run lock heartbeat stopped");
+            })
+                .finally(() => {
+                // Identity guard: only an obsolete completion of the CURRENT in-flight
+                // beat clears the slot; a newer promise is never clobbered.
+                if (this.heartbeatInFlight === beat) {
+                    this.heartbeatInFlight = undefined;
+                }
+            });
+            this.heartbeatInFlight = beat;
+        }, RUN_LOCK_HEARTBEAT_MS);
+        this.heartbeat.unref();
+    }
+    /**
+     * One descriptor-bound heartbeat beat: open the lock O_NOFOLLOW read-only,
+     * fstat + bounded positional read through the SAME handle, verify it is still
+     * our token, then utimes that still-open handle. Throws
+     * `RunLockOwnershipLostError` only on a complete valid record whose token
+     * differs; every other failure is an ordinary Error.
+     */
+    async beatRunLock() {
+        if (!this.lockHeld || this.heartbeatError != null)
+            return;
+        const handle = await open(this.lockPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
+        let propagating;
+        try {
+            const stats = await handle.stat();
+            if (!stats.isFile()) {
+                throw new Error(`run.lock at ${this.lockPath} is not a regular file`);
+            }
+            if (stats.size > RUN_LOCK_MAX_BYTES) {
+                throw new Error(`run.lock at ${this.lockPath} exceeds ${RUN_LOCK_MAX_BYTES} bytes`);
+            }
+            const size = Number(stats.size);
+            const buffer = Buffer.allocUnsafe(size);
+            let offset = 0;
+            while (offset < size) {
+                const { bytesRead } = await handle.read(buffer, offset, size - offset, offset);
+                if (bytesRead === 0) {
+                    throw new Error(`run.lock at ${this.lockPath} ended prematurely after ${offset} of ${size} bytes`);
+                }
+                offset += bytesRead;
+            }
+            const value = JSON.parse(buffer.toString("utf-8"));
+            if (!isRunLockRecord(value)) {
+                throw new Error(`run.lock at ${this.lockPath} is not a valid lock record`);
+            }
+            if (typeof value.token !== "string") {
+                // A present-but-mistyped token is a corrupt/invalid record, NOT proven
+                // ownership loss: treat it as an ordinary heartbeat error so it does not
+                // revoke unlink authority. Under single-producer this producer always
+                // writes a string token; this only guards hostile/corrupt lock bodies.
+                throw new Error(`run.lock token is not a string at ${this.lockPath}`);
+            }
+            if (value.token !== this.lockToken) {
+                throw new RunLockOwnershipLostError(this.lockPath, "lock token no longer matches this store");
+            }
+            const now = new Date();
+            await handle.utimes(now, now);
+        }
+        catch (error) {
+            propagating = error;
+            throw error;
+        }
+        finally {
+            try {
+                await handle.close();
+            }
+            catch (closeError) {
+                // Swallow a close failure only when another error is already
+                // propagating; otherwise surface it.
+                if (propagating === undefined)
+                    throw closeError;
+            }
+        }
+    }
+    async stopHeartbeat() {
+        if (this.heartbeat !== undefined) {
+            clearInterval(this.heartbeat);
+            this.heartbeat = undefined;
+        }
+        // At most one beat is ever active. Await it, swallowing the already-latched
+        // failure; do NOT loop.
+        const inFlight = this.heartbeatInFlight;
+        if (inFlight !== undefined) {
+            await inFlight.catch(() => { });
+        }
+    }
     async close() {
         await this.flush();
+        // Ordering is load-bearing in both directions. Not before flush(): draining
+        // the write chain can take time, and a lock aging while the run is provably
+        // alive reads `orphaned` to a foreign host. Not after the unlink: a beat
+        // racing the unlink would utimes a deleted — or recycled — path.
+        await this.stopHeartbeat();
         if (this.state.status === "succeeded" && !this.debugKeepArtifacts) {
-            this.lockHeld = false;
-            await rm(this.runDir, { recursive: true, force: true });
+            // Successful runs remain observable until their retention window expires.
+            // Release ownership so classification/pruning and immediate run-ID recycling
+            // can safely operate on both synchronous and asynchronous successes.
+            if (this.lockHeld) {
+                await safeUnlink(this.lockPath);
+                this.lockHeld = false;
+            }
             return;
         }
         if (this.lockHeld) {
@@ -401,12 +1063,34 @@ export class WorkflowStateStore {
             const nextState = structuredClone(this.state);
             const callEntry = getCallEntry(nextState, event.stageId, event.callId);
             callEntry.retryCount = event.nextRetryCount;
-            callEntry.errorMessage = event.error;
+            callEntry.errorMessage = capPersistedError(event.error);
             callEntry.failureClass = event.failureClass;
             this.state = nextState;
             await this.writeState();
         });
     }
+    /** Runs inside the caller's existing queue task; never enqueue from here. */
+    async finalizePlanReviewTerminalIntent(intent) {
+        const previousState = this.state;
+        const intended = applyPlanReviewTerminalIntent(previousState, intent);
+        if (intended === undefined) {
+            return;
+        }
+        const finalized = await finalizePlanReviewTerminalState({
+            runDir: this.runDir,
+            previousState,
+            intendedState: intended,
+            intent,
+            chairmanCallId: resolvePlanReviewChairmanCallId(this.definition),
+            serializedResult: this.planReviewTerminalResult,
+        });
+        validateStrictStateSubtrees(finalized.state, join(this.runDir, STATE_FILENAME));
+        this.state = finalized.state;
+        await this.writeState();
+        if (finalized.throwAfterCommit !== undefined) {
+            throw finalized.throwAfterCommit;
+        }
+    }
     async enqueueWrite(task) {
         const next = this.writeChain.then(task, task);
         this.writeChain = next.catch(() => { });
@@ -420,7 +1104,7 @@ async function prepareRunDirForCreate(runDir, runId) {
     await mkdir(dirname(runDir), { recursive: true });
     try {
         await mkdir(runDir);
-        return false;
+        return { recycled: false };
     }
     catch (error) {
         if (!(error instanceof Error) || !("code" in error) || error.code !== "EEXIST") {
@@ -428,7 +1112,7 @@ async function prepareRunDirForCreate(runDir, runId) {
         }
     }
     const recycled = await recycleSucceededRunDir(runDir);
-    if (recycled) {
+    if (recycled.recycled) {
         const recycledPath = `${runDir}.recycled`;
         try {
             await mkdir(runDir);
@@ -445,7 +1129,7 @@ async function prepareRunDirForCreate(runDir, runId) {
             }
             throw error;
         }
-        return true;
+        return recycled;
     }
     throw duplicateRunDirError(runId);
 }
@@ -453,6 +1137,14 @@ async function pruneRunDirectoriesBeforeCreate(runDir) {
     const runsRoot = getRunsDir();
     if (dirname(runDir) !== runsRoot)
         return;
+    // Reconcile before pruning so a stale running run terminalized here becomes
+    // prunable by a later create once past its retention window.
+    try {
+        await reconcileStaleRunDirectories(runsRoot);
+    }
+    catch (error) {
+        logger.warn({ runs_root_dir: runsRoot, err: error instanceof Error ? error.message : String(error) }, "stale workflow run reconcile failed before create");
+    }
     try {
         await pruneRunDirectories(runsRoot, getRetentionConfig());
     }
@@ -460,11 +1152,89 @@ async function pruneRunDirectoriesBeforeCreate(runDir) {
         logger.warn({ runs_root_dir: runsRoot, err: error instanceof Error ? error.message : String(error) }, "workflow run directory pruning failed before create");
     }
 }
+/**
+ * Bounds the per-create duplicate-replay scan: at most this many sibling run
+ * directories have their persisted state read. Best-effort by design — a runs
+ * root larger than the cap simply yields a possibly-incomplete signal.
+ */
+const MAX_DUPLICATE_REPLAY_SCAN_DIRS = 1_000;
+/**
+ * Warn-only duplicate-replay detection (failure investigation F6): find a
+ * sibling run whose persisted `workflowSnapshotHash` AND `inputsSnapshotHash`
+ * both equal the run being created, and whose status is terminal `failed`
+ * with `failureClass:"permanent"`. Cancelled runs and transient failures are
+ * deliberately excluded; with multiple matches the most recent `completedAt`
+ * wins.
+ *
+ * WARN-ONLY, never a gate: identical hashes do NOT imply an identical outcome.
+ * The hashes cover only the workflow definition and resolved input variables —
+ * not workspace content, which may have changed since the prior run — and
+ * providers are nondeterministic, so an exact replay can legitimately succeed.
+ * This function therefore never throws and never blocks creation: any scan or
+ * per-sibling read error degrades silently (or with a warn log) to "no
+ * signal", and admission, mutex, cap, and state-creation behavior are
+ * unchanged.
+ */
+async function detectDuplicateReplay(runDir, workflowSnapshotHash, inputsSnapshotHash) {
+    try {
+        const runsRoot = dirname(runDir);
+        const selfName = basename(runDir);
+        let entries;
+        try {
+            entries = await readdir(runsRoot, { withFileTypes: true });
+        }
+        catch {
+            return undefined;
+        }
+        let best;
+        let bestCompletedAtMs = Number.NEGATIVE_INFINITY;
+        let examined = 0;
+        for (const entry of entries) {
+            if (!entry.isDirectory() || entry.name === selfName)
+                continue;
+            if (examined >= MAX_DUPLICATE_REPLAY_SCAN_DIRS)
+                break;
+            examined += 1;
+            let state;
+            try {
+                state = readRunStateSync(join(runsRoot, entry.name));
+            }
+            catch {
+                // Unreadable siblings are skipped silently — mid-write, corrupt, or
+                // foreign directories must never affect the new run.
+                continue;
+            }
+            if (state.status !== "failed" ||
+                state.failureClass !== "permanent" ||
+                state.workflowSnapshotHash !== workflowSnapshotHash ||
+                state.inputsSnapshotHash !== inputsSnapshotHash) {
+                continue;
+            }
+            const parsedCompletedAt = state.completedAt !== undefined
+                ? Date.parse(state.completedAt)
+                : Number.NaN;
+            const orderKey = Number.isFinite(parsedCompletedAt)
+                ? parsedCompletedAt
+                : Number.NEGATIVE_INFINITY;
+            if (best === undefined || orderKey > bestCompletedAtMs) {
+                best = { runId: state.runId, completedAt: state.completedAt };
+                bestCompletedAtMs = orderKey;
+            }
+        }
+        return best;
+    }
+    catch (error) {
+        logger.warn({ run_dir: runDir, err: error instanceof Error ? error.message : String(error) }, "duplicate-replay scan failed; proceeding without signal");
+        return undefined;
+    }
+}
 function normalizePruneOptions(options) {
     return {
         retainFailedDays: normalizeRetentionDays(options?.retainFailedDays, DEFAULT_RETAIN_FAILED_DAYS),
         retainCancelledDays: normalizeRetentionDays(options?.retainCancelledDays, DEFAULT_RETAIN_CANCELLED_DAYS),
         retainDebugDays: normalizeRetentionDays(options?.retainDebugDays, DEFAULT_RETAIN_DEBUG_DAYS),
+        retainAsyncDays: normalizeRetentionDays(options?.retainAsyncDays, DEFAULT_RETAIN_ASYNC_DAYS),
+        retainSucceededDays: normalizeRetentionDays(options?.retainSucceededDays, DEFAULT_RETAIN_SUCCEEDED_DAYS),
     };
 }
 function normalizeRetentionDays(value, fallback) {
@@ -478,6 +1248,10 @@ function retentionDaysForClassification(classification, options) {
             return options.retainCancelledDays;
         case "debug_kept":
             return options.retainDebugDays;
+        case "succeeded_async":
+            return options.retainAsyncDays;
+        case "succeeded_sync":
+            return options.retainSucceededDays;
         case "active_locked":
         case "running_or_pending":
         case "unknown":
@@ -493,6 +1267,44 @@ async function tryAcquirePruneLock(runDirPath) {
         return false;
     }
 }
+async function admitWorkflowRunGeneration(runsRootDir, runDirPath, runId) {
+    const requestedRoot = resolve(runsRootDir);
+    const requestedRun = resolve(runDirPath);
+    if (requestedRun !== join(requestedRoot, runId)) {
+        throw new Error("workflow run identity is not contained");
+    }
+    const rootLinked = await lstat(requestedRoot);
+    if (rootLinked.isSymbolicLink() || !rootLinked.isDirectory()) {
+        throw new Error("workflow runs root identity is unsafe");
+    }
+    const canonicalRoot = await realpath(requestedRoot);
+    const canonicalRun = join(canonicalRoot, runId);
+    const flags = fsConstants.O_RDONLY |
+        (fsConstants.O_DIRECTORY ?? 0) |
+        (fsConstants.O_NOFOLLOW ?? 0);
+    const rootHandle = await open(canonicalRoot, flags);
+    let runHandle;
+    try {
+        runHandle = await open(canonicalRun, flags);
+        const rootStats = await rootHandle.stat({ bigint: true });
+        const runStats = await runHandle.stat({ bigint: true });
+        if (!rootStats.isDirectory() ||
+            !runStats.isDirectory() ||
+            rootStats.nlink === 0n ||
+            runStats.nlink === 0n ||
+            BigInt(rootLinked.dev) !== rootStats.dev ||
+            BigInt(rootLinked.ino) !== rootStats.ino ||
+            await resolveOpenedDescriptorPath(rootHandle.fd, canonicalRoot) !== canonicalRoot ||
+            await resolveOpenedDescriptorPath(runHandle.fd, canonicalRun) !== canonicalRun) {
+            throw new Error("workflow run identity changed during admission");
+        }
+        return deriveWorkflowRunGeneration(runId, rootStats, runStats);
+    }
+    finally {
+        await runHandle?.close().catch(() => undefined);
+        await rootHandle.close().catch(() => undefined);
+    }
+}
 async function getRunDirectoryAgeMs(runDirPath) {
     const completedAt = await readRunCompletedAt(runDirPath);
     if (completedAt != null) {
@@ -522,8 +1334,85 @@ function readRunStateSync(runDir) {
     if (state.state_version !== 1) {
         throw new Error(`Unsupported state_version ${String(state.state_version)} in ${statePath}`);
     }
+    validateStrictStateSubtrees(state, statePath);
     return state;
 }
+/**
+ * Strictly validate optional state subtrees that are resume or lifecycle
+ * authorities. Older state_version:1 files may omit either subtree.
+ *
+ * The rest of RunState is still validated only by `state_version`. These are
+ * exceptions because resume derives execution bindings from plan_review_state,
+ * while lifecycle is a strict bounded H0 contract and must never acquire
+ * arbitrary producer fields.
+ */
+function validateStrictStateSubtrees(state, statePath) {
+    if (state.workflow_context !== undefined) {
+        try {
+            state = {
+                ...state,
+                workflow_context: parseWorkflowContext(state.workflow_context),
+            };
+        }
+        catch (error) {
+            throw new Error(`Invalid workflow_context in ${statePath}: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    let parsedLifecycle;
+    if (state.context_builder_lifecycle !== undefined) {
+        try {
+            parsedLifecycle = parseContextBuilderLifecycleStateV1(state.context_builder_lifecycle);
+        }
+        catch (error) {
+            throw new Error(`Invalid context_builder_lifecycle in ${statePath}: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    const nestedLifecycle = state.workflow_context?.context_builder.lifecycle;
+    let parsedNestedLifecycle;
+    if (nestedLifecycle !== undefined) {
+        try {
+            parsedNestedLifecycle = parseContextBuilderLifecycleStateV1(nestedLifecycle);
+        }
+        catch (error) {
+            throw new Error(`Invalid workflow_context.context_builder.lifecycle in ${statePath}: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    if (state.workflow_context !== undefined) {
+        const sourceRunId = state.workflow_context.context_builder.source_run_id;
+        for (const lifecycle of [parsedLifecycle, parsedNestedLifecycle]) {
+            if (lifecycle !== undefined &&
+                ((lifecycle.status !== "succeeded" && lifecycle.status !== "partial") ||
+                    lifecycle.phase !== "complete" ||
+                    lifecycle.artifact_run_id !== sourceRunId)) {
+                throw new Error(`Inconsistent workflow context lifecycle in ${statePath}`);
+            }
+        }
+        if (parsedLifecycle !== undefined &&
+            parsedNestedLifecycle !== undefined &&
+            !lifecycleSnapshotsEqual(parsedLifecycle, parsedNestedLifecycle)) {
+            throw new Error(`Inconsistent workflow context lifecycle in ${statePath}`);
+        }
+    }
+    if (state.plan_review_state !== undefined) {
+        const parsed = PlanReviewRunStateSchema.safeParse(state.plan_review_state);
+        if (!parsed.success) {
+            throw new Error(`Invalid plan_review_state in ${statePath}: ${parsed.error.issues
+                .map((issue) => `${issue.path.join(".") || "plan_review_state"}: ${issue.message}`)
+                .join("; ")}`);
+        }
+    }
+    if (state.plan_review_error !== undefined) {
+        const parsed = CouncilPlanReviewErrorSchema.safeParse(state.plan_review_error);
+        if (!parsed.success) {
+            throw new Error(`Invalid plan_review_error in ${statePath}: ${parsed.error.issues
+                .map((issue) => `${issue.path.join(".") || "plan_review_error"}: ${issue.message}`)
+                .join("; ")}`);
+        }
+    }
+}
+function lifecycleSnapshotsEqual(left, right) {
+    return JSON.stringify(left) === JSON.stringify(right);
+}
 function classifyRunLockSync(lockPath) {
     try {
         const lock = JSON.parse(readFileSync(lockPath, "utf-8"));
@@ -540,11 +1429,275 @@ function classifyRunLockSync(lockPath) {
         return code === "ENOENT" ? "stale_or_absent" : "unknown";
     }
 }
+export async function readRunStateForStatus(runId) {
+    const runDir = getRunStateDir(runId);
+    let state;
+    try {
+        state = await readRunState(runDir);
+    }
+    catch (error) {
+        const code = error.code;
+        if (code === "ENOENT") {
+            return null;
+        }
+        throw error;
+    }
+    const lockStatus = classifyRunLockSync(join(runDir, RUN_LOCK_FILENAME));
+    return {
+        state,
+        lock: lockStatus,
+    };
+}
+/**
+ * Reap a run that has no live worker (orphaned/stale/prior-process) by writing a
+ * terminal "cancelled" state and unlinking its lock. Used by cancel_workflow when
+ * the run is not in this process's in-memory cancel registry.
+ *
+ * The run.lock is the single source of truth for ownership: this acquires the
+ * lock itself (reusing acquireRunLock, which atomically creates the lock and
+ * reclaims a stale same-host/dead-PID lock) BEFORE mutating, closing the TOCTOU
+ * where a concurrent resume_workflow could grab the lock between a caller's
+ * status read and the reap write.
+ *
+ * Returns:
+ *  - "locked"    — a LIVE lock blocks acquisition (held by a running worker, or a
+ *                  same-process blocking run); nothing is mutated.
+ *  - "ownership_unknown" — the lock is malformed/unreadable; nothing is mutated
+ *                           or reclaimed.
+ *  - "skipped"   — lock acquired but the run is already terminal (not
+ *                  pending/running); the lock is released and nothing is mutated.
+ *                  The caller re-reads to report already_terminal.
+ *  - "cancelled" — lock acquired and run was pending/running; terminal
+ *                  "cancelled" state is written and the lock removed.
+ *
+ * NOTE: This intentionally does NOT use WorkflowStateStore.resume() (which
+ * recovers + re-locks the run — the wrong intent here).
+ */
+export async function reapRunCancelled(runId) {
+    const runDir = getRunStateDir(runId);
+    try {
+        await acquireRunLock(runDir);
+    }
+    catch (error) {
+        if (error instanceof RunLockOwnershipUnknownError) {
+            return "ownership_unknown";
+        }
+        // A live lock holder (running worker or same-process blocking run) prevents
+        // acquisition; a truly stale/dead lock would have been reclaimed instead.
+        return "locked";
+    }
+    const lockPath = join(runDir, RUN_LOCK_FILENAME);
+    let lockHeld = true;
+    try {
+        const state = await readRunState(runDir);
+        if (state.status !== "pending" && state.status !== "running") {
+            return "skipped";
+        }
+        let nextState = structuredClone(state);
+        if (isPlanReviewRun(state)) {
+            const intent = {
+                status: "cancelled",
+                completedAt: new Date().toISOString(),
+                error: "Reaped: no live worker process",
+                replaceFailureClass: false,
+            };
+            const intended = applyPlanReviewTerminalIntent(state, intent);
+            if (intended === undefined) {
+                return "skipped";
+            }
+            const finalized = await finalizePlanReviewTerminalState({
+                runDir,
+                previousState: state,
+                intendedState: intended,
+                intent,
+            });
+            nextState = finalized.state;
+            validateStrictStateSubtrees(nextState, join(runDir, STATE_FILENAME));
+        }
+        else {
+            nextState.status = "cancelled";
+            nextState.completedAt = new Date().toISOString();
+            nextState.error = "Reaped: no live worker process";
+        }
+        await writeJsonAtomic(join(runDir, STATE_FILENAME), nextState);
+        await safeUnlink(lockPath);
+        lockHeld = false;
+        return "cancelled";
+    }
+    finally {
+        if (lockHeld) {
+            await safeUnlink(lockPath);
+        }
+    }
+}
+/**
+ * Terminalize run directories frozen as running/pending with no live worker.
+ *
+ * A worker that dies between state writes (e.g. during a retry backoff delay,
+ * where applyRetryUpdate has already persisted errorMessage/failureClass while
+ * the call state stays "running") leaves the run persisted as `running`
+ * forever: the only other reconcile lives in WorkflowStateStore.resume(),
+ * which non-resumable runs never reach, and pruning skips running_or_pending
+ * directories. This scan converts such runs to terminal `failed` so status
+ * projection and retention can move on. No new persisted status is introduced;
+ * STALE_RUN_RECONCILE_ERROR is the distinct terminal representation.
+ *
+ * Liveness authority is classifyRunLockSync's same-host PID probe only:
+ * a run is touched solely when its lock is "stale_or_absent" (missing lock or
+ * same-host dead PID). "held" (live PID, or ANY foreign-host lock) and
+ * "unknown" (malformed lock) are skipped entirely, and no decision here
+ * consults heartbeat mtime freshness. TOCTOU-safety mirrors reapRunCancelled:
+ * acquire the run.lock (no heartbeat declaration, like other reap
+ * acquisitions), re-read state under the lock, re-verify non-terminal, write
+ * once, then release. Idempotent: reconciled runs are terminal `failed` and
+ * count as skipped_terminal on re-scan.
+ */
+export async function reconcileStaleRunDirectories(runsRootDir) {
+    const result = {
+        scanned: 0,
+        reconciled: 0,
+        skipped_held: 0,
+        skipped_terminal: 0,
+        errors: 0,
+    };
+    let entries;
+    try {
+        entries = await readdir(runsRootDir, { withFileTypes: true });
+    }
+    catch (error) {
+        const code = error.code;
+        if (code !== "ENOENT") {
+            result.errors += 1;
+            logger.warn({ runs_root_dir: runsRootDir, err: error instanceof Error ? error.message : String(error) }, "failed to read workflow runs directory for stale-run reconcile");
+        }
+        return result;
+    }
+    for (const entry of entries) {
+        if (!entry.isDirectory()) {
+            continue;
+        }
+        const runDirPath = join(runsRootDir, entry.name);
+        result.scanned += 1;
+        let state;
+        try {
+            state = readRunStateSync(runDirPath);
+        }
+        catch (error) {
+            result.errors += 1;
+            logger.warn({ run_dir: runDirPath, err: error instanceof Error ? error.message : String(error) }, "skipping run with unreadable state during stale-run reconcile");
+            continue;
+        }
+        if (state.status !== "running" && state.status !== "pending") {
+            result.skipped_terminal += 1;
+            continue;
+        }
+        const lockStatus = classifyRunLockSync(join(runDirPath, RUN_LOCK_FILENAME));
+        if (lockStatus !== "stale_or_absent") {
+            // "held" (live same-host PID or any foreign-host lock) and "unknown"
+            // (malformed lock) are both untouchable — never infer liveness beyond
+            // the same-host PID probe.
+            result.skipped_held += 1;
+            continue;
+        }
+        try {
+            const outcome = await reconcileStaleRun(runDirPath);
+            result[outcome] += 1;
+        }
+        catch (error) {
+            result.errors += 1;
+            logger.warn({ run_dir: runDirPath, err: error instanceof Error ? error.message : String(error) }, "failed to reconcile stale workflow run directory");
+        }
+    }
+    logger.info({ runs_root_dir: runsRootDir, ...result }, "stale workflow run reconcile complete");
+    return result;
+}
+/**
+ * Terminalize one pre-screened stale run under its own lock. See
+ * reconcileStaleRunDirectories for the authority and safety model.
+ */
+async function reconcileStaleRun(runDir) {
+    try {
+        await acquireRunLock(runDir);
+    }
+    catch {
+        // A live lock holder appeared between the pre-screen and acquisition, or
+        // ownership is unknown; either way nothing is mutated.
+        return "skipped_held";
+    }
+    const lockPath = join(runDir, RUN_LOCK_FILENAME);
+    try {
+        const state = await readRunState(runDir);
+        if (state.status !== "pending" && state.status !== "running") {
+            return "skipped_terminal";
+        }
+        const reconciledAt = new Date().toISOString();
+        let nextState = structuredClone(state);
+        for (const stage of nextState.stages) {
+            for (const call of stage.calls) {
+                if (call.state === "running") {
+                    call.state = "failed";
+                    call.completedAt = reconciledAt;
+                    // Preserve an already-recorded error verbatim: a retry-scheduled
+                    // "permanent" classification must stay permanent (deliberately NOT
+                    // resume()'s downgrade-to-transient recovery behavior).
+                    if (call.errorMessage === undefined) {
+                        call.errorMessage = STALE_RUN_RECONCILE_CALL_ERROR;
+                    }
+                    if (call.failureClass === undefined) {
+                        call.failureClass = "transient";
+                    }
+                }
+            }
+            stage.state = deriveStageState(stage.calls);
+            stage.completedAt = deriveStageCompletedAt(stage.calls);
+        }
+        if (isPlanReviewRun(state)) {
+            const intent = {
+                status: "failed",
+                completedAt: reconciledAt,
+                error: STALE_RUN_RECONCILE_ERROR,
+                failureClass: "transient",
+                replaceFailureClass: true,
+            };
+            const intended = applyPlanReviewTerminalIntent(nextState, intent);
+            if (intended === undefined) {
+                return "skipped_terminal";
+            }
+            const finalized = await finalizePlanReviewTerminalState({
+                runDir,
+                previousState: state,
+                intendedState: intended,
+                intent,
+            });
+            nextState = finalized.state;
+            validateStrictStateSubtrees(nextState, join(runDir, STATE_FILENAME));
+        }
+        else {
+            nextState.status = "failed";
+            nextState.completedAt = reconciledAt;
+            nextState.error = STALE_RUN_RECONCILE_ERROR;
+            nextState.failureClass = "transient";
+        }
+        await writeJsonAtomic(join(runDir, STATE_FILENAME), nextState);
+        logger.info({ run_dir: runDir, run_id: state.runId }, "reconciled stale running workflow run to terminal failed");
+        return "reconciled";
+    }
+    finally {
+        await safeUnlink(lockPath);
+    }
+}
 function isRunLockRecord(value) {
     const record = asRecord(value);
-    return Number.isSafeInteger(record["pid"]) &&
-        typeof record["hostname"] === "string" &&
-        typeof record["startedAt"] === "string";
+    const pid = record["pid"];
+    const host = record["hostname"];
+    const startedAt = record["startedAt"];
+    return typeof pid === "number" &&
+        Number.isSafeInteger(pid) &&
+        pid > 0 &&
+        typeof host === "string" &&
+        host.length > 0 &&
+        typeof startedAt === "string" &&
+        Number.isFinite(Date.parse(startedAt));
 }
 function duplicateRunDirError(runId) {
     return new WorkflowError(`Run directory for run_id "${runId}" already exists. Use a unique run_id for new runs. If this is a failed/cancelled resumable run, try resume_workflow.`);
@@ -557,19 +1710,40 @@ async function recycleSucceededRunDir(runDir) {
         lockHeld = true;
     }
     catch {
-        return false;
+        return { recycled: false };
     }
     try {
         const state = await readRunState(runDir);
         if (state.status !== "succeeded") {
-            return false;
+            return { recycled: false };
+        }
+        const identity = await readSessionLogGeneration(runDir);
+        let retiredSessionLogGeneration;
+        if (identity.kind === "available") {
+            retiredSessionLogGeneration = identity.sessionLogGeneration;
+        }
+        else if (identity.kind === "legacy") {
+            try {
+                retiredSessionLogGeneration = await admitWorkflowRunGeneration(dirname(runDir), runDir, basename(runDir));
+            }
+            catch {
+                logger.warn({ run_id: basename(runDir) }, "legacy recycled workflow session-log generation could not be admitted");
+            }
+        }
+        else {
+            logger.warn({ run_id: basename(runDir) }, "recycled workflow session-log identity was unsafe");
         }
         await rename(runDir, `${runDir}.recycled`);
         lockHeld = false;
-        return true;
+        return {
+            recycled: true,
+            ...(retiredSessionLogGeneration === undefined
+                ? {}
+                : { retiredSessionLogGeneration }),
+        };
     }
     catch {
-        return false;
+        return { recycled: false };
     }
     finally {
         if (lockHeld) {
@@ -577,6 +1751,21 @@ async function recycleSucceededRunDir(runDir) {
         }
     }
 }
+async function cleanupWorkflowSessionLogGeneration(runsRoot, runId, sessionLogGeneration) {
+    try {
+        const cleanup = await removeSessionLogGeneration({
+            rootDir: join(dirname(resolve(runsRoot)), "session-logs"),
+            family: "workflow",
+            generation: sessionLogGeneration,
+        });
+        if (cleanup.kind === "unsafe" || cleanup.kind === "unavailable") {
+            logger.warn({ run_id: runId, cleanup: cleanup.kind }, "recycled workflow parent removed but exact session-log cleanup was unavailable");
+        }
+    }
+    catch {
+        logger.warn({ run_id: runId }, "recycled workflow parent removed but exact session-log cleanup failed");
+    }
+}
 function getCallEntry(state, stageId, callId) {
     const stage = state.stages.find((entry) => entry.stageId === stageId);
     if (stage == null) {
@@ -610,6 +1799,27 @@ function deriveStageCompletedAt(calls) {
         .sort();
     return completedValues.length > 0 ? completedValues[completedValues.length - 1] : undefined;
 }
+function capPersistedError(message) {
+    if (message === undefined) {
+        return undefined;
+    }
+    const encoded = Buffer.from(message, "utf-8");
+    if (encoded.length <= MAX_PERSISTED_ERROR_BYTES) {
+        return message;
+    }
+    let end = MAX_PERSISTED_ERROR_BYTES;
+    // Never split a multi-byte UTF-8 sequence: back off over continuation bytes
+    // so the kept head decodes cleanly.
+    while (end > 0) {
+        const boundaryByte = encoded[end];
+        if (boundaryByte === undefined || (boundaryByte & 0b1100_0000) !== 0b1000_0000) {
+            break;
+        }
+        end -= 1;
+    }
+    const omitted = encoded.length - end;
+    return `${encoded.subarray(0, end).toString("utf-8")} …[truncated ${omitted} bytes]`;
+}
 async function persistOutput(runDir, stageId, callId, outputText) {
     const sizeBytes = Buffer.byteLength(outputText, "utf-8");
     if (sizeBytes <= OUTPUT_INLINE_THRESHOLD_BYTES) {
@@ -629,6 +1839,7 @@ async function buildResumeState(definition, state, runDir) {
     const completedCalls = new Map();
     const preparedInputs = new Map();
     const failedCalls = new Map();
+    const nextRetryCounts = new Map();
     const firstIncompleteStageIndex = definition.stages.findIndex((stage) => {
         const persistedStage = state.stages.find((entry) => entry.stageId === stage.id);
         return !persistedStage?.calls.every((call) => call.state === "succeeded");
@@ -653,6 +1864,7 @@ async function buildResumeState(definition, state, runDir) {
                         stageId: stage.id,
                         callId: call.id,
                         error: entry.errorMessage,
+                        terminal: true,
                     });
                 }
             }
@@ -670,10 +1882,16 @@ async function buildResumeState(definition, state, runDir) {
                         stageId: stage.id,
                         callId: call.id,
                         error: entry.errorMessage,
+                        // Only terminal failures are persisted as `failed`; retryable
+                        // intermediate attempts remain `running` with attemptsConsumed.
+                        // Never redispatch a persisted failed call after process restart.
+                        terminal: true,
                     });
                 }
                 if (entry?.materializedInputPath != null) {
                     preparedInputs.set(`${stage.id}.${call.id}`, await readJsonFile(join(runDir, entry.materializedInputPath)));
+                    nextRetryCounts.set(`${stage.id}.${call.id}`, entry.attemptsConsumed ??
+                        (entry.state === "running" ? entry.retryCount + 1 : entry.retryCount));
                 }
             }
         }
@@ -682,6 +1900,7 @@ async function buildResumeState(definition, state, runDir) {
         completedCalls,
         preparedInputs,
         failedCalls,
+        nextRetryCounts,
     };
 }
 async function rehydrateExecutedCall(runDir, entry) {
@@ -878,6 +2097,7 @@ async function readRunState(runDir) {
     if (state.state_version !== 1) {
         throw new Error(`Unsupported state_version ${String(state.state_version)} in ${statePath}`);
     }
+    validateStrictStateSubtrees(state, statePath);
     return state;
 }
 async function verifySnapshotHash(filePath, expectedHash, label) {
@@ -913,12 +2133,25 @@ function hashContent(content) {
 async function readJsonFile(filePath) {
     return JSON.parse(await readFile(filePath, "utf-8"));
 }
-async function acquireRunLock(runDir) {
+/**
+ * Atomically create `run.lock`, reclaiming a same-host dead-PID lock.
+ *
+ * `declareHeartbeat` is deliberately opt-in: only a caller that goes on to hold
+ * the lock behind a live `WorkflowStateStore` (create/resume) beats it. The
+ * prune/reap/recycle callers hold the lock briefly with no beating owner, so
+ * declaring a heartbeat there would publish a capability claim nobody honors —
+ * and a foreign-host reader would then age it into a false `orphaned`.
+ */
+async function acquireRunLock(runDir, options = {}) {
     const lockPath = join(runDir, RUN_LOCK_FILENAME);
     const record = {
         pid: process.pid,
         hostname: hostname(),
         startedAt: new Date().toISOString(),
+        token: randomBytes(16).toString("base64url"),
+        ...(options.declareHeartbeat === true
+            ? { heartbeatIntervalMs: RUN_LOCK_HEARTBEAT_MS }
+            : {}),
     };
     for (let attempt = 0; attempt < 2; attempt += 1) {
         try {
@@ -929,13 +2162,26 @@ async function acquireRunLock(runDir) {
             finally {
                 await handle.close();
             }
-            return;
+            return record;
         }
         catch (error) {
             if (!(error instanceof Error) || !("code" in error) || error.code !== "EEXIST") {
                 throw error;
             }
-            const existing = await readJsonFile(lockPath);
+            let existingValue;
+            try {
+                existingValue = await readJsonFile(lockPath);
+            }
+            catch (readError) {
+                if (readError.code === "ENOENT") {
+                    continue;
+                }
+                throw new RunLockOwnershipUnknownError(lockPath, readError instanceof Error ? readError.message : String(readError));
+            }
+            if (!isRunLockRecord(existingValue)) {
+                throw new RunLockOwnershipUnknownError(lockPath, "record must contain a positive integer pid, non-empty hostname, and valid startedAt timestamp");
+            }
+            const existing = existingValue;
             if (existing.hostname === hostname() && !isPidLive(existing.pid)) {
                 await unlink(lockPath);
                 continue;
@@ -943,6 +2189,7 @@ async function acquireRunLock(runDir) {
             throw new Error(`Run lock is held for ${runDir} by pid ${String(existing.pid)} on ${existing.hostname} since ${existing.startedAt}`);
         }
     }
+    throw new Error(`Failed to acquire run lock for ${runDir} after 2 attempts; lock ownership was not established`);
 }
 function isPidLive(pid) {
     try {