@deksden-com/dd-flow-cli 0.9.0-beta.143 → 0.9.0-beta.145

package/dist/services/run-controller-recovery.js

@@ -255,7 +255,12 @@ export async function resumeControllerSession(context, row, manifest, state, ass
                     || current.process.lease_expires_at <= context.now() || metadata.run_generation !== row.generation || !processIsAlive(current.process))
                     throw new AppError("recovery_capacity_owner_changed", "Replacement native owner changed during capacity wait", 1);
             },
-            decision: () => recoveryGuard(context, row.project_id, row.run_id)?.status === "recovered" ? "completed" : "continue",
+            decision: (boundary) => {
+                if (boundary && (boundary.lifecycle_operation !== "recovery_accept" || boundary.recovery_id !== recovery.recovery_id
+                    || !context.db.get("SELECT 1 FROM run_recovery_bindings b JOIN hook_events h ON h.event_key = b.hook_event_key WHERE b.recovery_id = ? AND b.status = 'accepted' AND b.daemon_id = ? AND b.provider_session_id = ? AND h.project_id = ? AND h.status = 'claimed' AND h.event_key = ?", [recovery.recovery_id, daemonId, session.id, row.project_id, boundary.native_hook_event_key])))
+                    return "reconciliation_blocked";
+                return recoveryGuard(context, row.project_id, row.run_id)?.status === "recovered" ? "completed" : "continue";
+            },
             retainIntent: intent => retainRecoveryCapacityIntent(executionContext, { projectId: row.project_id, runId: row.run_id, recoveryId: recovery.recovery_id,
                 controllerId: row.controller_id, generation: row.generation, daemonId, intent, prompt: continuationText }),
             prompt: (operationId, prompt) => sendAck(operationId, prompt) });

package/dist/services/run-controller.js

@@ -25,11 +25,13 @@ import { sameSessionProfile, runtimeHarness } from "./execution-policy.js";
 import { selectControllerExecution } from "./controller-execution.js";
 import { resolveHarnessCommand, harnessRuntimeArguments } from "./harness-config.js";
 import { adapterSessionId, prepareHarnessFlowExecutable, runHarnessAdapter } from "./harness-adapter.js";
-import { assertManagedProcessLease, confirmManagedProcess, renewManagedProcess, finishManagedProcess, processStartedAt, processTreeIsAlive, resourceHome } from "./managed-processes.js";
+import { assertManagedProcessLease, confirmManagedProcess, prepareManagedProcessConfirmation, renewManagedProcess, finishManagedProcess, processStartedAt, processTreeIsAlive, resourceHome } from "./managed-processes.js";
 import { managedLeaseMonitor } from "./managed-lease-monitor.js";
+import { physicalStartup } from "./physical-startup.js";
 import { readNativeDaemonState, requireManagedDaemonBinding } from "./managed-daemon-binding.js";
 import { initCodexHome } from "./hooks.js";
 import { launchExternalWork } from "./external-work-launch.js";
+import { runControllerMemoryTask } from "./controller-memory-task.js";
 import { controllerNativeChildren, controllerStageEntryPrompt, controllerStageResumePrompt, nextControllerFanout } from "./controller-fanout.js";
 import { claimQueuedMergeRequest } from "./merge-server.js";
 import { prepareVnextCodeWorkspace } from "./vnext-code.js";
@@ -38,7 +40,7 @@ import { assertNoCorrelatedRejection } from "./continuation-outcome.js";
 import { spawnController, recordControllerLauncher, claimControllerProcess, markControllerDispatchReady, assertControllerLeaseAllowed } from "./run-controller-process.js";
 import { controllerAdapter, controllerEnvironment, sessionArguments } from "./run-controller-adapter.js";
 import { retainedChildObserver } from "./controller-child-observer.js";
-import { capacityDelay, continueProviderCapacity, eligibleProviderChildOverload, providerChildFailure } from "./provider-capacity.js";
+import { capacityDelay, continueProviderCapacity, eligibleProviderChildOverload, providerChildFailure, ownedProviderBoundary } from "./provider-capacity.js";
 import { resumeControllerSession, failControllerRecovery } from "./run-controller-recovery.js";
 import { captureControllerBoundary } from "./run-controller-capture.js";
 import { requestRunControl } from "./run-control-worker.js";
@@ -215,92 +217,120 @@ export async function serveRunController(context, input) {
     const row = requireController(context, input.controllerId);
     if (row.project_id !== project.id || row.run_id !== input.runId || row.owner_token !== context.env.DD_FLOW_CONTROLLER_TOKEN)
         throw new AppError("controller_owner_mismatch", "Managed serve does not own this launch", 1);
-    claimControllerProcess(context, row);
-    if (!row.process_id || !row.process_lease_token)
-        throw new AppError("controller_owner_mismatch", "Managed owner has no resource lease", 1);
-    const confirmed = confirmManagedProcess(context, { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid, processGroupId: process.pid, ownerPid: process.pid });
-    const lease = controllerLeaseMonitor(attemptMs => renewManagedProcess({ ...context, resourceDatabaseMode: "prepared_existing", databaseDeadlineAt: Date.now() + attemptMs }, { id: row.process_id, leaseToken: row.process_lease_token }), confirmed.lease_expires_at);
-    const state = JSON.parse(row.state_json);
-    // Resource and RUN commits are distinct effects. A RUN keepalive failure must
-    // neither erase a resource receipt nor grant RUN dispatch authority.
-    const mergeLease = controllerLeaseMonitor(attemptMs => {
-        const expiry = new Date(Date.now() + 120_000).toISOString();
-        const db = context.db.withDeadline(Date.now() + attemptMs);
-        if (db.get("SELECT 1 FROM merge_requests WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status = 'dispatching' LIMIT 1", [row.project_id, row.run_id, row.controller_id]))
-            db.writeTransaction(() => {
-                const guard = recoveryGuard({ ...context, db }, row.project_id, row.run_id);
-                if ((guard?.generation ?? 0) !== row.generation || ["draining", "sealed", "resuming"].includes(guard?.status ?? ""))
-                    throw new AppError("controller_lease_lost", "RUN generation changed before MERGE keepalive", 1);
-                db.run("UPDATE merge_requests SET dispatch_lease_expires_at = ?, updated_at = ? WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status = 'dispatching'", [expiry, context.now(), row.project_id, row.run_id, row.controller_id]);
-            });
-        return { owned: true, lease_expires_at: expiry };
-    });
-    const assertLeaseFence = () => {
-        assertControllerLeaseAllowed(context, row);
-    };
-    const heartbeat = setInterval(() => {
-        lease.background(assertLeaseFence, () => { });
-        if (state.merge_claim)
-            mergeLease.background(assertLeaseFence, () => { });
-    }, 30_000);
-    heartbeat.unref();
-    const manifest = JSON.parse(row.manifest_json);
-    const assertPhysicalOwner = () => {
-        lease.assertNoRenewal();
-        if (state.merge_claim)
-            mergeLease.assertNoRenewal();
-        // This isolated read already bounds SQLite contention. A wall deadline
-        // before opening the reader mistakes scheduler delay for heartbeat loss;
-        // the observed row's actual expiry/token still fences productive dispatch.
-        assertManagedProcessLease(context, { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid });
-        const current = requireController(context, row.controller_id);
-        if (current.owner_pid !== process.pid || current.owner_token !== row.owner_token || !["running", "capturing", "waiting_for_user", "waiting_for_context", "waiting_for_children", "waiting_for_capacity", "waiting_for_merge"].includes(current.status))
-            throw new AppError("controller_owner_mismatch", "Controller no longer owns productive dispatch", 1);
-    };
-    const assertOwner = () => { assertPhysicalOwner(); assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation); };
+    let lease;
+    let mergeLease;
+    let heartbeat;
+    let state;
+    let claimedProcess = false;
     let primaryFailure;
     const retainSecondary = (phase, error) => {
         process.stderr.write(`${JSON.stringify({ controller_id: row.controller_id, phase, primary: errorRecord(primaryFailure), secondary: errorRecord(error) })}\n`);
     };
     try {
+        const currentState = state = JSON.parse(row.state_json);
+        const startup = physicalStartup(context.databaseDeadlineAt);
+        const assertStartingOwner = () => {
+            const current = requireController(context, row.controller_id);
+            if (current.owner_token !== row.owner_token || current.generation !== row.generation || current.owner_pid !== null || current.status !== "starting")
+                throw new AppError("controller_owner_mismatch", "Controller startup no longer owns this launch", 1);
+            if (["done", "failed", "cancelled"].includes(requireRun(context, row.project_id, row.run_id).status))
+                throw new AppError("invalid_run_state", "A terminal RUN cannot start a physical controller", 1);
+        };
+        const owner = await startup.birth(assertStartingOwner);
+        try {
+            claimControllerProcess(context, row, owner);
+        }
+        finally {
+            claimedProcess = row.owner_pid === process.pid;
+        }
+        if (!row.process_id || !row.process_lease_token)
+            throw new AppError("controller_owner_mismatch", "Managed owner has no resource lease", 1);
+        const confirmation = { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid, processGroupId: process.pid, ownerPid: process.pid };
+        const originalLease = prepareManagedProcessConfirmation(context, confirmation).lease_expires_at;
+        const confirmed = await startup.observe(() => confirmManagedProcess(context, confirmation), () => assertControllerLeaseAllowed(context, row), originalLease);
+        lease = controllerLeaseMonitor(attemptMs => renewManagedProcess({ ...context, resourceDatabaseMode: "prepared_existing", databaseDeadlineAt: Date.now() + attemptMs }, { id: row.process_id, leaseToken: row.process_lease_token }), confirmed.lease_expires_at);
+        // Resource and RUN commits are distinct effects. A RUN keepalive failure must
+        // neither erase a resource receipt nor grant RUN dispatch authority.
+        mergeLease = controllerLeaseMonitor(attemptMs => {
+            const expiry = new Date(Date.now() + 120_000).toISOString();
+            const db = context.db.withDeadline(Date.now() + attemptMs);
+            if (db.get("SELECT 1 FROM merge_requests WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status = 'dispatching' LIMIT 1", [row.project_id, row.run_id, row.controller_id]))
+                db.writeTransaction(() => {
+                    const guard = recoveryGuard({ ...context, db }, row.project_id, row.run_id);
+                    if ((guard?.generation ?? 0) !== row.generation || ["draining", "sealed", "resuming"].includes(guard?.status ?? ""))
+                        throw new AppError("controller_lease_lost", "RUN generation changed before MERGE keepalive", 1);
+                    db.run("UPDATE merge_requests SET dispatch_lease_expires_at = ?, updated_at = ? WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status = 'dispatching'", [expiry, context.now(), row.project_id, row.run_id, row.controller_id]);
+                });
+            return { owned: true, lease_expires_at: expiry };
+        });
+        const assertLeaseFence = () => {
+            assertControllerLeaseAllowed(context, row);
+        };
+        heartbeat = setInterval(() => {
+            lease.background(assertLeaseFence, () => { });
+            if (currentState.merge_claim)
+                mergeLease.background(assertLeaseFence, () => { });
+        }, 30_000);
+        heartbeat.unref();
+        const manifest = JSON.parse(row.manifest_json);
+        const assertPhysicalOwner = () => {
+            lease.assertNoRenewal();
+            if (currentState.merge_claim)
+                mergeLease.assertNoRenewal();
+            // This isolated read already bounds SQLite contention. A wall deadline
+            // before opening the reader mistakes scheduler delay for heartbeat loss;
+            // the observed row's actual expiry/token still fences productive dispatch.
+            assertManagedProcessLease(context, { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid });
+            const current = requireController(context, row.controller_id);
+            if (current.owner_pid !== process.pid || current.owner_token !== row.owner_token || !["running", "capturing", "waiting_for_user", "waiting_for_context", "waiting_for_children", "waiting_for_capacity", "waiting_for_merge"].includes(current.status))
+                throw new AppError("controller_owner_mismatch", "Controller no longer owns productive dispatch", 1);
+        };
+        const assertOwner = () => { assertPhysicalOwner(); assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation); };
         await lease.ensureConfirmed(assertLeaseFence);
-        if (state.merge_claim)
+        if (currentState.merge_claim)
             await mergeLease.ensureConfirmed(assertLeaseFence);
         context.db.writeTransaction(() => {
             markControllerDispatchReady(context, row);
-            if (!state.recovery && (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) === row.generation && requireRun(context, row.project_id, row.run_id).status === "paused")
+            if (!currentState.recovery && (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) === row.generation && requireRun(context, row.project_id, row.run_id).status === "paused")
                 setFlowRunRecoveryPaused(context, row.project_id, row.run_id, false);
         });
         appendEvent(context, row.controller_id, "owner_started", { pid: process.pid, generation: row.generation });
-        if (state.recovery && !["done", "failed", "cancelled"].includes(requireRun(context, row.project_id, row.run_id).status))
-            await resumeControllerSession(context, row, manifest, state, assertPhysicalOwner);
-        await executeController(context, row, manifest, state, assertOwner, assertPhysicalOwner, async () => { await lease.ensureConfirmed(assertLeaseFence); if (state.merge_claim)
+        if (currentState.recovery && !["done", "failed", "cancelled"].includes(requireRun(context, row.project_id, row.run_id).status))
+            await resumeControllerSession(context, row, manifest, currentState, assertPhysicalOwner);
+        await executeController(context, row, manifest, currentState, assertOwner, assertPhysicalOwner, async () => { await lease.ensureConfirmed(assertLeaseFence); if (currentState.merge_claim)
             await mergeLease.ensureConfirmed(assertLeaseFence); });
         return controllerReceipt(requireController(context, row.controller_id));
     }
     catch (error) {
         primaryFailure = error;
         try {
-            try {
-                failControllerRecovery(context, row, state, error);
-            }
-            catch (secondary) {
-                retainSecondary("fail_controller_recovery", secondary);
-            }
-            if ((error instanceof AppError && ["run_recovery_guarded", "run_recovery_generation_stale"].includes(error.code) || (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) !== row.generation)
-                && context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controls'")) {
-                const guard = recoveryGuard(context, row.project_id, row.run_id);
-                const control = guard && ["draining", "sealed"].includes(guard.status) && context.db.get("SELECT control_id, action FROM run_controls WHERE project_id = ? AND run_id = ? AND recovery_id = ? AND status IN ('requested','settled')", [row.project_id, row.run_id, guard.recovery_id]);
-                if (control) {
-                    // Control is a later state transition, not evidence that the causal
-                    // failure which fenced admission disappeared.
-                    context.db.run("UPDATE run_controllers SET status = CASE WHEN last_error_json IS NULL THEN 'control_requested' ELSE status END, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [context.now(), row.controller_id, row.owner_token]);
-                    appendEvent(context, row.controller_id, "control_boundary_reached", { control_id: control.control_id, action: control.action, generation: guard.generation, settled: false });
-                    return controllerReceipt(requireController(context, row.controller_id));
+            const controlled = context.db.writeTransaction(() => {
+                const currentOwner = requireController(context, row.controller_id);
+                const sameBinding = currentOwner.owner_token === row.owner_token && currentOwner.generation === row.generation
+                    && currentOwner.process_id === row.process_id && currentOwner.process_lease_token === row.process_lease_token;
+                const ownsFailure = claimedProcess ? currentOwner.owner_pid === process.pid : row.owner_pid === null && currentOwner.owner_pid === null && row.status === "starting" && currentOwner.status === "starting";
+                if (!sameBinding || !ownsFailure || ["completed", "cancelled", "superseded"].includes(currentOwner.status))
+                    throw error;
+                try {
+                    if (state)
+                        failControllerRecovery(context, row, state, error);
                 }
-            }
-            try {
-                context.db.writeTransaction(() => {
+                catch (secondary) {
+                    retainSecondary("fail_controller_recovery", secondary);
+                }
+                if ((error instanceof AppError && ["run_recovery_guarded", "run_recovery_generation_stale"].includes(error.code) || (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) !== row.generation)
+                    && context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controls'")) {
+                    const guard = recoveryGuard(context, row.project_id, row.run_id);
+                    const control = guard && ["draining", "sealed"].includes(guard.status) && context.db.get("SELECT control_id, action FROM run_controls WHERE project_id = ? AND run_id = ? AND recovery_id = ? AND status IN ('requested','settled')", [row.project_id, row.run_id, guard.recovery_id]);
+                    if (control) {
+                        // Control is a later state transition, not evidence that the causal
+                        // failure which fenced admission disappeared.
+                        context.db.run("UPDATE run_controllers SET status = CASE WHEN last_error_json IS NULL THEN 'control_requested' ELSE status END, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [context.now(), row.controller_id, row.owner_token]);
+                        appendEvent(context, row.controller_id, "control_boundary_reached", { control_id: control.control_id, action: control.action, generation: guard.generation, settled: false });
+                        return controllerReceipt(requireController(context, row.controller_id));
+                    }
+                }
+                try {
                     // A failed capture must not reopen admission when this process exits.
                     const changed = context.db.run("UPDATE run_controllers SET status = CASE WHEN status IN ('capturing','capture_failed') THEN 'capture_failed' ELSE 'recovery_required' END, last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status NOT IN ('completed','cancelled','superseded')", [JSON.stringify(errorRecord(error)), context.now(), row.controller_id, row.owner_token]);
                     context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status IN ('dispatching','active')", [JSON.stringify(errorRecord(error)), context.now(), row.project_id, row.run_id, row.controller_id]);
@@ -308,11 +338,14 @@ export async function serveRunController(context, input) {
                         setFlowRunRecoveryPaused(context, row.project_id, row.run_id, true);
                     }
                     appendEvent(context, row.controller_id, "recovery_required", errorRecord(error));
-                });
-            }
-            catch (failure) {
-                retainSecondary("persist_controller_failure", failure);
-            }
+                }
+                catch (failure) {
+                    retainSecondary("persist_controller_failure", failure);
+                }
+                return null;
+            });
+            if (controlled)
+                return controlled;
         }
         catch (secondary) {
             retainSecondary("controller_failure_boundary", secondary);
@@ -320,12 +353,15 @@ export async function serveRunController(context, input) {
         throw error;
     }
     finally {
-        lease.close();
-        mergeLease.close();
-        clearInterval(heartbeat);
+        lease?.close();
+        mergeLease?.close();
+        if (heartbeat)
+            clearInterval(heartbeat);
         finishOwner();
     }
     function finishOwner() {
+        if (!claimedProcess || row.owner_pid !== process.pid || !row.process_id || !row.process_lease_token)
+            return;
         try {
             finishManagedProcess(context, { id: row.process_id, leaseToken: row.process_lease_token, state: "stopped", reason: "controller_owner_exited" });
         }
@@ -706,7 +742,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
                     assertCurrent: () => {
                         assertOwner();
                     },
-                    decision: async () => {
+                    decision: async (boundary) => {
                         if (kind === "child") {
                             const target = Object.entries(state.child_capacity ?? {}).find(([, retained]) => retained.operation_id === firstOperationId);
                             if (!target)
@@ -718,6 +754,11 @@ async function executeController(context, row, manifest, state, assertOwnership,
                         }
                         const view = getFlowRunStatus(executionContext, { projectRoot: manifest.request.project_root, runId: row.run_id });
                         const next = view.continuation;
+                        if (boundary) {
+                            if (!ownedProviderBoundary(context, { projectId: row.project_id, runId: row.run_id, stage: state.stage, sessionId: session.id, generation: row.generation, scope: "stage" }, boundary))
+                                return "reconciliation_blocked";
+                            return boundary.outcome === "paused" ? "paused" : "completed";
+                        }
                         if (next.kind === "blocked")
                             return "reconciliation_blocked";
                         if (next.kind === "paused" && next.stage !== state.stage)
@@ -1088,7 +1129,12 @@ async function executeController(context, row, manifest, state, assertOwnership,
                     continue;
                 }
             }
-            const fanout = await nextControllerFanout(executionContext, { projectRoot: run.project_root, runId: run.id, stage: next.stage, harness: runtimeHarness(session.profile), sessionId: session.id, children, policy, profiles, dispatchFence: retainedFailures.length > 0 });
+            const memoryFlow = ["mb-init", "mb-upgrade"].includes(run.flow_kind);
+            const fanoutInput = { projectRoot: run.project_root, runId: run.id, stage: next.stage, harness: runtimeHarness(session.profile), sessionId: session.id, children, policy, profiles, dispatchFence: retainedFailures.length > 0 };
+            const fanout = memoryFlow ? await runControllerMemoryTask(executionContext, { operation: "fanout", input: fanoutInput }, cancellation.signal)
+                : await nextControllerFanout(executionContext, fanoutInput);
+            if (memoryFlow)
+                await renewLease();
             assertOwner();
             if (fanout.kind === "none") {
                 // An ACK-only recovery Turn may leave the original root Work unfinished.
@@ -1129,7 +1175,12 @@ async function executeController(context, row, manifest, state, assertOwnership,
                     try {
                         await renewLease();
                         assertOwner();
-                        const receipt = await launchExternalWork(executionContext, { workId: work.work_id, stage: next.stage, signal: cancellation.signal });
+                        const receipt = memoryFlow ? await runControllerMemoryTask(executionContext, { operation: "external", input: { projectRoot: run.project_root, runId: run.id, workId: work.work_id, stage: next.stage } }, cancellation.signal)
+                            : await launchExternalWork(executionContext, { workId: work.work_id, stage: next.stage, signal: cancellation.signal });
+                        if (memoryFlow) {
+                            await renewLease();
+                            assertOwner();
+                        }
                         if (fatal)
                             throw fatal;
                         appendEvent(context, row.controller_id, "external_work_observed", { work_id: work.work_id, receipt });

package/dist/services/run-recovery-guard.js

@@ -1,6 +1,36 @@
+import crypto from "node:crypto";
 import { AppError } from "../shared/errors.js";
 import { CAPACITY_POLICY, MAX_OVERLOAD_RETRIES, overloadExhausted } from "../harness-runtime/lib/provider-overload-policy.mjs";
 import { assertRunStartupSettled } from "./run-start-custody.js";
+export function recoveryPacketReceipt(binding) {
+    if (!binding.packet_json || !binding.packet_sha256 || crypto.createHash("sha256").update(binding.packet_json).digest("hex") !== binding.packet_sha256)
+        throw new AppError("recovery_packet_missing", "Recovery requires its original verified Work packet", 1);
+    return { packet_json: binding.packet_json, packet_sha256: binding.packet_sha256, packet: JSON.parse(binding.packet_json) };
+}
+/** A cooperative drain permits only the WorkSession present at its fence. */
+export function assertWorkSettlementAllowed(context, projectId, runId, workId) {
+    const guard = recoveryGuard(context, projectId, runId);
+    if (guard?.status === "draining" && context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controls'")) {
+        const admission = context.db.get("SELECT a.work_session_id FROM run_control_admissions a JOIN run_controls c ON c.control_id = a.control_id JOIN work_sessions ws ON ws.id = a.work_session_id JOIN works w ON w.work_id = a.work_id WHERE c.project_id = ? AND c.run_id = ? AND c.recovery_id = ? AND c.action = 'pause' AND c.status = 'requested' AND a.work_id = ? AND ws.work_id = a.work_id AND ws.session_id = a.session_id AND ws.status = 'running' AND w.status = 'running' AND w.project_id = c.project_id AND w.run_id = c.run_id LIMIT 1", [projectId, runId, guard.recovery_id, workId]);
+        if (admission)
+            return;
+    }
+    assertRunMutationAllowed(context, projectId, runId);
+}
+/** Completion rights are pinned to the Stage attempt present at the pause fence. */
+export function assertStageSettlementAllowed(context, projectId, runId, stage, expectedGeneration) {
+    const guard = recoveryGuard(context, projectId, runId);
+    const generation = guard?.generation ?? 0;
+    if (expectedGeneration !== undefined && generation !== expectedGeneration)
+        throw new AppError("run_recovery_generation_stale", "RUN execution generation changed while the operation was in flight", 1, { run_id: runId, expected_generation: expectedGeneration, generation });
+    if (guard?.status === "draining" && context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_control_stage_admissions'")) {
+        const admitted = context.db.get(`SELECT a.work_session_id FROM run_control_stage_admissions a JOIN run_controls c ON c.control_id = a.control_id JOIN run_control_admissions w ON w.control_id = a.control_id AND w.work_session_id = a.work_session_id JOIN work_sessions ws ON ws.id = w.work_session_id JOIN runs r ON r.project_id = c.project_id AND r.id = c.run_id, json_each(r.index_json, '$.stage_runs') live
+      WHERE c.project_id = ? AND c.run_id = ? AND c.recovery_id = ? AND c.action = 'pause' AND c.status = 'requested' AND a.stage = ? AND json_extract(live.value, '$.stage') = a.stage AND COALESCE(json_extract(live.value, '$.attempt'), 'try-001') = a.attempt AND ws.work_id = w.work_id AND ws.session_id = w.session_id AND NOT EXISTS (SELECT 1 FROM work_sessions newer WHERE newer.work_id = w.work_id AND newer.status = 'running' AND newer.id <> ws.id) LIMIT 1`, [projectId, runId, guard.recovery_id, stage]);
+        if (admitted)
+            return generation;
+    }
+    return assertRunMutationAllowed(context, projectId, runId, expectedGeneration);
+}
 export function recoveryGuard(context, projectId, runId) { return context.db.get("SELECT recovery_id, project_id, run_id, generation, status, interruption_json, settlement_json, capture_path, created_at, updated_at, sealed_at, resumed_at FROM run_recovery_guards WHERE project_id = ? AND run_id = ? ORDER BY generation DESC LIMIT 1", [projectId, runId]) ?? null; }
 export function assertRunRecoveryAllowed(context, projectId, runId, expectedGeneration) {
     const guard = recoveryGuard(context, projectId, runId);

package/dist/services/run-recovery.js

@@ -1,5 +1,6 @@
 import { hasLifecyclePacketAssignments, managedLifecycleCommand, recoverLifecycleCommands } from "./lifecycle-invocations.js";
-import { recoveryGuard, assertRunMutationAllowed } from "./run-recovery-guard.js";
+import { recoveryGuard, recoveryPacketReceipt } from "./run-recovery-guard.js";
+export { assertStageSettlementAllowed, assertWorkSettlementAllowed, recoveryPacketReceipt } from "./run-recovery-guard.js";
 export { recoveryGuard, assertRunMutationAllowed, assertRunRecoveryAllowed } from "./run-recovery-guard.js";
 import crypto from "node:crypto";
 import fs from "node:fs";
@@ -19,11 +20,6 @@ import { findRecentMatchingHookEvent, hookSessionIdentity, recoveryAcceptMatchKe
 import { flowCommand } from "./stage-pause.js";
 import { materializePendingWorkResults } from "./work-registry.js";
 import { currentStagePrompt } from "./stage-context.js";
-export function recoveryPacketReceipt(binding) {
-    if (!binding.packet_json || !binding.packet_sha256 || crypto.createHash("sha256").update(binding.packet_json).digest("hex") !== binding.packet_sha256)
-        throw new AppError("recovery_packet_missing", "Recovery requires its original verified Work packet", 1);
-    return { packet_json: binding.packet_json, packet_sha256: binding.packet_sha256, packet: JSON.parse(binding.packet_json) };
-}
 /** Read-only custody permission for one prepared recovery ACK, not lifecycle admission. */
 export class PreparedRecoveryCoordinatorRead {
     #scope;
@@ -90,12 +86,16 @@ export function recoveredCoordinatorBinding(original, input) {
 export function capturedStageDelivery(capture, scope, originalPrompt) {
     const db = new DatabaseSync(path.join(capture, "runtime", "db.sqlite"), { readOnly: true });
     try {
+        const resumed = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='audit_events'").get()
+            ? db.prepare("SELECT json_extract(payload_json, '$.pause_id') AS pause_id FROM audit_events WHERE event_type = 'flow_run.stage_resumed' AND json_extract(payload_json, '$.run_id') = ? AND json_extract(payload_json, '$.work_id') = ? AND json_extract(payload_json, '$.stage') = ? ORDER BY id DESC LIMIT 1").get(scope.runId, scope.workId, scope.stage) : undefined;
         const rows = db.prepare("SELECT command, outcome_json FROM lifecycle_invocations WHERE status = 'settled' AND json_extract(scope_json, '$.projectRoot') = ? AND json_extract(scope_json, '$.runId') = ? AND json_extract(scope_json, '$.rootSessionId') = ? AND json_extract(scope_json, '$.generation') < ? ORDER BY rowid DESC").all(scope.projectRoot, scope.runId, scope.sessionId, scope.generation);
         for (const row of rows) {
             const result = row.outcome_json ? JSON.parse(row.outcome_json).result : null;
             const parsed = parseLifecycleCommand(row.command);
-            if (parsed.kind === "none" || !parsed.supported || parsed.invocation.operation !== "stage_start" || parsed.invocation.args.options.get("stage")?.[0] !== scope.stage
-                || result?.stage !== scope.stage || result.attempt !== scope.attempt || result.work_id !== scope.workId || typeof result.worker_prompt_markdown !== "string")
+            if (parsed.kind === "none" || !parsed.supported || !["stage_start", "stage_resume"].includes(parsed.invocation.operation) || parsed.invocation.args.options.get("stage")?.[0] !== scope.stage
+                || result?.stage !== scope.stage || parsed.invocation.operation === "stage_start" && result.attempt !== scope.attempt || result.work_id !== scope.workId || typeof result.worker_prompt_markdown !== "string")
+                continue;
+            if (resumed && (parsed.invocation.operation !== "stage_resume" || result.pause_id !== resumed.pause_id))
                 continue;
             const text = result.worker_prompt_markdown;
             if (!text.startsWith(originalPrompt))
@@ -105,6 +105,8 @@ export function capturedStageDelivery(capture, scope, originalPrompt) {
                 throw new AppError("recovery_packet_missing", "Captured Stage delivery has no committed lifecycle assignment", 1);
             return { text, sha256 };
         }
+        if (resumed)
+            throw new AppError("recovery_packet_missing", "Accepted HITL answer has no captured settled current delivery; reconcile its retained resume receipt, never fall back to the pre-answer prompt", 1, { work_id: scope.workId, pause_id: resumed.pause_id });
         return null;
     }
     finally {
@@ -174,42 +176,7 @@ export function prepareRecoveryPayload(value, label) {
         throw new AppError("recovery_not_settled", "Recovery capture requires an explicit settled tree receipt", 1);
     return payload;
 }
-/** A cooperative drain permits only the WorkSession present at its fence. */
-export function assertWorkSettlementAllowed(context, projectId, runId, workId) {
-    const guard = recoveryGuard(context, projectId, runId);
-    if (guard?.status === "draining" && context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controls'")) {
-        const admission = context.db.get("SELECT a.work_session_id FROM run_control_admissions a JOIN run_controls c ON c.control_id = a.control_id JOIN work_sessions ws ON ws.id = a.work_session_id JOIN works w ON w.work_id = a.work_id WHERE c.project_id = ? AND c.run_id = ? AND c.recovery_id = ? AND c.action = 'pause' AND c.status = 'requested' AND a.work_id = ? AND ws.work_id = a.work_id AND ws.session_id = a.session_id AND ws.status = 'running' AND w.status = 'running' AND w.project_id = c.project_id AND w.run_id = c.run_id LIMIT 1", [projectId, runId, guard.recovery_id, workId]);
-        if (admission)
-            return;
-    }
-    assertRunMutationAllowed(context, projectId, runId);
-}
-/** Completion rights are pinned to the Stage attempt present at the pause fence. */
-export function assertStageSettlementAllowed(context, projectId, runId, stage, expectedGeneration) {
-    const guard = recoveryGuard(context, projectId, runId);
-    const generation = guard?.generation ?? 0;
-    if (expectedGeneration !== undefined && generation !== expectedGeneration)
-        throw new AppError("run_recovery_generation_stale", "RUN execution generation changed while the operation was in flight", 1, { run_id: runId, expected_generation: expectedGeneration, generation });
-    if (guard?.status === "draining" && context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_control_stage_admissions'")) {
-        const admitted = context.db.get(`SELECT a.work_session_id FROM run_control_stage_admissions a JOIN run_controls c ON c.control_id = a.control_id JOIN run_control_admissions w ON w.control_id = a.control_id AND w.work_session_id = a.work_session_id JOIN work_sessions ws ON ws.id = w.work_session_id JOIN runs r ON r.project_id = c.project_id AND r.id = c.run_id, json_each(r.index_json, '$.stage_runs') live
-      WHERE c.project_id = ? AND c.run_id = ? AND c.recovery_id = ? AND c.action = 'pause' AND c.status = 'requested' AND a.stage = ? AND json_extract(live.value, '$.stage') = a.stage AND COALESCE(json_extract(live.value, '$.attempt'), 'try-001') = a.attempt AND ws.work_id = w.work_id AND ws.session_id = w.session_id AND NOT EXISTS (SELECT 1 FROM work_sessions newer WHERE newer.work_id = w.work_id AND newer.status = 'running' AND newer.id <> ws.id) LIMIT 1`, [projectId, runId, guard.recovery_id, stage]);
-        if (admitted)
-            return generation;
-    }
-    return assertRunMutationAllowed(context, projectId, runId, expectedGeneration);
-}
-/** A recovered RUN never grants the retired process a new execution segment. */
-export function assertRecoveryHookCurrent(context, projectId, runId, identity) {
-    const generation = assertRunMutationAllowed(context, projectId, runId);
-    if (!generation)
-        return;
-    const current = identity.daemonId && context.db.get("SELECT b.recovery_id FROM run_recovery_bindings b JOIN run_recovery_guards g ON g.recovery_id = b.recovery_id WHERE g.project_id = ? AND g.run_id = ? AND g.generation = ? AND g.status = 'recovered' AND b.status = 'accepted' AND b.daemon_id = ? LIMIT 1", [projectId, runId, generation, identity.daemonId]);
-    if (current)
-        return;
-    const retired = identity.daemonId && (context.db.get(`SELECT ws.id FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id JOIN hook_events he ON he.project_id = w.project_id AND he.event_key = ws.hook_event_id WHERE w.project_id = ? AND w.run_id = ? AND he.daemon_id = ? LIMIT 1`, [projectId, runId, identity.daemonId]) || context.db.get("SELECT b.recovery_id FROM run_recovery_bindings b JOIN run_recovery_guards g ON g.recovery_id = b.recovery_id WHERE g.project_id = ? AND g.run_id = ? AND b.daemon_id = ? AND (g.generation <> ? OR b.status = 'abandoned') LIMIT 1", [projectId, runId, identity.daemonId, generation]));
-    if (!identity.daemonId || retired)
-        throw new AppError("recovery_daemon_stale", "Lifecycle receipt belongs to an unidentified or retired recovery daemon", 1, { run_id: runId, generation, daemon_id: identity.daemonId });
-}
+export { assertRecoveryHookCurrent } from "./hook-recovery-guard.js";
 export function beginRunRecovery(context, input, interruption = prepareRecoveryPayload(input.interruption, "interruption")) {
     const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
     const run = requireRecoveryRun(context, project.id, input.runId);

package/dist/services/run-start-custody.js

@@ -1,8 +1,12 @@
 import { createHash } from "node:crypto";
 import path from "node:path";
 import { AppError } from "../shared/errors.js";
-import { hybridDigest } from "./hybrid-contracts.js";
+import { hybridDigest } from "./hybrid-digest.js";
+import { runContextRelocations, projectPacketLocators } from "./stage-context.js";
 export const runStartPayloadLimit = 64 * 1024 * 1024;
+// One exact immutable payload per database, not cached custody or returned
+// objects. Every read still checks current SQL hashes, markers and receipts.
+const validatedPayloadBytes = new WeakMap();
 export function runStartBytesSha256(bytes) { return createHash("sha256").update(bytes).digest("hex"); }
 export function refuseRunStart(reason) {
     throw new AppError("run_start_custody_invalid", "RUN creation requires its exact retained request", 1, { reason, effect: "no_effect", next_action: "reconcile_exact_request" });
@@ -59,8 +63,11 @@ export function validateRunStartPayload(value) {
                 || !entry.path || entry.path.includes("\\") || entry.path.includes("\0") || entry.path.split("/").some(part => !part || part === "." || part === "..")
                 || seen.has(entry.path) || !canonicalBase64(entry.base64))
                 refuseRunStart("invalid_allocation_assets");
-            if (key === "artifacts" && (!["engine-binding.json", "operational-policy.source.md", "operational-contract.json"].includes(entry.path)
-                || Buffer.from(entry.base64, "base64").length > 1024 * 1024))
+            const upgradeEvidence = value.initial_index.flow_kind === "mb-upgrade" && (["upgrade-baseline.json", "upgrade-target.json"].includes(entry.path)
+                || entry.path === `upgrade-target-${runStartBytesSha256(Buffer.from(entry.base64, "base64"))}.bin`
+                || entry.path === `upgrade-baseline-${runStartBytesSha256(Buffer.from(entry.base64, "base64"))}.bin`);
+            if (key === "artifacts" && (!["engine-binding.json", "operational-policy.source.md", "operational-contract.json"].includes(entry.path) && !upgradeEvidence
+                || Buffer.from(entry.base64, "base64").length > (upgradeEvidence ? 16 : 1) * 1024 * 1024))
                 refuseRunStart("invalid_allocation_artifact");
             seen.add(entry.path);
         }
@@ -96,7 +103,10 @@ function readRunStartRequestSnapshot(context, projectId, requestId) {
     catch {
         return refuseRunStart("invalid_allocation_json");
     }
-    validateRunStartPayload(payload);
+    if (validatedPayloadBytes.get(context.db) !== row.payload_json) {
+        validateRunStartPayload(payload);
+        validatedPayloadBytes.set(context.db, row.payload_json);
+    }
     if (!object(receipt))
         refuseRunStart("invalid_allocation_receipt");
     const run = context.db.get("SELECT project_root, workspace_root, run_root, flow_kind, subject_type, subject_id, index_json FROM runs WHERE project_id = ? AND id = ?", [projectId, row.run_id]);
@@ -114,6 +124,8 @@ function readRunStartRequestSnapshot(context, projectId, requestId) {
     if (!object(index))
         refuseRunStart("allocated_run_changed");
     const marker = index.start_request, initial = payload.initial_index;
+    const relocations = runContextRelocations(context, projectId, row.run_id);
+    const currentPath = (value) => projectPacketLocators(value, relocations);
     if (!payload || payload.schema_id !== "dd-flow/run-start-allocation@1" || payload.project_id !== projectId
         || payload.request_id !== requestId || payload.run_id !== row.run_id || !Array.isArray(payload.blobs) || !Array.isArray(payload.artifacts)
         || !initial || initial.run_id !== row.run_id || initial.project?.id !== projectId || initial.start_request !== undefined
@@ -121,7 +133,7 @@ function readRunStartRequestSnapshot(context, projectId, requestId) {
         || !marker || marker.schema_id !== "dd-flow/run-start-request@1" || marker.request_id !== requestId
         || marker.request_sha256 !== payload.request_sha256 || marker.allocation_sha256 !== row.payload_sha256
         || receipt.run_id !== row.run_id || receipt.request_id !== requestId || receipt.request_sha256 !== payload.request_sha256 || receipt.allocation_sha256 !== row.payload_sha256
-        || run.project_root !== payload.project_root || run.workspace_root !== payload.workspace_root || run.run_root !== payload.run_root
+        || run.project_root !== currentPath(payload.project_root) || run.workspace_root !== currentPath(payload.workspace_root) || run.run_root !== currentPath(payload.run_root)
         || run.flow_kind !== initial.flow_kind || run.subject_id !== payload.subject_id || run.subject_type !== payload.subject_type
         || !object(index.execution_bundle) || hybridDigest(index.execution_bundle) !== hybridDigest(initial.execution_bundle))
         refuseRunStart("allocated_run_changed");