@deksden-com/dd-flow-cli 0.9.0-beta.123 → 0.9.0-beta.127

package/dist/services/run-controller.js

@@ -19,23 +19,25 @@ import { flowCommand } from "./stage-pause.js";
 import { resolveStageTransition, sameSessionProfile, stageExecutionCwd, runtimeHarness } from "./execution-policy.js";
 import { resolveHarnessCommand, harnessRuntimeArguments } from "./harness-config.js";
 import { adapterSessionId, prepareHarnessFlowExecutable, runHarnessAdapter } from "./harness-adapter.js";
-import { assertManagedProcessLease, confirmManagedProcess, heartbeatManagedProcess, finishManagedProcess, processStartedAt, processTreeIsAlive, resourceHome } from "./managed-processes.js";
+import { assertManagedProcessLease, confirmManagedProcess, renewManagedProcess, finishManagedProcess, processStartedAt, processTreeIsAlive, resourceHome } from "./managed-processes.js";
+import { managedLeaseMonitor } from "./managed-lease-monitor.js";
 import { readNativeDaemonState, requireManagedDaemonBinding } from "./managed-daemon-binding.js";
 import { initCodexHome } from "./hooks.js";
 import { launchExternalWork } from "./external-work-launch.js";
 import { controllerNativeChildren, controllerStageEntryPrompt, controllerStageResumePrompt, nextControllerFanout } from "./controller-fanout.js";
 import { claimQueuedMergeRequest } from "./merge-server.js";
 import { prepareVnextCodeWorkspace } from "./vnext-code.js";
-import { requireRun, requireController, controllerReceipt, appendEvent, saveState, retainControllerContinuation, digest, digestBytes, errorRecord } from "./run-controller-state.js";
+import { requireRun, requireController, controllerReceipt, appendEvent, saveState, retainControllerContinuation, retainControllerCapacityIntent, digest, digestBytes, errorRecord } from "./run-controller-state.js";
 import { assertNoCorrelatedRejection } from "./continuation-outcome.js";
-import { spawnController, recordControllerLauncher, claimControllerProcess, markControllerDispatchReady } from "./run-controller-process.js";
+import { spawnController, recordControllerLauncher, claimControllerProcess, markControllerDispatchReady, assertControllerRecoveryAllowed } from "./run-controller-process.js";
 import { controllerAdapter, controllerEnvironment, sessionArguments } from "./run-controller-adapter.js";
-import { capacityDelay, continueCodexCapacity } from "./codex-capacity.js";
+import { capacityDelay, continueCodexCapacity, eligibleCodexChildOverload } from "./codex-capacity.js";
 import { resumeControllerSession, failControllerRecovery } from "./run-controller-recovery.js";
 import { captureControllerBoundary } from "./run-controller-capture.js";
 import { requestRunControl } from "./run-control-worker.js";
 import { storageSessionId } from "./session-identity.js";
-import { terminalCodexOverload } from "../harness-runtime/lib/dd-codex.mjs";
+import { CAPACITY_POLICY, nativeItemsSettled, overloadBurst, refusalObservation, terminalCodexOverload } from "../harness-runtime/lib/codex-capacity-policy.mjs";
+import { isObservationLoss } from "../harness-runtime/lib/operation-errors.mjs";
 export { resumeRunController } from "./run-controller-recovery.js";
 export function prepareRunControllerLaunch(context, input) {
     if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(input.requestId) || !/^[a-f0-9]{64}$/.test(input.contextSha256))
@@ -134,53 +136,23 @@ export async function launchRunController(context, input, prepared = prepareRunC
  * proceeding, while an explicit negative renewal remains a hard ownership
  * loss.
  */
-export function controllerLeaseMonitor(heartbeat) {
-    let lost = false;
-    let unconfirmed = null;
-    let confirmedUntil = 0;
-    const lostError = () => new AppError("controller_lease_lost", "Managed owner lease could not be confirmed", 1);
-    const uncertainError = () => new AppError("controller_lease_unconfirmed", "Managed owner lease requires renewal outside the RUN writer", 1);
-    return {
-        tick() {
-            try {
-                if (!heartbeat())
-                    lost = true;
-                else {
-                    unconfirmed = null;
-                    confirmedUntil = Date.now() + 14 * 60_000;
-                }
-            }
-            catch (error) {
-                unconfirmed = error;
-            }
-        },
-        isConfirmed() { return !lost && unconfirmed === null && Date.now() < confirmedUntil; },
-        assertNoRenewal() {
-            if (lost)
-                throw lostError();
-            if (unconfirmed !== null || Date.now() >= confirmedUntil)
-                throw uncertainError();
-        },
-        assertConfirmed() {
-            if (lost)
-                throw lostError();
-            if (unconfirmed === null && Date.now() < confirmedUntil)
-                return;
-            try {
-                if (!heartbeat()) {
-                    lost = true;
-                    throw lostError();
-                }
-                unconfirmed = null;
-                confirmedUntil = Date.now() + 14 * 60_000;
-            }
-            catch (error) {
-                if (error instanceof AppError)
-                    throw error;
-                throw new AppError("controller_lease_unconfirmed", "Managed owner lease could not be refreshed", 1, { cause: error instanceof Error ? error.message : String(error) });
-            }
-        }
-    };
+export const controllerLeaseMonitor = managedLeaseMonitor;
+/** Unconfirmed ownership is not evidence authorizing a native cancellation. */
+export function retainControllerLeaseUncertainty(context, row, error) {
+    if (!(error instanceof AppError) || error.code !== "controller_lease_unconfirmed")
+        return false;
+    try {
+        const db = context.db.withDeadline(Date.now() + 5_000);
+        db.writeTransaction(() => {
+            const changed = db.run("UPDATE run_controllers SET status = CASE WHEN status IN ('capturing','capture_failed') THEN 'capture_failed' ELSE 'recovery_required' END, last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND generation = ? AND status NOT IN ('completed','cancelled','superseded','recovery_required','capture_failed')", [JSON.stringify(errorRecord(error)), context.now(), row.controller_id, row.owner_token, row.generation]);
+            if (changed.changes)
+                appendEvent({ ...context, db }, row.controller_id, "owner_confirmation_required", { ...errorRecord(error), native_cancelled: false });
+        });
+    }
+    catch (persistence) {
+        process.stderr.write(`${JSON.stringify({ controller_id: row.controller_id, primary: errorRecord(error), persistence: errorRecord(persistence), native_cancelled: false })}\n`);
+    }
+    return true;
 }
 /** No initialization, provider calls or lease takeover on the observation path. */
 export function runControllerStatus(context, input) {
@@ -205,25 +177,37 @@ export async function serveRunController(context, input) {
     if (!row.process_id || !row.process_lease_token)
         throw new AppError("controller_owner_mismatch", "Managed owner has no resource lease", 1);
     confirmManagedProcess(context, { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid, processGroupId: process.pid, ownerPid: process.pid });
-    const lease = controllerLeaseMonitor(() => heartbeatManagedProcess(context, { id: row.process_id, leaseToken: row.process_lease_token }));
-    lease.assertConfirmed();
-    const heartbeat = setInterval(() => {
-        try {
-            lease.tick();
-            if (lease.isConfirmed()) {
-                const guard = recoveryGuard(context, row.project_id, row.run_id);
-                if ((guard?.generation ?? 0) === row.generation && !["draining", "sealed", "resuming"].includes(guard?.status ?? ""))
-                    context.db.run("UPDATE merge_requests SET dispatch_lease_expires_at = ?, updated_at = ? WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status = 'dispatching'", [new Date(Date.now() + 120_000).toISOString(), context.now(), row.project_id, row.run_id, row.controller_id]);
-            }
+    const lease = controllerLeaseMonitor(attemptMs => {
+        const deadline = Date.now() + attemptMs;
+        const receipt = renewManagedProcess({ ...context, resourceDatabaseMode: "prepared_existing", databaseDeadlineAt: deadline }, { id: row.process_id, leaseToken: row.process_lease_token });
+        if (receipt.owned) {
+            // These exact-owner keepalive writes are idempotent. Finish the resource
+            // transaction before acquiring RUN writer; both share one attempt budget.
+            const db = context.db.withDeadline(deadline);
+            if (db.get("SELECT 1 FROM merge_requests WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status = 'dispatching' LIMIT 1", [row.project_id, row.run_id, row.controller_id]))
+                db.writeTransaction(() => {
+                    const guard = recoveryGuard({ ...context, db }, row.project_id, row.run_id);
+                    if ((guard?.generation ?? 0) === row.generation && !["draining", "sealed", "resuming"].includes(guard?.status ?? ""))
+                        db.run("UPDATE merge_requests SET dispatch_lease_expires_at = ?, updated_at = ? WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status = 'dispatching'", [new Date(Date.now() + 120_000).toISOString(), context.now(), row.project_id, row.run_id, row.controller_id]);
+                });
         }
-        catch { /* The lease monitor distinguishes renewal uncertainty from loss. */ }
+        return receipt;
+    });
+    const assertLeaseFence = () => {
+        const current = requireController(context, row.controller_id);
+        if (current.owner_token !== row.owner_token || current.generation !== row.generation)
+            throw new AppError("controller_owner_mismatch", "Lease renewal no longer owns this controller generation", 1);
+        assertControllerRecoveryAllowed(context, current);
+    };
+    const heartbeat = setInterval(() => {
+        lease.background(assertLeaseFence, () => { });
     }, 30_000);
     heartbeat.unref();
     const manifest = JSON.parse(row.manifest_json);
     const state = JSON.parse(row.state_json);
     const assertPhysicalOwner = () => {
         lease.assertNoRenewal();
-        assertManagedProcessLease(context, { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid });
+        assertManagedProcessLease({ ...context, databaseDeadlineAt: Date.now() + 5_000 }, { id: row.process_id, leaseToken: row.process_lease_token, pid: process.pid });
         const current = requireController(context, row.controller_id);
         if (current.owner_pid !== process.pid || current.owner_token !== row.owner_token || !["running", "capturing", "waiting_for_user", "waiting_for_context", "waiting_for_children", "waiting_for_capacity", "waiting_for_merge"].includes(current.status))
             throw new AppError("controller_owner_mismatch", "Controller no longer owns productive dispatch", 1);
@@ -234,15 +218,16 @@ export async function serveRunController(context, input) {
         process.stderr.write(`${JSON.stringify({ controller_id: row.controller_id, phase, primary: errorRecord(primaryFailure), secondary: errorRecord(error) })}\n`);
     };
     try {
+        await lease.ensureConfirmed(assertLeaseFence);
         context.db.writeTransaction(() => {
             markControllerDispatchReady(context, row);
             if (!state.recovery && (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) === row.generation && requireRun(context, row.project_id, row.run_id).status === "paused")
                 setFlowRunRecoveryPaused(context, row.project_id, row.run_id, false);
         });
         appendEvent(context, row.controller_id, "owner_started", { pid: process.pid, generation: row.generation });
-        if (state.recovery)
+        if (state.recovery && !["done", "failed", "cancelled"].includes(requireRun(context, row.project_id, row.run_id).status))
             await resumeControllerSession(context, row, manifest, state, assertPhysicalOwner);
-        await executeController(context, row, manifest, state, assertOwner, assertPhysicalOwner, () => lease.assertConfirmed());
+        await executeController(context, row, manifest, state, assertOwner, assertPhysicalOwner, () => lease.ensureConfirmed(assertLeaseFence));
         return controllerReceipt(requireController(context, row.controller_id));
     }
     catch (error) {
@@ -287,6 +272,7 @@ export async function serveRunController(context, input) {
         throw error;
     }
     finally {
+        lease.close();
         clearInterval(heartbeat);
         finishOwner();
     }
@@ -446,6 +432,19 @@ async function executeController(context, row, manifest, state, assertOwnership,
     const checkOutcomes = () => assertLifecycleOutcomes(observer, { projectRoot: manifest.request.project_root, runId: row.run_id, generation: row.generation });
     const assertOwner = () => { if (fatal)
         throw fatal; checkOutcomes(); assertOwnership(); };
+    const prepareMutation = async (action) => {
+        for (;;) {
+            await renewLease();
+            try {
+                context.db.writeTransaction(() => { assertOwner(); action(); });
+                return;
+            }
+            catch (error) {
+                if (!(error instanceof AppError) || error.code !== "controller_lease_unconfirmed")
+                    throw error;
+            }
+        }
+    };
     const policy = manifest.execution_profile.settings.execution;
     const profiles = manifest.execution_profile.agent_profiles;
     let executionContext = { ...context, env: { ...context.env, DD_FLOW_BIN: manifest.flow_executable, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_RUNTIME_BUDGET: manifest.budget ? JSON.stringify(manifest.budget) : undefined, DD_FLOW_CODE_READINESS_OWNER: row.controller_id } };
@@ -454,10 +453,18 @@ async function executeController(context, row, manifest, state, assertOwnership,
             projectRoot: manifest.request.project_root, runId: row.run_id, generation: row.generation });
     };
     let lastCapture = null;
+    let terminalFinalization = false;
     let fatal;
     let stopping;
     const cancellation = new AbortController();
+    const capacityUncertain = (error) => error instanceof AppError && ["capacity_continuation_blocked", "operation_observation_lost"].includes(error.code) && error.details.capacity_uncertainty === true || Boolean(state.pending_capacity) && (String(error?.code) === "capacity_continuation_blocked" || isObservationLoss(error));
     const stopAfterFailure = (error) => {
+        // A retained successor may already be running. Fencing/recovery records
+        // uncertainty; cancellation is not evidence of its missing outcome.
+        if (capacityUncertain(error))
+            return Promise.resolve();
+        if (retainControllerLeaseUncertainty(context, row, error))
+            return Promise.resolve();
         if (stopping)
             return stopping;
         fatal = error;
@@ -487,7 +494,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
         if (changed.changes)
             appendEvent(context, row.controller_id, "status_changed", { status, stage: state.stage });
     };
-    const call = (session, args, productive = true) => controllerAdapter(executionContext, row, manifest, session, args, productive ? assertOwner : undefined, undefined, productive ? cancellation.signal : undefined);
+    const call = (session, args, productive = true) => controllerAdapter(executionContext, row, manifest, session, args, productive ? assertOwner : undefined, undefined, productive ? cancellation.signal : undefined, productive ? renewLease : undefined);
     const confirmStopped = async (stateDir, receipt) => {
         const fail = () => new AppError("controller_native_shutdown_unproven", "The managed daemon is not proven clean and closed", 1, { state_dir: stateDir, receipt });
         if (receipt.stopped !== true || receipt.clean !== true)
@@ -531,25 +538,40 @@ async function executeController(context, row, manifest, state, assertOwnership,
     };
     const prompt = async (session, text, operationId, kind) => {
         const hitl = kind === "hitl";
-        await reconcilePromptPredecessor(session);
-        const firstOperationId = operationId ?? `${row.controller_id}:prompt:${crypto.randomUUID()}`;
+        const pending = state.pending_capacity;
+        if (pending && (pending.session_id !== session.id || pending.generation !== row.generation || pending.kind !== (kind ?? "stage") || operationId && operationId !== pending.root_operation_id))
+            throw new AppError("capacity_continuation_conflict", "Retained capacity owner reference differs from its Session/task", 1);
+        if (!pending)
+            await reconcilePromptPredecessor(session);
+        const firstOperationId = pending?.root_operation_id ?? operationId ?? `${row.controller_id}:prompt:${crypto.randomUUID()}`;
         const send = (value, id) => {
             const retained = context.db.get("SELECT input_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [id, row.controller_id, row.generation]);
             if (retained) {
                 const dispatch = JSON.parse(retained.input_json);
                 if (![value, `${value}\n`].some(text => dispatch.prompt_sha256 === crypto.createHash("sha256").update(text).digest("hex")))
                     throw new AppError("capacity_continuation_conflict", "Retained prompt differs from continuation", 1);
-                return controllerAdapter(executionContext, row, manifest, session, dispatch.args, assertOwner, id, cancellation.signal);
+                return controllerAdapter(executionContext, row, manifest, session, dispatch.args, assertOwner, id, cancellation.signal, renewLease);
             }
             const file = path.join(row.state_dir, `turn-${String(state.turns + 1).padStart(5, "0")}.md`);
             fs.writeFileSync(file, value, { flag: "wx", mode: 0o600 });
             state.turns += 1;
             saveState(context, row, state);
-            return controllerAdapter(executionContext, row, manifest, session, ["session", "prompt", "--session-id", session.id, "--prompt-file", file, ...sessionArguments(context, manifest, session)], assertOwner, id, cancellation.signal);
+            return controllerAdapter(executionContext, row, manifest, session, ["session", "prompt", "--session-id", session.id, "--prompt-file", file, ...sessionArguments(context, manifest, session)], assertOwner, id, cancellation.signal, renewLease);
         };
         let receipt;
         try {
             try {
+                if (pending) {
+                    const predecessor = context.db.get("SELECT error_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'failed'", [pending.predecessor, row.controller_id, row.generation]);
+                    if (!predecessor?.error_json)
+                        throw new AppError("capacity_continuation_conflict", "Retained capacity predecessor has no failure receipt", 1);
+                    // The root identifies the whole chain; inspect its frozen failure,
+                    // then reconcile all retained successors before any dispatch.
+                    const root = context.db.get("SELECT error_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [firstOperationId, row.controller_id, row.generation]);
+                    if (!root?.error_json)
+                        throw new AppError("capacity_continuation_conflict", "Retained capacity root has no failure receipt", 1);
+                    throw JSON.parse(root.error_json);
+                }
                 receipt = await send(text, firstOperationId);
             }
             catch (error) {
@@ -560,9 +582,16 @@ async function executeController(context, row, manifest, state, assertOwnership,
                     throw error;
                 const inspect = (args, id) => runHarnessAdapter({ executable: manifest.adapters["codex-desktop"].executable,
                     args, env: controllerEnvironment(context, manifest, session), runtimeOwner: session.owner, operationId: id, timeoutMs: 30_000, signal: cancellation.signal });
-                const resumed = await continueCodexCapacity({ error, sessionId: session.id, operationId: firstOperationId, signal: cancellation.signal, allowSettledDescendants: true,
-                    retainedIntents: () => context.db.all("SELECT receipt_json FROM run_controller_operations WHERE controller_id = ? AND generation = ? AND operation_id IN (?, ?)", [row.controller_id, row.generation, firstOperationId, `${firstOperationId}:capacity:1:session.prompt`])
-                        .flatMap(item => { const retained = JSON.parse(item.receipt_json || "null"); return retained?.capacity_intent ? [retained.capacity_intent] : []; }).sort((a, b) => a.ordinal - b.ordinal),
+                const resumed = await continueCodexCapacity({ error, sessionId: session.id, operationId: firstOperationId, signal: cancellation.signal, allowSettledDescendants: true, atomicIntent: true,
+                    retainedIntents: () => context.db.all("SELECT receipt_json FROM run_controller_operations WHERE controller_id = ? AND generation = ?", [row.controller_id, row.generation])
+                        .flatMap(item => { const retained = JSON.parse(item.receipt_json || "null"); return retained?.capacity_intent?.root_operation_id === firstOperationId ? [retained.capacity_intent] : []; }).sort((a, b) => a.ordinal - b.ordinal),
+                    retainedFailures: () => context.db.all("SELECT receipt_json FROM run_controller_operations WHERE controller_id = ? AND generation = ?", [row.controller_id, row.generation])
+                        .flatMap(item => { const retained = JSON.parse(item.receipt_json || "null"); return retained?.capacity_root === firstOperationId && retained.capacity_failure ? [retained.capacity_failure] : []; }),
+                    retainFailure: (id, refusal) => {
+                        const changed = context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'outcome_unknown'", [JSON.stringify({ capacity_root: firstOperationId, capacity_failure: refusal }), context.now(), id, row.controller_id, row.generation]);
+                        if (changed.changes !== 1)
+                            throw new AppError("capacity_continuation_blocked", "Failed Turn owner changed before its refusal was retained", 1);
+                    },
                     inspectOperation: id => inspect(["daemon", "operation", "--state-dir", session.state_dir, "--operation-id", id, "--json"], `${id}:capacity-observe`),
                     inspectSession: turnId => inspect(["session", "inspect", "--session-id", session.id, ...(turnId ? ["--turn-id", turnId] : []), ...sessionArguments(context, manifest, session)], `${firstOperationId}:capacity-inspect`),
                     assertCurrent: () => {
@@ -598,14 +627,9 @@ async function executeController(context, row, manifest, state, assertOwnership,
                         }
                         else if (next.kind === "paused")
                             return "paused";
-                        return next.kind === "wait" ? "reconciliation_blocked" : "continue";
+                        return next.kind === "wait" && !["native_children_unsettled", "work_receipts_pending", "qualified_capacity_required"].includes(next.reason) ? "reconciliation_blocked" : "continue";
                     },
-                    retainIntent: intent => context.db.writeTransaction(() => {
-                        const changed = context.db.run("UPDATE run_controller_operations SET status = 'failed', receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'outcome_unknown'", [JSON.stringify({ capacity_intent: intent }), context.now(), intent.predecessor, row.controller_id, row.generation]);
-                        if (changed.changes !== 1)
-                            throw new AppError("capacity_continuation_blocked", "Failed native operation changed before continuation", 1);
-                        appendEvent(context, row.controller_id, "capacity_continuation", intent);
-                    }),
+                    retainIntent: intent => retainControllerCapacityIntent(context, row, state, intent, session.id, kind ?? "stage", assertOwner),
                     prompt: (id, value) => send(value, id),
                     ...(resumeCommand ? { continuationText: `Continue the same accepted HITL task in this Session. First inspect current durable RUN and pause state. Only if the exact pause still awaits the accepted answer, run this previously issued command once:\n\n${resumeCommand}\n\nNever replay an already committed answer. Otherwise use only the current issued successor.` } : {}) });
                 if (!resumed)
@@ -620,10 +644,12 @@ async function executeController(context, row, manifest, state, assertOwnership,
             if (!fatal && error instanceof AppError && ["run_recovery_guarded", "run_recovery_generation_stale", "controller_owner_mismatch"].includes(error.code))
                 throw error;
             await stopAfterFailure(error);
-            cancellation.abort();
+            if (!capacityUncertain(error) && !(error instanceof AppError && error.code === "controller_lease_unconfirmed"))
+                cancellation.abort();
             throw fatal ?? error;
         }
         state.last_receipt = receipt;
+        delete state.pending_capacity;
         saveState(context, row, state);
         return receipt;
     };
@@ -642,14 +668,37 @@ async function executeController(context, row, manifest, state, assertOwnership,
         for (;;) {
             // Renewal touches the resource registry; never do it from a callback that
             // may already own the RUN writer.
-            renewLease();
+            await renewLease();
             assertOwner();
             if (state.current_session !== null && !state.sessions[state.current_session].stopped)
                 bindInvocationScope(state.sessions[state.current_session]);
             const run = requireRun(context, row.project_id, row.run_id);
+            if (state.pending_capacity && !["done", "failed", "cancelled"].includes(run.status)) {
+                const pending = state.pending_capacity;
+                const session = state.sessions.find(item => item.id === pending.session_id && !item.stopped);
+                const operation = context.db.get("SELECT input_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [pending.root_operation_id, row.controller_id, row.generation]);
+                const dispatch = operation && JSON.parse(operation.input_json);
+                const args = dispatch?.args;
+                const file = args?.includes("--prompt-file") ? args[args.indexOf("--prompt-file") + 1] : undefined;
+                if (!session || pending.generation !== row.generation || !file || digestBytes(fs.readFileSync(file)) !== dispatch?.prompt_sha256)
+                    throw new AppError("capacity_continuation_conflict", "Retained capacity resume reference has no exact owned prompt", 1);
+                await prompt(session, fs.readFileSync(file, "utf8"), pending.root_operation_id, pending.kind === "stage" ? undefined : pending.kind);
+                continue;
+            }
             const view = getFlowRunStatus(executionContext, { projectRoot: run.project_root, runId: run.id });
             const next = view.continuation;
+            terminalFinalization = next.kind === "terminal";
             const completed = state.stage && view.index.stage_runs.find(stage => stage.stage === state.stage && ["done", "skipped"].includes(stage.status));
+            if (completed && manifest.request.stop_after === state.stage) {
+                // A bounded run is final evidence, not an entry fixture for its successor.
+                // Stop owned observers before capture; keep the controller lease/fence.
+                await settle();
+                if (manifest.request.capture_root)
+                    await captureControllerBoundary(executionContext, row, manifest, completed, null, assertPhysicalOwner);
+                setStatus("stop_target_reached");
+                appendEvent(context, row.controller_id, "stop_target_reached", { stage: state.stage, attempt: completed.attempt ?? null });
+                return;
+            }
             if (completed && manifest.request.capture_root && ["start_stage", "terminal"].includes(next.kind)) {
                 const key = JSON.stringify([completed.stage, completed.attempt, completed.started_at, completed.completed_at]);
                 if (key !== lastCapture) {
@@ -657,12 +706,6 @@ async function executeController(context, row, manifest, state, assertOwnership,
                     lastCapture = key;
                 }
             }
-            if (completed && manifest.request.stop_after === state.stage) {
-                await settle();
-                setStatus("stop_target_reached");
-                appendEvent(context, row.controller_id, "stop_target_reached", { stage: state.stage, attempt: completed.attempt ?? null });
-                return;
-            }
             if (next.kind === "terminal") {
                 await settle();
                 setStatus(run.status === "cancelled" ? "cancelled" : "completed");
@@ -684,6 +727,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
                 const input = JSON.parse(answer.input_json);
                 if (digestBytes(fs.readFileSync(input.file)) !== input.sha256)
                     throw new AppError("context_checksum_mismatch", "Accepted HITL answer bytes changed", 1);
+                await renewLease();
                 assertOwner();
                 // The accepted answer is logical intent; only its generation-bound
                 // adapter operation represents a physical dispatch that stop must drain.
@@ -738,6 +782,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
                 if (!session || transition.session_mode === "new_session" || !sameSessionProfile(session.profile, transition.coordinator) || path.resolve(session.cwd) !== stageCwd) {
                     if (session)
                         await stopSession(session);
+                    await renewLease();
                     assertOwner();
                     const directory = path.join(row.state_dir, `session-${state.sessions.length + 1}`);
                     fs.mkdirSync(directory, { mode: 0o700 });
@@ -768,12 +813,14 @@ async function executeController(context, row, manifest, state, assertOwnership,
                 }
                 state.native_baseline = controllerNativeChildren(state.last_receipt ?? {}, session.id).map(child => child.session_id);
                 state.stage = next.stage;
+                state.child_capacity = {};
                 saveState(context, row, state);
                 setStatus("running");
                 if (next.stage === "code" || next.stage === "code-review") {
                     // A CODE_REVIEW fork has source files but intentionally no installed
                     // dependencies. Its repair Work executes checks just like CODE Work.
                     const readiness = await prepareVnextCodeWorkspace(executionContext, { projectRoot: run.project_root, runId: run.id, ownerId: row.controller_id, stage: next.stage });
+                    await renewLease();
                     assertOwner();
                     appendEvent(context, row.controller_id, "code_workspace_prepared", { stage: next.stage, readiness_status: readiness.status ?? null });
                 }
@@ -794,45 +841,88 @@ async function executeController(context, row, manifest, state, assertOwnership,
                 saveState(context, row, state);
                 children = controllerNativeChildren(inspected, session.id).filter(child => !baseline.has(child.session_id));
                 const failed = children.filter(child => child.status === "failed");
-                const child = failed.length === 1 && children.every(item => item === failed[0] || item.status === "completed") ? failed[0] : undefined;
+                // Freeze each distinct child refusal on its first exact observation,
+                // even while a healthy sibling still owns an active native Turn.
+                for (const failedChild of failed.filter(item => eligibleCodexChildOverload(item, session.id))) {
+                    const linked = context.db.all("SELECT w.status FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.session_id = ?", [row.project_id, row.run_id, storageSessionId({ harness_id: "codex-desktop", session_id: failedChild.session_id })]);
+                    if (linked.length > 1 || linked.some(link => link.status !== "running"))
+                        continue;
+                    const retained = state.child_capacity?.[failedChild.session_id];
+                    const refusals = [...(retained?.refusals ?? [])];
+                    if (refusals.some(item => item.turn_id === failedChild.terminal_turn_id))
+                        continue;
+                    const nativeFailure = { code: "turn_interrupted", details: { provider_session_id: failedChild.session_id, turn_id: failedChild.terminal_turn_id, native_turn_id: failedChild.terminal_turn_id, terminal_status: "failed", provider_error: failedChild.provider_error, observed_at: failedChild.terminal_observed_at } };
+                    refusals.push(refusalObservation(nativeFailure, retained?.attempts ?? 0, failedChild.terminal_observed_at ?? context.now()));
+                    state.child_capacity ??= {};
+                    state.child_capacity[failedChild.session_id] = { ...retained, turn_id: retained?.turn_id ?? failedChild.terminal_turn_id, attempts: retained?.attempts ?? 0, policy: CAPACITY_POLICY, refusals };
+                    await prepareMutation(() => { saveState(context, row, state); appendEvent(context, row.controller_id, "capacity_child_refusal", { child_session_id: failedChild.session_id, refusal: refusals.at(-1) }); });
+                    const burst = overloadBurst(refusals);
+                    if (burst)
+                        throw new AppError("provider_overload_burst", "Native child had two continuation refusals within two minutes", 1, { ...burst, child_session_id: failedChild.session_id, primary_error: nativeFailure });
+                }
+                // Continue one existing child at a time; another failed child does not
+                // create a second fanout wave or share this child's refusal counter.
+                const child = children.every(item => item.status === "failed" || item.status === "completed") ? failed.find(item => item.parent_session_id === session.id && item.terminal_turn_id && nativeItemsSettled(item.native_turn_items) && terminalCodexOverload({ code: "turn_interrupted", details: { provider_session_id: item.session_id, turn_id: item.terminal_turn_id, native_turn_id: item.terminal_turn_id, terminal_status: "failed", provider_error: item.provider_error } }, item.session_id)) : undefined;
                 const turnId = child?.terminal_turn_id;
                 const native = child && turnId ? { code: "turn_interrupted", details: { provider_session_id: child.session_id, turn_id: turnId, native_turn_id: turnId,
-                        terminal_status: "failed", provider_error: child.provider_error } } : null;
+                        terminal_status: "failed", provider_error: child.provider_error, observed_at: child.terminal_observed_at } } : null;
                 const prior = child && state.child_capacity?.[child.session_id];
                 const links = child ? context.db.all("SELECT w.status FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.session_id = ?", [row.project_id, row.run_id, storageSessionId({ harness_id: "codex-desktop", session_id: child.session_id })]) : [];
                 if (child && turnId && native && terminalCodexOverload(native, child.session_id, turnId)
-                    && child.native_turn_items?.observed === true && (child.native_turn_items.possible_effects === false || child.native_turn_items.pending === false)
+                    && nativeItemsSettled(child.native_turn_items)
                     && inspected.settled === true && child.parent_session_id === session.id && links.length <= 1
-                    && (!links.length || links[0].status === "running") && ((prior?.attempts ?? 0) < 2 || prior?.turn_id === turnId && prior.dispatched === false)) {
+                    && (!links.length || links[0].status === "running")) {
                     const prepared = prior?.turn_id === turnId && prior.dispatched === false && prior.operation_id && prior.not_before ? prior : null;
-                    if (prior?.turn_id === turnId && !prepared)
+                    if (prior?.turn_id === turnId && prior.dispatched === true && !prepared)
                         throw new AppError("capacity_child_followup_unconfirmed", "Parent continuation did not start a new native child Turn", 1, { child_session_id: child.session_id, turn_id: turnId });
                     const attempt = prepared ? prepared.attempts : (prior?.attempts ?? 0) + 1;
+                    if (!Number.isSafeInteger(attempt) || prior && prior.policy !== CAPACITY_POLICY)
+                        throw new AppError("capacity_continuation_conflict", "Native child continuation policy or ordinal is incompatible", 1);
+                    const refusals = [...(prior?.refusals ?? [])];
+                    if (!refusals.some(item => item.turn_id === turnId)) {
+                        const refusal = refusalObservation(native, attempt - 1, child.terminal_observed_at ?? context.now());
+                        refusals.push(refusal);
+                        state.child_capacity ??= {};
+                        state.child_capacity[child.session_id] = { ...prior, turn_id: prior?.turn_id ?? turnId, attempts: prior?.attempts ?? 0, policy: CAPACITY_POLICY, refusals };
+                        saveState(context, row, state);
+                    }
+                    const burst = overloadBurst(refusals);
+                    if (burst)
+                        throw new AppError("provider_overload_burst", "Native child had two continuation refusals within two minutes", 1, { ...burst, child_session_id: child.session_id, primary_error: native });
                     const wait = prepared ? Math.max(0, Date.parse(prepared.not_before) - Date.now()) : capacityDelay(native, attempt);
-                    if (!Number.isFinite(wait) || wait > 30_000)
-                        throw new AppError("capacity_child_retry_delay_exceeded", "Native child retry delay exceeds the bounded continuation window", 1, { child_session_id: child.session_id, delay_ms: wait });
-                    const operationId = prepared?.operation_id ?? `${row.controller_id}:child:${digest([child.session_id, turnId]).slice(0, 16)}:followup:${attempt}`;
+                    if (!Number.isFinite(wait) || Date.now() + wait > 8.64e15)
+                        throw new AppError("capacity_retry_metadata_invalid", "Native child retry time is invalid", 1, { child_session_id: child.session_id, delay_ms: wait });
+                    const rootOperationId = prior?.root_operation_id ?? `${row.controller_id}:child:${digest([child.session_id, turnId]).slice(0, 16)}`;
+                    const operationId = prepared?.operation_id ?? `${rootOperationId}:followup:${attempt}`;
                     state.child_capacity ??= {};
-                    state.child_capacity[child.session_id] = { turn_id: turnId, attempts: attempt, operation_id: operationId, not_before: prepared?.not_before ?? new Date(Date.now() + wait).toISOString(), dispatched: false };
-                    saveState(context, row, state);
-                    appendEvent(context, row.controller_id, "capacity_child_followup", { child_session_id: child.session_id, turn_id: turnId, attempt, operation_id: operationId });
-                    await delay(wait, undefined, { signal: cancellation.signal });
+                    state.child_capacity[child.session_id] = { turn_id: turnId, attempts: attempt, operation_id: operationId, root_operation_id: rootOperationId, policy: CAPACITY_POLICY, refusals, not_before: prepared?.not_before ?? new Date(Date.now() + wait).toISOString(), dispatched: false };
+                    await prepareMutation(() => { saveState(context, row, state); appendEvent(context, row.controller_id, "capacity_child_followup", { child_session_id: child.session_id, turn_id: turnId, attempt, operation_id: operationId }); });
+                    const until = performance.now() + wait;
+                    while (performance.now() < until) {
+                        await delay(Math.min(250, Math.max(0, until - performance.now())), undefined, { signal: cancellation.signal });
+                        await renewLease();
+                        assertOwner();
+                    }
+                    await renewLease();
                     assertOwner();
                     const current = getFlowRunStatus(executionContext, { projectRoot: manifest.request.project_root, runId: row.run_id });
-                    if (current.continuation.stage !== state.stage || ["terminal", "paused", "blocked", "wait"].includes(current.continuation.kind))
+                    if (current.continuation.stage !== state.stage || ["terminal", "paused"].includes(current.continuation.kind))
+                        continue;
+                    if (["blocked", "wait"].includes(current.continuation.kind))
                         throw new AppError("capacity_child_followup_blocked", "RUN changed during native child capacity wait", 1, { child_session_id: child.session_id });
                     const fresh = await call(session, ["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], false);
                     const still = controllerNativeChildren(fresh, session.id).find(item => item.session_id === child.session_id);
                     const currentLinks = context.db.all("SELECT w.status FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.session_id = ?", [row.project_id, row.run_id, storageSessionId({ harness_id: "codex-desktop", session_id: child.session_id })]);
-                    if (still?.status === "completed" && currentLinks.length <= 1 && currentLinks.every(link => link.status === "completed")) {
+                    if (currentLinks.length === 1 && ["completed", "cancelled", "paused"].includes(currentLinks[0].status)) {
                         state.last_receipt = fresh;
                         saveState(context, row, state);
                         continue;
                     }
                     if (fresh.settled !== true || !still || still.parent_session_id !== session.id || still.status !== "failed" || still.terminal_turn_id !== turnId
-                        || still.native_turn_items?.observed !== true || still.native_turn_items.possible_effects !== false && still.native_turn_items.pending !== false || currentLinks.length > 1
+                        || !nativeItemsSettled(still.native_turn_items) || currentLinks.length > 1
                         || currentLinks.some(link => link.status !== "running"))
                         throw new AppError("capacity_child_followup_blocked", "Native child or Work changed during capacity wait", 1, { child_session_id: child.session_id, turn_id: turnId });
+                    await renewLease();
                     assertOwner();
                     const text = `The existing native child ${child.session_id} has a confirmed terminal overloaded Turn. Use the native followup_task tool exactly once for that same child ID with this short message: "Continue your assigned Work from its current durable state. Do not repeat an observed lifecycle command. Finish only the original Work." Do not spawn another child, claim another Work, or perform the child's Work in this parent Turn. End this Turn after the followup_task response.`;
                     await prompt(session, text, operationId, "child");
@@ -859,7 +949,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
             if (fanout.kind === "wait") {
                 setStatus(fanout.reason === "qualified_capacity_required" ? "waiting_for_capacity" : "waiting_for_children");
                 await delay(1000);
-                state.last_receipt = await call(session, ["session", "inspect", "--session-id", session.id, "--state-dir", session.state_dir, "--json"], false);
+                state.last_receipt = await call(session, ["session", "inspect", "--session-id", session.id, ...sessionArguments(context, manifest, session)], false);
                 saveState(context, row, state);
                 continue;
             }
@@ -867,6 +957,7 @@ async function executeController(context, row, manifest, state, assertOwnership,
             if (fanout.kind === "external") {
                 const results = await Promise.allSettled(fanout.works.map(async (work) => {
                     try {
+                        await renewLease();
                         assertOwner();
                         const receipt = await launchExternalWork(executionContext, { workId: work.work_id, stage: next.stage, signal: cancellation.signal });
                         if (fatal)
@@ -904,8 +995,17 @@ async function executeController(context, row, manifest, state, assertOwnership,
     catch (error) {
         if (!fatal && error instanceof AppError && ["run_recovery_guarded", "run_recovery_generation_stale", "controller_owner_mismatch"].includes(error.code))
             throw error;
-        await stopAfterFailure(fatal ?? error);
-        throw fatal ?? error;
+        let cause = fatal ?? error;
+        if (!fatal && terminalFinalization) {
+            const record = errorRecord(error);
+            const provenance = { phase: "terminal_finalization", project_root: manifest.request.project_root, run_id: row.run_id, controller_id: row.controller_id };
+            if (error instanceof AppError)
+                Object.assign(error.details, provenance);
+            else
+                cause = new AppError(String(record.code), String(record.message), 1, { ...record.details, ...provenance, cause: record });
+        }
+        await stopAfterFailure(cause);
+        throw fatal ?? cause;
     }
     finally {
         clearInterval(monitor);

package/dist/services/run-fork.js

@@ -6,7 +6,7 @@ import { AppError } from "../shared/errors.js";
 import { canonicalPath } from "../storage/paths.js";
 import { importGuardPath, rearmCompletedSnapshotImport } from "../storage/import-guard.js";
 import { installCurrentEngine, upgradeForkedRunEngine, prepareForkEngine } from "./engines.js";
-import { restoreEvalRunSnapshot, prepareEvalRunSnapshot, prepareEvalRunRestore, restoredWorkspaceRoot } from "./eval-snapshots.js";
+import { restoreEvalRunSnapshot, prepareEvalRunSnapshot, prepareEvalRunRestore, prepareSnapshotOperationalDecision, restoredWorkspaceRoot } from "./eval-snapshots.js";
 export function prepareRunFork(input) {
     const requestId = requireId(input.requestId, "request-id");
     const snapshot = canonicalPath(input.snapshot, false);
@@ -21,7 +21,8 @@ export function prepareRunFork(input) {
     if (fs.existsSync(receiptFile)) {
         const receipt = readJson(receiptFile);
         const prior = receipt?.intent;
-        if (!prior || prior.request_id !== requestId || prior.snapshot !== snapshot || prior.engine_version !== engineVersion || prior.integrity_checksum !== (input.integrityChecksum ?? null))
+        if (!prior || prior.request_id !== requestId || prior.snapshot !== snapshot || prior.engine_version !== engineVersion || prior.integrity_checksum !== (input.integrityChecksum ?? null)
+            || (prior.operational_decision_sha256 ?? null) !== (input.operationalDecisionFile ? fileHash(input.operationalDecisionFile) : null))
             throw new AppError("fork_request_conflict", "Fork output already belongs to another request", 1, { output });
         return { receipt, output };
     }
@@ -30,6 +31,7 @@ export function prepareRunFork(input) {
     if (manifest.schema_id !== "dd-flow/eval-run-snapshot@5" || manifest.purpose !== "stage_entry" || typeof manifest.stage_entry !== "string" || typeof manifest.run_id !== "string") {
         throw new AppError("fork_checkpoint_invalid", "Fork requires a sealed stage-entry snapshot", 2, { snapshot });
     }
+    const operational = prepareSnapshotOperationalDecision({ snapshot, runId: manifest.run_id, projectRoot: path.join(output, "project"), now: new Date().toISOString(), ...(input.operationalDecisionFile ? { file: input.operationalDecisionFile } : {}) });
     const intent = {
         schema_id: "dd-flow/run-fork-intent@1",
         request_id: requestId,
@@ -37,6 +39,7 @@ export function prepareRunFork(input) {
         snapshot_sha256: fileHash(manifestFile),
         engine_version: engineVersion,
         integrity_checksum: input.integrityChecksum ?? null,
+        ...(operational ? { operational_decision_sha256: crypto.createHash("sha256").update(operational.bytes).digest("hex") } : {}),
         target_stage: manifest.stage_entry
     };
     assertForkOutput(output, intentFile, intent);
@@ -58,6 +61,8 @@ export function forkRunFromSnapshot(context, input, prepared = prepareRunFork(in
     if ("receipt" in prepared)
         return prepared.receipt;
     const { manifest, snapshot, output, intent, intentFile, receiptFile, requestId } = prepared;
+    if (input.operationalDecisionFile && fileHash(input.operationalDecisionFile) !== intent.operational_decision_sha256)
+        throw new AppError("operational_decision_changed", "Accepted fork decision changed after preparation", 2);
     assertForkOutput(output, intentFile, intent);
     const projectRoot = path.join(output, "project");
     const home = path.join(output, "dd-flow-home");
@@ -69,7 +74,7 @@ export function forkRunFromSnapshot(context, input, prepared = prepareRunFork(in
     writeJsonAtomic(intentFile, { schema_id: "dd-flow/run-fork-intent@1", intent });
     fs.mkdirSync(projectRoot, { recursive: true, mode: 0o700 });
     const targetContext = createContext({ ...context.env, DD_FLOW_HOME: home, DD_FLOW_ENGINE_MODE: "1" }, "initialize", prearmedImportId ?? undefined);
-    const restoreInput = { snapshot, projectRoot };
+    const restoreInput = { snapshot, projectRoot, ...(input.operationalDecisionFile ? { operationalDecisionFile: input.operationalDecisionFile } : {}) };
     const restored = restoreEvalRunSnapshot(targetContext, restoreInput, prepareEvalRunRestore(targetContext, restoreInput, manifest), prearmedImportId ?? undefined);
     const restoredContext = createContext({ ...context.env, DD_FLOW_HOME: home, DD_FLOW_ENGINE_MODE: "1" });
     try {