@deksden-com/dd-flow-cli 0.9.0-beta.55 → 0.9.0-beta.62

package/dist/services/recovery-observation-budget.js

@@ -0,0 +1,53 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { AppError } from "../shared/errors.js";
+export const recoveryObservationBudgetMs = 120_000;
+const activeObservation = new AsyncLocalStorage();
+export const withRecoveryObservation = (budget, action) => activeObservation.run(budget, action);
+export const recoveryRpcTimeout = (maximum) => activeObservation.getStore()?.timeout(maximum) ?? maximum;
+/** Call only while holding the existing owner-claim transaction. A replay of
+ * an explicit request never replenishes time, even after another request. */
+export function renewRecoveryObservation(snapshot, requestId) {
+    if (!requestId.trim() || requestId.length > 200)
+        throw new AppError("usage", "Reconciliation requires a nonempty request ID of at most 200 characters", 2);
+    const previous = snapshot.recovery_observation;
+    const ids = previous?.reconciliation_ids ?? [];
+    if (!Array.isArray(ids) || ids.some(id => typeof id !== "string"))
+        throw new AppError("recovery_observation_invalid", "Retained reconciliation identities are malformed", 1);
+    if (ids.includes(requestId))
+        return snapshot;
+    return { ...snapshot, recovery_observation: { budget_ms: recoveryObservationBudgetMs, remaining_ms: recoveryObservationBudgetMs, reconciliation_ids: [...ids, requestId], poll_index: 0 } };
+}
+/**
+ * Measures only time this owner can actually observe. Large scheduler gaps
+ * reset the sample instead of turning host sleep into evidence about a remote
+ * process. Persisting the remaining budget makes a recovered owner continue
+ * reconciliation rather than silently receiving a new unlimited wait.
+ */
+export function recoveryObservationBudget(saved, monotonic = () => performance.now()) {
+    const prior = saved && typeof saved === "object" ? saved.recovery_observation : null;
+    let remaining = typeof prior?.remaining_ms === "number" && Number.isFinite(prior.remaining_ms)
+        ? Math.max(0, Math.min(recoveryObservationBudgetMs, prior.remaining_ms))
+        : prior == null ? recoveryObservationBudgetMs : 0;
+    let last = monotonic();
+    let pollIndex = Number.isSafeInteger(prior?.poll_index) && prior.poll_index >= 0 ? prior.poll_index : 0;
+    const observation = {
+        exhausted: () => {
+            const current = monotonic();
+            const elapsed = current - last;
+            last = current;
+            // A gap means this process was not observing; it never proves progress
+            // or death and therefore does not consume the reconciliation budget.
+            if (elapsed >= 0 && elapsed <= 60_000)
+                remaining = Math.max(0, remaining - elapsed);
+            return remaining === 0;
+        },
+        state: () => ({ budget_ms: recoveryObservationBudgetMs, remaining_ms: Math.ceil(remaining), reconciliation_ids: prior?.reconciliation_ids ?? [], poll_index: pollIndex }),
+        timeout: (maximum = 30_000) => {
+            if (observation.exhausted())
+                throw new AppError("recovery_observation_budget_exhausted", "Recovery observation budget exhausted; explicit reconciliation is required", 1);
+            return Math.max(1, Math.min(maximum, Math.ceil(remaining)));
+        },
+        nextDelay: () => Math.min([1000, 2000, 5000, 10000][Math.min(pollIndex++, 3)], Math.ceil(remaining)),
+    };
+    return observation;
+}

package/dist/services/run-control-worker.js

@@ -13,6 +13,17 @@ import { confirmManagedProcess, finishManagedProcess, heartbeatManagedProcess, m
 import { interruptControlledSessions, observeControlledSessions, recordRunControlIntent, runControlStatus } from "./run-control.js";
 import { recoveryGuard, recordRecoveryCapture, sealRunRecovery } from "./run-recovery.js";
 import { createEvalRunSnapshot, verifyRecoverySnapshot } from "./eval-snapshots.js";
+import { recoveryObservationBudget, renewRecoveryObservation, withRecoveryObservation } from "./recovery-observation-budget.js";
+export async function reconcileRunControl(context, input) {
+    const status = runControlStatus(context, input);
+    const control = status.control;
+    if (!control?.current || !["requested", "settled"].includes(control.status))
+        throw new AppError("run_control_state_changed", "Reconciliation requires the current draining/sealed control", 1);
+    renewRecoveryObservation({}, input.requestId); // Validate before claiming anything.
+    if (control.status === "requested")
+        await launchControlWorker(context, { ...input, reconcileId: input.requestId });
+    return runControlStatus(context, input);
+}
 /** The caller publishes intent and launches a control owner; it never owns drain. */
 export async function requestRunControl(context, input) {
     if (input.waitMs !== undefined && (!Number.isSafeInteger(input.waitMs) || input.waitMs < 0 || input.waitMs > 60_000))
@@ -24,7 +35,7 @@ export async function requestRunControl(context, input) {
     };
     if (!receipt.control_id || receipt.status !== "requested")
         return projectReceipt();
-    const scope = { projectRoot: input.projectRoot, runId: input.runId, controlId: String(receipt.control_id) };
+    const scope = { projectRoot: input.projectRoot, runId: input.runId, controlId: String(receipt.control_id), ...(input.reconcileId ? { reconcileId: input.reconcileId } : {}) };
     await launchControlWorker(context, scope);
     const deadline = performance.now() + (input.waitMs ?? 0);
     while (performance.now() < deadline) {
@@ -49,8 +60,11 @@ async function launchControlWorker(context, input) {
     context.db.exec("BEGIN IMMEDIATE");
     try {
         claimed = context.db.run("INSERT OR IGNORE INTO run_control_workers (control_id, owner_token, status, resource_home, state_dir, created_at, updated_at) VALUES (?, ?, 'starting', ?, ?, ?, ?)", [input.controlId, token, home, directory, now, now]).changes === 1;
-        if (claimed)
+        if (claimed) {
             recordControlLauncher(context, input.controlId, token);
+            if (input.reconcileId)
+                context.db.run("UPDATE run_control_workers SET snapshot_json = ? WHERE control_id = ?", [JSON.stringify(renewRecoveryObservation({}, input.reconcileId)), input.controlId]);
+        }
         context.db.exec("COMMIT");
     }
     catch (error) {
@@ -93,6 +107,10 @@ function reclaimControlWorker(context, input, projectId, token, home) {
     const previous = context.db.get("SELECT * FROM run_control_workers WHERE control_id = ?", [input.controlId]);
     if (!previous || !["starting", "running", "failed"].includes(previous.status) || previous.resource_home !== home)
         return false;
+    const saved = JSON.parse(previous.snapshot_json ?? "{}");
+    const requested = input.reconcileId ? renewRecoveryObservation(saved, input.reconcileId) : saved;
+    if (recoveryObservationBudget(requested).state().remaining_ms === 0)
+        return false;
     const registry = fs.existsSync(path.join(home, "runtime.sqlite")) ? getResourceDatabase(home) : null;
     let record = previous.process_id ? registry?.get("SELECT * FROM managed_processes WHERE id = ?", [previous.process_id]) : null;
     const operationId = launchOperationId(input.controlId, previous.owner_token);
@@ -157,6 +175,10 @@ function reclaimControlWorker(context, input, projectId, token, home) {
             context.db.exec("ROLLBACK");
             return false;
         }
+        if (input.reconcileId) {
+            const snapshot = renewRecoveryObservation(JSON.parse(previous.snapshot_json ?? "{}"), input.reconcileId);
+            context.db.run("UPDATE run_control_workers SET snapshot_json = ?, error_json = NULL WHERE control_id = ? AND owner_token = ?", [JSON.stringify(snapshot), input.controlId, token]);
+        }
         const now = context.now();
         context.db.run("INSERT INTO run_control_operations (operation_id, control_id, target_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, 'completed', ?, ?, ?)", [`${launchOperationId(input.controlId, previous.owner_token)}:reconciled`, input.controlId, JSON.stringify({ kind: "control_worker_reconciliation", process_id: record?.id ?? null, resource_home: home }), JSON.stringify({ process_id: record?.id ?? null, pid: record?.pid ?? null, pid_started_at: record?.pid_started_at ?? null, prior_status: previous.status, snapshot: JSON.parse(previous.snapshot_json ?? "null"), error: JSON.parse(previous.error_json ?? "null") }), now, now]);
         recordControlLauncher(context, input.controlId, token);
@@ -216,6 +238,14 @@ export async function serveRunControl(context, input) {
         if (!worker.process_lease_token)
             throw new AppError("run_control_owner_mismatch", "Control owner has no resource lease", 1);
         confirmManagedProcess(resourceContext, { id: worker.process_id, leaseToken: worker.process_lease_token, pid: process.pid, processGroupId: process.pid, ownerPid: process.pid });
+        let priorObservation = null;
+        try {
+            priorObservation = worker.snapshot_json ? JSON.parse(worker.snapshot_json) : null;
+        }
+        catch { /* A malformed prior snapshot cannot grant a new budget. */
+            priorObservation = { recovery_observation: { remaining_ms: 0 } };
+        }
+        const observation = recoveryObservationBudget(priorObservation);
         for (;;) {
             if (leaseLost)
                 throw new AppError("run_control_lease_lost", "Control worker lost its physical ownership lease", 1);
@@ -226,6 +256,18 @@ export async function serveRunControl(context, input) {
                 context.db.run("UPDATE run_control_workers SET status = ?, updated_at = ? WHERE control_id = ? AND owner_token = ?", [status, context.now(), input.controlId, worker.owner_token]);
                 return { ok: true, control_id: input.controlId, status };
             }
+            if (observation.exhausted()) {
+                const latest = context.db.get("SELECT snapshot_json FROM run_control_workers WHERE control_id = ?", [input.controlId]);
+                let retained = {};
+                try {
+                    retained = JSON.parse(latest?.snapshot_json ?? "{}") ?? {};
+                }
+                catch { /* Retain a blocked status even when the old projection is malformed. */ }
+                const snapshot = { ...retained, status: "recovery_blocked", settled: false, pending_reasons: ["recovery_observation_budget_exhausted"],
+                    error: { code: "recovery_observation_budget_exhausted", message: "Control settlement has no confirmed result within the active observation budget" }, recovery_observation: observation.state() };
+                context.db.run("UPDATE run_control_workers SET status = 'failed', snapshot_json = ?, error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify(snapshot), JSON.stringify(snapshot.error), context.now(), input.controlId, worker.owner_token]);
+                return { ok: true, control_id: input.controlId, ...snapshot };
+            }
             if (guard.status === "sealed") {
                 // Capture retries are local evidence work. Never dispatch another
                 // native interrupt/inspect merely because publication previously failed.
@@ -250,14 +292,15 @@ export async function serveRunControl(context, input) {
               AND EXISTS (SELECT 1 FROM run_recovery_guards g WHERE g.recovery_id = run_controls.recovery_id AND g.status = 'sealed' AND g.capture_path = ? AND g.generation = (SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = g.project_id AND run_id = g.run_id))`, [context.now(), input.controlId, guard.recovery_id, worker.owner_token, capture]);
                     if (applied.changes !== 1)
                         throw new AppError("run_control_state_changed", "Control changed before its capture could be acknowledged", 1);
-                    const snapshot = { ...settlement, status: "settled", capture_path: capture };
+                    const snapshot = { ...settlement, status: "settled", capture_path: capture, recovery_observation: observation.state() };
                     context.db.run("UPDATE run_control_workers SET status = 'completed', snapshot_json = ?, error_json = NULL, updated_at = ? WHERE control_id = ? AND owner_token = ?", [JSON.stringify(snapshot), context.now(), input.controlId, worker.owner_token]);
                     return { ok: true, control_id: input.controlId, ...snapshot };
                 }
                 catch (error) {
-                    const snapshot = { ...settlement, status: "requested", capture: { status: "pending", error: failure(error) }, pending_reasons: ["recovery_capture_required"] };
+                    observation.exhausted();
+                    const snapshot = { ...settlement, status: "requested", capture: { status: "pending", error: failure(error) }, pending_reasons: ["recovery_capture_required"], recovery_observation: observation.state() };
                     context.db.run("UPDATE run_control_workers SET snapshot_json = ?, error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify(snapshot), JSON.stringify(failure(error)), context.now(), input.controlId, worker.owner_token]);
-                    await delay(5_000);
+                    await delay(observation.nextDelay());
                     continue;
                 }
             }
@@ -265,12 +308,16 @@ export async function serveRunControl(context, input) {
             const interrupt = current.action === "stop" || request.escalate === "interrupt" && Date.now() >= Date.parse(current.created_at) + (request.grace_ms ?? 30_000);
             let snapshot;
             try {
-                const cancellation = interrupt ? await interruptControlledSessions(context, input) : null;
-                snapshot = { ...await observeControlledSessions(context, input), ...(cancellation ? { cancellation } : {}) };
+                snapshot = await withRecoveryObservation(observation, async () => {
+                    const cancellation = interrupt ? await interruptControlledSessions(context, input) : null;
+                    return { ...await observeControlledSessions(context, input), ...(cancellation ? { cancellation } : {}) };
+                });
             }
             catch (error) {
                 snapshot = { ok: true, control_id: input.controlId, status: "requested", settled: false, pending_reasons: ["control_observation_failed"], error: failure(error) };
             }
+            observation.exhausted();
+            snapshot = { ...snapshot, recovery_observation: observation.state() };
             context.db.run("UPDATE run_control_workers SET snapshot_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify(snapshot), context.now(), input.controlId, worker.owner_token]);
             if (snapshot.settled === true) {
                 try {
@@ -281,7 +328,7 @@ export async function serveRunControl(context, input) {
                     context.db.run("UPDATE run_control_workers SET snapshot_json = ?, error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify({ ...snapshot, pending_reasons: ["recovery_seal_required"], error: failure(error) }), JSON.stringify(failure(error)), context.now(), input.controlId, worker.owner_token]);
                 }
             }
-            await delay(1_000);
+            await delay(observation.nextDelay());
         }
     }
     catch (error) {

package/dist/services/run-control.js

@@ -1,3 +1,4 @@
+import { setFlowRunRecoveryPaused } from "./runs.js";
 import crypto from "node:crypto";
 import fs from "node:fs";
 import { ensureRunControlStorage } from "../storage/database.js";
@@ -456,20 +457,32 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
         assertCurrent();
         const inserted = context.db.run("INSERT OR IGNORE INTO run_control_operations (operation_id, control_id, target_json, status, created_at, updated_at) VALUES (?, ?, ?, 'dispatching', ?, ?)", [operationId, observationId, JSON.stringify({ session_id: session.id, state_dir: session.state_dir, executable }), now, now]);
         const prior = context.db.get("SELECT status, receipt_json FROM run_control_operations WHERE operation_id = ?", [operationId]);
+        let cancellation;
+        let cancellationError;
         try {
             if (!inserted.changes && prior.status === "completed") {
                 const saved = JSON.parse(prior.receipt_json);
-                assertNativeCancellationReceipt(saved, session.id, harness === "zcode-acp");
-                return { operation_id: operationId, session_id: session.id, status: prior.status, receipt: saved, reused: true };
+                assertNativeCancellationReceipt(saved, session.id);
+                cancellation = saved;
+            }
+            else {
+                const { status, receipt } = await cancelControlledNativeSession({ executable, sessionId: session.id, stateDir: session.state_dir, env, operationId, observe: !inserted.changes, assertCurrent });
+                context.db.run("UPDATE run_control_operations SET status = ?, receipt_json = ?, error_json = NULL, updated_at = ? WHERE operation_id = ?", [status, JSON.stringify(receipt), context.now(), operationId]);
+                cancellation = receipt;
             }
-            const { status, receipt } = await cancelControlledNativeSession({ executable, sessionId: session.id, stateDir: session.state_dir, env, operationId, observe: !inserted.changes, assertCurrent, requireSettled: harness === "zcode-acp" });
-            context.db.run("UPDATE run_control_operations SET status = ?, receipt_json = ?, error_json = NULL, updated_at = ? WHERE operation_id = ?", [status, JSON.stringify(receipt), context.now(), operationId]);
-            return { operation_id: operationId, session_id: session.id, status, receipt };
         }
         catch (error) {
-            const failure = { code: error instanceof AppError ? error.code : "control_observation_failed", message: String(error) };
-            context.db.run("UPDATE run_control_operations SET status = 'outcome_unknown', error_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(failure), context.now(), operationId]);
-            return { operation_id: operationId, session_id: session.id, status: "outcome_unknown", error: failure };
+            cancellationError = { code: error instanceof AppError ? error.code : "control_observation_failed", message: String(error) };
+            context.db.run("UPDATE run_control_operations SET status = 'outcome_unknown', error_json = ?, updated_at = ? WHERE operation_id = ? AND status != 'completed'", [JSON.stringify(cancellationError), context.now(), operationId]);
+            if (cancellationError.code === "controller_operation_mismatch")
+                return { operation_id: operationId, session_id: session.id, status: "outcome_unknown", error: cancellationError };
+        }
+        try {
+            const observed = await inspectControlledNativeSession({ executable, sessionId: session.id, stateDir: session.state_dir, harness, env, operationId: `${observationId}:inspect:${crypto.randomUUID()}`, assertCurrent });
+            return { operation_id: operationId, session_id: session.id, status: observed.settled === true ? "completed" : "pending", receipt: observed, cancellation, reused: !inserted.changes, ...(cancellationError ? { cancellation_error: cancellationError } : {}) };
+        }
+        catch (error) {
+            return { operation_id: operationId, session_id: session.id, status: "pending", cancellation, cancellation_error: cancellationError, error: { code: error instanceof AppError ? error.code : "control_observation_failed", message: String(error) } };
         }
     }));
     const processResults = Promise.all(processTargets.map(async ({ home, record }) => {
@@ -532,7 +545,17 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
         const executable = directoryAdapters.get(directory) ?? (leader && JSON.parse(leader.metadata_json).adapter_executable);
         if (!daemonId || !leader || !executable)
             return { state_dir: directory, status: "pending", reason: "daemon_close_owner_unproven" };
-        const operationId = `${observationId}:close-daemon:${crypto.createHash("sha256").update(JSON.stringify([directory, daemonId])).digest("hex")}`;
+        let operationId = `${observationId}:close-daemon:${crypto.createHash("sha256").update(JSON.stringify([directory, daemonId])).digest("hex")}`;
+        const nativeOperations = retainedDaemonOperations(directory);
+        const targetList = [...targets.values()];
+        const sessionsSettled = ids.every(id => nodes.some((node, index) => targetList[index]?.session.state_dir === directory && targetList[index]?.session.id === id && node.status === "completed"));
+        // A confirmed failed tree guard admits a new shutdown attempt only after
+        // fresh tree settlement. Unknown outcomes retain their original ID.
+        if (sessionsSettled) {
+            while (nativeOperations.some(operation => operation.operation_id === operationId && operation.daemon_id === daemonId && operation.operation === "daemon.stop"
+                && operation.state === "failed" && operation.error?.code === "tree_not_settled"))
+                operationId += ":finalize";
+        }
         assertCurrent();
         const inserted = context.db.run("INSERT OR IGNORE INTO run_control_operations (operation_id, control_id, target_json, status, created_at, updated_at) VALUES (?, ?, ?, 'dispatching', ?, ?)", [operationId, observationId, JSON.stringify({ state_dir: directory, daemon_id: daemonId, executable, cancel_tree: true }), context.now(), context.now()]);
         const prior = context.db.get("SELECT status, receipt_json FROM run_control_operations WHERE operation_id = ?", [operationId]);
@@ -746,6 +769,7 @@ export function recordRunControlIntent(context, input) {
       FROM runs r, json_each(r.index_json, '$.stage_runs') stage
       WHERE r.project_id = ? AND r.id = ? AND json_extract(stage.value, '$.status') = 'running'
         AND EXISTS (SELECT 1 FROM run_control_admissions a JOIN sessions s ON s.session_id = a.session_id AND s.project_id = r.project_id WHERE a.control_id = ? AND s.current_stage = json_extract(stage.value, '$.stage'))`, [controlId, controlId, project.id, input.runId, controlId]);
+        setFlowRunRecoveryPaused(context, project.id, input.runId, true);
         context.db.exec("COMMIT");
         return { ok: true, control_id: controlId, recovery_id: recoveryId, generation, action: input.action, status: "requested", settled: false };
     }

package/dist/services/run-controller-capture.js

@@ -7,6 +7,7 @@ import { observeControllerBoundary } from "./run-control.js";
 import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
 import { appendEvent, digest, errorRecord, requireController, requireRun } from "./run-controller-state.js";
 import { getFlowRunStatus } from "./runs.js";
+import { recoveryObservationBudget, withRecoveryObservation } from "./recovery-observation-budget.js";
 /** Capture is performed by the productive owner, before it releases the next
  * Stage. An observer never races the controller to copy a completed boundary. */
 export async function captureControllerBoundary(context, row, manifest, stage, nextStage, assertPhysicalOwner) {
@@ -63,6 +64,14 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
         if (nextStage ? continuation.kind !== "start_stage" || continuation.stage !== nextStage : continuation.kind !== "terminal")
             throw new AppError("controller_boundary_changed", "Stage successor changed after its capture fence", 1);
     };
+    let priorObservation = null;
+    try {
+        priorObservation = prior?.receipt_json ? JSON.parse(prior.receipt_json) : null;
+    }
+    catch {
+        priorObservation = { recovery_observation: { remaining_ms: 0 } };
+    }
+    const observationBudget = recoveryObservationBudget(priorObservation);
     try {
         let receipt;
         if (fs.existsSync(output)) {
@@ -72,9 +81,13 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
         else {
             let lastPending = "";
             for (;;) {
+                if (observationBudget.exhausted()) {
+                    context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), context.now(), operationId]);
+                    throw new AppError("recovery_observation_budget_exhausted", "Boundary capture has no stable settlement within the active observation budget", 1, { recovery_observation: observationBudget.state() });
+                }
                 assertCurrent();
                 let verify = () => { throw new AppError("controller_boundary_unsettled", "Boundary has no stable writer observation", 1); };
-                const observation = await observeControllerBoundary(context, { ...scope, controllerId: row.controller_id, operationId, assertCurrent, retainVerification: value => { verify = value; } });
+                const observation = await withRecoveryObservation(observationBudget, () => observeControllerBoundary(context, { ...scope, controllerId: row.controller_id, operationId, assertCurrent, retainVerification: value => { verify = value; } }));
                 if (observation.settled === true) {
                     try {
                         createEvalRunSnapshot(context, { ...scope, output, ...(nextStage ? { stageEntry: nextStage } : { candidate: true }), boundary: { controllerId: row.controller_id, operationId, boundaryKey, observation, verify } });
@@ -88,13 +101,15 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
                         observation.pending_reasons = [error.code];
                     }
                 }
+                observationBudget.exhausted();
+                const published = { ...observation, recovery_observation: observationBudget.state() };
                 const pending = digest(observation.pending_reasons);
+                context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(published), context.now(), operationId]);
                 if (pending !== lastPending) {
-                    context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(observation), context.now(), operationId]);
                     appendEvent(context, row.controller_id, "boundary_capture_pending", { operation_id: operationId, pending_reasons: observation.pending_reasons });
                     lastPending = pending;
                 }
-                await delay(1000);
+                await delay(observationBudget.nextDelay());
             }
         }
         context.db.exec("BEGIN IMMEDIATE");
@@ -111,6 +126,8 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
         }
     }
     catch (error) {
+        observationBudget.exhausted();
+        context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)) WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), operationId]);
         context.db.run("UPDATE run_controller_operations SET status = 'failed', error_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(errorRecord(error)), context.now(), operationId]);
         throw error;
     }

package/dist/services/run-controller-process.js

@@ -6,6 +6,7 @@ import { fileURLToPath } from "node:url";
 import { AppError } from "../shared/errors.js";
 import { getResourceDatabase } from "../storage/database.js";
 import { finishManagedProcess, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processStartedAt, processTreeIsAlive, registerManagedProcess } from "./managed-processes.js";
+import { setFlowRunRecoveryPaused } from "./runs.js";
 import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
 import { requireRun, requireController, controllerReceipt, appendEvent, digest, errorRecord } from "./run-controller-state.js";
 const launchId = (row) => `${row.controller_id}:owner-launch:${digest(row.owner_token)}`;
@@ -172,7 +173,17 @@ export async function spawnController(context, row) {
         return controllerReceipt(requireController(context, controllerId));
     }
     catch (error) {
-        context.db.run("UPDATE run_controllers SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status = 'starting' AND owner_pid IS NULL", [JSON.stringify(errorRecord(error)), context.now(), controllerId, token]);
+        context.db.exec("SAVEPOINT controller_spawn_status");
+        try {
+            const changed = context.db.run("UPDATE run_controllers SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status = 'starting' AND owner_pid IS NULL", [JSON.stringify(errorRecord(error)), context.now(), controllerId, token]);
+            if (changed.changes && (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) === row.generation)
+                setFlowRunRecoveryPaused(context, row.project_id, row.run_id, true);
+            context.db.exec("RELEASE controller_spawn_status");
+        }
+        catch (failure) {
+            context.db.exec("ROLLBACK TO controller_spawn_status; RELEASE controller_spawn_status");
+            throw failure;
+        }
         throw error;
     }
 }

package/dist/services/run-controller-state.js

@@ -16,7 +16,7 @@ export function requireController(context, id) {
 export function controllerReceipt(row) {
     const state = JSON.parse(row.state_json);
     const manifest = JSON.parse(row.manifest_json);
-    return { ok: true, schema_id: "dd-flow/run-controller-receipt@1", controller: { controller_id: row.controller_id, request_id: row.request_id, project_id: row.project_id, run_id: row.run_id, generation: row.generation, runtime_budget: manifest.budget ?? null, status: row.status, process_id: row.process_id, owner_pid: row.owner_pid, state_dir: row.state_dir, stage: state.stage, sessions: state.sessions.map(session => ({ session_id: session.id, harness: runtimeHarness(session.profile), profile_id: session.profile.id, state_dir: session.state_dir, stopped: session.stopped === true })), error: row.last_error_json ? JSON.parse(row.last_error_json) : null } };
+    return { ok: true, schema_id: "dd-flow/run-controller-receipt@1", controller: { controller_id: row.controller_id, request_id: row.request_id, project_id: row.project_id, run_id: row.run_id, generation: row.generation, runtime_budget: manifest.budget ?? null, status: row.status, process_id: row.process_id, owner_pid: row.owner_pid, state_dir: row.state_dir, stage: state.stage, sessions: state.sessions.map(session => ({ session_id: session.id, harness: runtimeHarness(session.profile), profile_id: session.profile.id, state_dir: session.state_dir, journal: session.journal, stopped: session.stopped === true })), error: row.last_error_json ? JSON.parse(row.last_error_json) : null } };
 }
 export function appendEvent(context, id, type, data) { context.db.run("INSERT INTO run_controller_events (controller_id, event_type, data_json, created_at) VALUES (?, ?, ?, ?)", [id, type, JSON.stringify(data), context.now()]); }
 export function saveState(context, row, state) { context.db.run("UPDATE run_controllers SET state_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [JSON.stringify(state), context.now(), row.controller_id, row.owner_token]); }

package/dist/services/run-controller.js

@@ -10,7 +10,7 @@ import { AppError } from "../shared/errors.js";
 import { parseRuntimeBudget } from "./runtime-budget.js";
 import { requireProjectByRoot } from "./projects.js";
 import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
-import { getFlowRunStatus } from "./runs.js";
+import { getFlowRunStatus, setFlowRunRecoveryPaused } from "./runs.js";
 import { loadExternalStageContext } from "./stage-context.js";
 import { flowCommand } from "./stage-pause.js";
 import { resolveStageTransition, sameSessionProfile, runtimeHarness } from "./execution-policy.js";
@@ -202,7 +202,17 @@ export async function serveRunController(context, input) {
     };
     const assertOwner = () => { assertPhysicalOwner(); assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation); };
     try {
-        markControllerDispatchReady(context, row);
+        context.db.exec("SAVEPOINT controller_ready_status");
+        try {
+            markControllerDispatchReady(context, row);
+            if (!state.recovery && (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) === row.generation && requireRun(context, row.project_id, row.run_id).status === "paused")
+                setFlowRunRecoveryPaused(context, row.project_id, row.run_id, false);
+            context.db.exec("RELEASE controller_ready_status");
+        }
+        catch (error) {
+            context.db.exec("ROLLBACK TO controller_ready_status; RELEASE controller_ready_status");
+            throw error;
+        }
         appendEvent(context, row.controller_id, "owner_started", { pid: process.pid, generation: row.generation });
         if (state.recovery)
             await resumeControllerSession(context, row, manifest, state, assertPhysicalOwner);
@@ -221,10 +231,21 @@ export async function serveRunController(context, input) {
                 return controllerReceipt(requireController(context, row.controller_id));
             }
         }
-        // A failed capture must not reopen admission when this process exits.
-        context.db.run("UPDATE run_controllers SET status = CASE WHEN status IN ('capturing','capture_failed') THEN 'capture_failed' ELSE 'recovery_required' END, last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [JSON.stringify(errorRecord(error)), context.now(), row.controller_id, row.owner_token]);
-        context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status IN ('dispatching','active')", [JSON.stringify(errorRecord(error)), context.now(), row.project_id, row.run_id, row.controller_id]);
-        appendEvent(context, row.controller_id, "recovery_required", errorRecord(error));
+        context.db.exec("SAVEPOINT controller_failure_status");
+        try {
+            // A failed capture must not reopen admission when this process exits.
+            const changed = context.db.run("UPDATE run_controllers SET status = CASE WHEN status IN ('capturing','capture_failed') THEN 'capture_failed' ELSE 'recovery_required' END, last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status NOT IN ('completed','cancelled','superseded')", [JSON.stringify(errorRecord(error)), context.now(), row.controller_id, row.owner_token]);
+            context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE project_id = ? AND run_id = ? AND dispatch_owner = ? AND status IN ('dispatching','active')", [JSON.stringify(errorRecord(error)), context.now(), row.project_id, row.run_id, row.controller_id]);
+            if (changed.changes && (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) === row.generation) {
+                setFlowRunRecoveryPaused(context, row.project_id, row.run_id, true);
+            }
+            appendEvent(context, row.controller_id, "recovery_required", errorRecord(error));
+            context.db.exec("RELEASE controller_failure_status");
+        }
+        catch (failure) {
+            context.db.exec("ROLLBACK TO controller_failure_status; RELEASE controller_failure_status");
+            throw failure;
+        }
         throw error;
     }
     finally {

package/dist/services/run-recovery.js

@@ -5,7 +5,7 @@ import path from "node:path";
 import { AppError } from "../shared/errors.js";
 import { resolveProjectRoot } from "../storage/paths.js";
 import { requireProjectByRoot } from "./projects.js";
-import { appendFlowRunTimelineEvent } from "./runs.js";
+import { appendFlowRunTimelineEvent, setFlowRunRecoveryPaused } from "./runs.js";
 import { verifyRecoverySnapshot } from "./eval-snapshots.js";
 import { findRecentMatchingHookEvent, hookSessionIdentity, recoveryAcceptMatchKey } from "./hooks.js";
 import { flowCommand } from "./stage-pause.js";
@@ -123,6 +123,7 @@ export function beginRunRecovery(context, input) {
             context.db.run("UPDATE run_recovery_bindings SET status = 'abandoned' WHERE recovery_id = ? AND status = 'prepared'", [prior.recovery_id]);
         }
         context.db.run("INSERT INTO run_recovery_guards (recovery_id, project_id, run_id, generation, status, interruption_json, settlement_json, capture_path, created_at, updated_at, sealed_at, resumed_at) VALUES (?, ?, ?, ?, 'draining', ?, NULL, NULL, ?, ?, NULL, NULL)", [recoveryId, project.id, run.id, generation, JSON.stringify(interruption), now, now]);
+        setFlowRunRecoveryPaused(context, project.id, run.id, true);
         context.db.exec("COMMIT");
     }
     catch (error) {
@@ -147,9 +148,18 @@ export function sealRunRecovery(context, input) {
         throw new AppError("recovery_invalid_state", "Recovery is not draining", 1, { recovery_id: guard.recovery_id, status: guard.status });
     materializePendingWorkResults(context, project.id, run.id);
     const now = context.now();
-    const updated = context.db.run("UPDATE run_recovery_guards SET status = 'sealed', settlement_json = ?, sealed_at = ?, updated_at = ? WHERE recovery_id = ? AND status = 'draining' AND generation = (SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = ? AND run_id = ?)", [JSON.stringify(settlement), now, now, guard.recovery_id, project.id, run.id]);
-    if (updated.changes !== 1)
-        throw new AppError("recovery_state_changed", "Recovery state changed before it could be sealed", 1, { recovery_id: guard.recovery_id });
+    context.db.exec("SAVEPOINT seal_run_status");
+    try {
+        const updated = context.db.run("UPDATE run_recovery_guards SET status = 'sealed', settlement_json = ?, sealed_at = ?, updated_at = ? WHERE recovery_id = ? AND status = 'draining' AND generation = (SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = ? AND run_id = ?)", [JSON.stringify(settlement), now, now, guard.recovery_id, project.id, run.id]);
+        if (updated.changes !== 1)
+            throw new AppError("recovery_state_changed", "Recovery state changed before it could be sealed", 1, { recovery_id: guard.recovery_id });
+        setFlowRunRecoveryPaused(context, project.id, run.id, true);
+        context.db.exec("RELEASE seal_run_status");
+    }
+    catch (error) {
+        context.db.exec("ROLLBACK TO seal_run_status; RELEASE seal_run_status");
+        throw error;
+    }
     appendFlowRunTimelineEvent(context, project.id, run.id, { type: "run_recovery_sealed", recovery_id: guard.recovery_id, generation: guard.generation, settlement });
     return { ok: true, recovery: { recovery_id: guard.recovery_id, run_id: run.id, generation: guard.generation, status: "sealed", settlement } };
 }
@@ -286,6 +296,7 @@ export function acceptRunRecovery(context, input) {
         }
         context.db.run("UPDATE run_recovery_bindings SET status = 'accepted', hook_event_key = ?, accepted_at = ? WHERE recovery_id = ?", [eventKey, now, binding.recovery_id]);
         context.db.run("UPDATE run_recovery_guards SET status = 'recovered', resumed_at = ?, updated_at = ? WHERE recovery_id = ?", [now, now, guard.recovery_id]);
+        setFlowRunRecoveryPaused(context, project.id, run.id, false);
         context.db.exec("COMMIT");
     }
     catch (error) {

package/dist/services/runs.js

@@ -5,7 +5,7 @@ import { spawnSync } from "node:child_process";
 import { formatFullId, isFullEntityId, isShortEntityId, parseFullEntityId } from "../domain/entity-ids.js";
 import { checksumForFlowFlagValues, isFlowFlagDowngrade, loadCanonicalFlowContract, loadProjectFlowContract, resolveFlowFlags } from "../domain/flow-contract.js";
 import { AppError } from "../shared/errors.js";
-import { ensureDir, projectRunHome, projectRunJsonPath, resolveProjectRoot } from "../storage/paths.js";
+import { ensureDir, canonicalPath, projectRunHome, projectRunJsonPath, resolveProjectRoot } from "../storage/paths.js";
 import { appendAudit } from "./audit.js";
 import { registerProject, requireProjectByRoot } from "./projects.js";
 import { buildRunFlowGuidance } from "./flow-guidance.js";
@@ -788,8 +788,24 @@ function operationalRunStatus(context, projectId, runId) {
     const leaves = unfinished.filter((work) => !parents.has(work.work_id));
     return leaves.length > 0 && leaves.every((work) => work.status === "paused") ? "paused" : "running";
 }
+/** Keep the logical RUN projection aligned with recovery authority. Stage/Work
+ * evidence and verdicts survive a reversible stop; terminal runs never reopen. */
+export function setFlowRunRecoveryPaused(context, projectId, runId, paused) {
+    const run = resolveRun(context, projectId, runId);
+    if (!["running", "paused"].includes(run.status))
+        return;
+    const index = authoritativeIndex(run);
+    const status = paused || index.stage_runs.some(stage => ["paused", "waiting_for_user", "blocked"].includes(stage.status))
+        ? "paused" : operationalRunStatus(context, projectId, runId);
+    if (run.status === status && index.status === status)
+        return;
+    index.status = status;
+    index.updated_at = context.now();
+    const project = requireProjectByRoot(context, run.project_root);
+    persistRunState(context, project, run, index);
+}
 export function completeFlowRun(context, input) {
-    const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
+    const project = requireProjectByRoot(context, input.manualOverrideReason ? canonicalPath(input.projectRoot, false) : resolveProjectRoot(input.projectRoot));
     const run = resolveRun(context, project.id, input.runId);
     assertRunMutationAllowed(context, project.id, run.id);
     const index = authoritativeIndex(run);
@@ -856,8 +872,8 @@ export function advanceFlowRun(context, input) {
     index.next_action = control ? null : input.nextAction ?? null;
     index.updated_at = now;
     persistRunState(context, project, run, index);
-    appendAudit(context, { projectId: project.id, eventType: "flow_run.progressed", payload: { run_id: run.id, status: input.status, verdict: input.verdict, next_action: index.next_action } });
-    appendRunTimeline(runArtifactRoot(run), { at: now, type: "run_progressed", run_id: run.id, status: input.status, next_action: index.next_action });
+    appendAudit(context, { projectId: project.id, eventType: "flow_run.progressed", payload: { run_id: run.id, status: index.status, verdict: input.verdict, next_action: index.next_action } });
+    appendRunTimeline(runArtifactRoot(run), { at: now, type: "run_progressed", run_id: run.id, status: index.status, next_action: index.next_action });
     const updatedRun = requireRunById(context, project.id, run.id);
     return { ok: true, ...(control ? { control } : {}), run: flowRunSummary(updatedRun), index, flow_guidance: guidanceForRun(context, updatedRun, index) };
 }
@@ -1189,6 +1205,10 @@ function vnextSpecifyGuidance(run, index) {
     };
 }
 function persistRunState(context, project, run, index, options = {}) {
+    // Fenced productive admission is a logical pause, even while physical work
+    // drains. Late admitted Stage results must not revive a stopped RUN.
+    if (index.status === "running" && ["draining", "sealed", "resuming"].includes(recoveryGuard(context, project.id, run.id)?.status ?? ""))
+        index.status = "paused";
     const runtimePath = run.runtime_path || projectRunJsonPath(context.ddFlowHome, project.id, run.id);
     const previousRuntime = readRuntimeSnapshot(runtimePath);
     const runtimeRevision = Math.max(index.runtime_revision ?? 0, previousRuntime?.runtime_revision ?? 0) + 1;
@@ -1199,22 +1219,30 @@ function persistRunState(context, project, run, index, options = {}) {
     }
     const persistedIndex = persistedVnextIndex(index);
     ensureDir(path.dirname(runtimePath));
-    writeJsonFile(runtimePath, runtimeSnapshotForIndex(persistedIndex, runtimeRevision, history));
-    context.db.run(`UPDATE runs
+    context.db.exec("SAVEPOINT persist_run_state");
+    try {
+        context.db.run(`UPDATE runs
      SET status = ?, verdict = ?, next_action = ?, index_json = ?, runtime_path = ?, run_index_path = ?,
        updated_at = ?, completed_at = ?
      WHERE project_id = ? AND id = ?`, [
-        index.status,
-        index.verdict,
-        index.next_action,
-        JSON.stringify(persistedIndex),
-        runtimePath,
-        runtimePath,
-        index.updated_at,
-        index.completed_at ?? null,
-        project.id,
-        run.id
-    ]);
+            index.status,
+            index.verdict,
+            index.next_action,
+            JSON.stringify(persistedIndex),
+            runtimePath,
+            runtimePath,
+            index.updated_at,
+            index.completed_at ?? null,
+            project.id,
+            run.id
+        ]);
+        writeJsonFile(runtimePath, runtimeSnapshotForIndex(persistedIndex, runtimeRevision, history));
+        context.db.exec("RELEASE persist_run_state");
+    }
+    catch (error) {
+        context.db.exec("ROLLBACK TO persist_run_state; RELEASE persist_run_state");
+        throw error;
+    }
 }
 /** vNext stores derived stage/work state only once: stage_runs, Works and Sessions. */
 function persistedVnextIndex(index) {
@@ -1576,5 +1604,5 @@ export function gitFacts(workspaceRoot) {
         status: status === null ? "unavailable" : status.length > 0 ? "dirty" : "clean"
     };
 }
-import { assertRunMutationAllowed, assertStageSettlementAllowed } from "./run-recovery.js";
+import { assertRunMutationAllowed, assertStageSettlementAllowed, recoveryGuard } from "./run-recovery.js";
 import { runControlDirective } from "./run-control.js";