@deksden-com/dd-flow-cli 0.9.0-beta.123 → 0.9.0-beta.127

package/dist/services/recovery-observation-budget.js

@@ -1,9 +1,12 @@
 import { AsyncLocalStorage } from "node:async_hooks";
 import { AppError } from "../shared/errors.js";
+import { validateObservationDuration } from "../harness-runtime/lib/observation-clock.mjs";
 export const recoveryObservationBudgetMs = 120_000;
 const activeObservation = new AsyncLocalStorage();
-export const withRecoveryObservation = (budget, action) => activeObservation.run(budget, action);
+export const withRecoveryObservation = (budget, action, retainActive = false) => activeObservation.run(retainActive ? activeObservation.getStore() ?? budget : budget, action);
 export const recoveryRpcTimeout = (maximum) => {
+    if (maximum != null)
+        validateObservationDuration(maximum);
     const observation = activeObservation.getStore();
     // An unbounded productive transport wait is still bounded while recovery is
     // actively reconciling. Outside recovery, its owner policy decides when to
@@ -23,19 +26,32 @@ export function renewRecoveryObservation(snapshot, requestId) {
         throw new AppError("recovery_observation_invalid", "Retained reconciliation identities are malformed", 1);
     if (ids.includes(requestId))
         return snapshot;
-    return { ...snapshot, recovery_observation: { budget_ms: recoveryObservationBudgetMs, remaining_ms: recoveryObservationBudgetMs, reconciliation_ids: [...ids, requestId], poll_index: 0 } };
+    return { ...snapshot, recovery_observation: { policy_id: "settlement-inactivity@1", observer_started: false, budget_ms: recoveryObservationBudgetMs, remaining_ms: recoveryObservationBudgetMs, progress_markers: [], observation_gaps: 0, reconciliation_ids: [...ids, requestId], poll_index: 0 } };
 }
 /**
- * Measures only time this owner can actually observe. Large scheduler gaps
- * reset the sample instead of turning host sleep into evidence about a remote
- * process. Persisting the remaining budget makes a recovered owner continue
- * reconciliation rather than silently receiving a new unlimited wait.
+ * New episodes measure silence between confirmed retirement facts; legacy
+ * receipts keep cumulative active-observer accounting. A gap never proves
+ * native death, and repeated gaps exhaust observation rather than grant time.
+ * Remaining time and seen proofs survive owner replacement.
  */
 export function recoveryObservationBudget(saved, monotonic = () => performance.now()) {
     const prior = saved && typeof saved === "object" ? saved.recovery_observation : null;
+    if (prior?.policy_id !== undefined && prior.policy_id !== "settlement-inactivity@1")
+        throw new AppError("recovery_observation_invalid", "Unknown retained observation policy", 1);
+    const sliding = !prior || prior.policy_id === "settlement-inactivity@1";
+    if (prior?.progress_markers !== undefined && (!Array.isArray(prior.progress_markers) || prior.progress_markers.some(marker => typeof marker !== "string"))
+        || prior?.observation_gaps !== undefined && (!Number.isSafeInteger(prior.observation_gaps) || prior.observation_gaps < 0)
+        || prior?.observer_started !== undefined && typeof prior.observer_started !== "boolean")
+        throw new AppError("recovery_observation_invalid", "Retained observation progress is malformed", 1);
+    const markers = new Set(prior?.progress_markers ?? []);
+    let gaps = prior?.observation_gaps ?? 0;
     let remaining = typeof prior?.remaining_ms === "number" && Number.isFinite(prior.remaining_ms)
         ? Math.max(0, Math.min(recoveryObservationBudgetMs, prior.remaining_ms))
         : prior == null ? recoveryObservationBudgetMs : 0;
+    // A restored observer cannot know what happened while it was absent. Retain
+    // one bounded uncertainty episode; repeated reconstruction never grants time.
+    if (sliding && prior && prior.observer_started !== false && ++gaps > 1)
+        remaining = 0;
     let last = monotonic();
     let pollIndex = Number.isSafeInteger(prior?.poll_index) && prior.poll_index >= 0 ? prior.poll_index : 0;
     const observation = {
@@ -47,10 +63,25 @@ export function recoveryObservationBudget(saved, monotonic = () => performance.n
             // or death and therefore does not consume the reconciliation budget.
             if (elapsed >= 0 && elapsed <= 60_000)
                 remaining = Math.max(0, remaining - elapsed);
+            else if (sliding && ++gaps > 1)
+                remaining = 0;
             return remaining === 0;
         },
-        state: () => ({ budget_ms: recoveryObservationBudgetMs, remaining_ms: Math.ceil(remaining), reconciliation_ids: prior?.reconciliation_ids ?? [], poll_index: pollIndex }),
+        observe: snapshot => {
+            observation.exhausted();
+            if (!sliding || remaining === 0)
+                return;
+            const proof = settlementProgressMarkers(snapshot);
+            const advanced = proof.some(marker => !markers.has(marker));
+            proof.forEach(marker => markers.add(marker));
+            if (advanced) {
+                remaining = recoveryObservationBudgetMs;
+                gaps = 0;
+            }
+        },
+        state: () => ({ ...(sliding ? { policy_id: "settlement-inactivity@1", observer_started: true, progress_markers: [...markers].sort(), observation_gaps: gaps } : {}), budget_ms: recoveryObservationBudgetMs, remaining_ms: Math.ceil(remaining), reconciliation_ids: prior?.reconciliation_ids ?? [], poll_index: pollIndex }),
         timeout: (maximum = 30_000) => {
+            validateObservationDuration(maximum);
             if (observation.exhausted())
                 throw new AppError("recovery_observation_budget_exhausted", "Recovery observation budget exhausted; explicit reconciliation is required", 1);
             return Math.max(1, Math.min(maximum, Math.ceil(remaining)));
@@ -59,3 +90,32 @@ export function recoveryObservationBudget(saved, monotonic = () => performance.n
     };
     return observation;
 }
+/** Only confirmed settlement facts advance recovery. Polls, errors, timestamps,
+ * and native model activity cannot replenish physical retirement observation. */
+export function settlementProgressMarkers(snapshot) {
+    const result = [];
+    const visit = (value, prefix, depth) => {
+        if (!value || typeof value !== "object" || depth > 12)
+            return;
+        if (Array.isArray(value)) {
+            for (const item of value)
+                visit(item, prefix, depth + 1);
+            return;
+        }
+        const object = value;
+        const identity = ["operation_id", "process_id", "session_id", "run_id", "control_id", "scope_id", "id"].map(key => typeof object[key] === "string" ? `${key}:${object[key]}` : "").filter(Boolean).join("|");
+        const location = `${prefix}/${identity}`;
+        for (const key of ["physical_settled", "native_settled", "settled", "retired", "published", "committed"])
+            if (object[key] === true)
+                result.push(`${location}/${key}`);
+        if (["completed", "released", "sealed", "verified", "retired"].includes(String(object.status)))
+            result.push(`${location}/status:${String(object.status)}`);
+        if (object.state === "settled")
+            result.push(`${location}/state:settled`);
+        for (const [key, item] of Object.entries(object))
+            if (key !== "recovery_observation" && key !== "error" && key !== "metadata" && key !== "capture_error")
+                visit(item, `${location}/${key}`, depth + 1);
+    };
+    visit(snapshot, "", 0);
+    return [...new Set(result)].sort();
+}

package/dist/services/recovery-snapshot-database.js

@@ -1,7 +1,7 @@
 import { DatabaseSync } from "node:sqlite";
 import fs from "node:fs";
 import path from "node:path";
-import { AppError } from "../shared/errors.js";
+import { AppError, errorRecord } from "../shared/errors.js";
 import { registerStoreWriter } from "../storage/writer-contract.js";
 import { managedProcessRuntimeHome } from "./managed-processes.js";
 import { cleanupScopeWriters } from "./scope-writer-cleanup.js";
@@ -56,6 +56,9 @@ export function captureRecoveryDatabase(source, output, projectId, runId, protoc
             rules[table] = ownRun;
         for (const table of ["plan_bindings", "plan_progress", "worktree_records", "audit_events"])
             rules[table] = `${ownProject} AND protocol_id IN (${protocols})`;
+        // Relocation proofs are RUN-owned provenance needed to map unchanged context
+        // bytes through subsequent recovery generations, not portable live authority.
+        rules.audit_events = `${ownProject} AND (protocol_id IN (${protocols}) OR (event_type = 'flow_run.publication_relocation' AND json_extract(payload_json, '$.run_id') IN (SELECT run_id FROM capture_scope)))`;
         for (const table of ["run_controller_events", "run_controller_operations"])
             rules[table] = `controller_id IN (SELECT controller_id FROM run_controllers WHERE ${ownRun})`;
         for (const table of ["run_control_admissions", "run_control_stage_admissions", "run_control_operations", "run_control_workers"])
@@ -89,7 +92,7 @@ export function captureRecoveryDatabase(source, output, projectId, runId, protoc
             fs.rmSync(output, { force: true });
         }
         catch (error) {
-            cleanup.push({ target: output, phase: "remove_incomplete_output", message: String(error) });
+            cleanup.push({ target: output, phase: "remove_incomplete_output", message: String(error), error: errorRecord(error) });
         }
     }
     if (failed || cleanup.length) {
@@ -98,7 +101,7 @@ export function captureRecoveryDatabase(source, output, projectId, runId, protoc
                 primary.details.snapshot_cleanup_errors = cleanup;
             throw primary;
         }
-        throw new AppError("snapshot_database_failed", primary instanceof Error ? primary.message : "Snapshot database cleanup failed", 1, { ...(failed ? { cause: String(primary) } : {}), snapshot_cleanup_errors: cleanup });
+        throw new AppError("snapshot_database_failed", primary instanceof Error ? primary.message : "Snapshot database cleanup failed", 1, { ...(failed ? { cause: errorRecord(primary) } : {}), snapshot_cleanup_errors: cleanup });
     }
 }
 /** Physical leases are evidence, not authority to resurrect on another host. */
@@ -136,7 +139,7 @@ export function readRecoveryResourceEvidence(resourceHome, projectId, runId, aut
                 primary.details.snapshot_cleanup_errors = cleanup;
             throw primary;
         }
-        throw new AppError("snapshot_resource_evidence_failed", primary instanceof Error ? primary.message : "Resource evidence cleanup failed", 1, { ...(failed ? { cause: String(primary) } : {}), snapshot_cleanup_errors: cleanup });
+        throw new AppError("snapshot_resource_evidence_failed", primary instanceof Error ? primary.message : "Resource evidence cleanup failed", 1, { ...(failed ? { cause: errorRecord(primary) } : {}), snapshot_cleanup_errors: cleanup });
     }
     return result;
 }

package/dist/services/review-deferral.js

@@ -0,0 +1,23 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { AppError } from "../shared/errors.js";
+import { assertPortableArtifactRef } from "./portable-refs.js";
+/** A deferral names one project-local file, never an arbitrary path. */
+export function reviewDefEvidence(workspaceRoot, defId) {
+    try {
+        if (!defId || !/^DEF-[A-Za-z0-9][A-Za-z0-9_-]*$/.test(defId))
+            throw new Error("invalid DEF identifier");
+        const relative = `.memory-bank/defs/${defId}.md`;
+        const file = assertPortableArtifactRef(relative, { workspaceRoot, runHome: workspaceRoot, runId: "" });
+        const directory = fs.realpathSync(path.join(workspaceRoot, ".memory-bank", "defs"));
+        const root = fs.realpathSync(workspaceRoot);
+        const directoryRelative = path.relative(root, directory);
+        if (directoryRelative === ".." || directoryRelative.startsWith(`..${path.sep}`) || path.isAbsolute(directoryRelative) || path.dirname(file) !== directory || !fs.statSync(file).isFile())
+            throw new Error("DEF must be a regular file in the project defs directory");
+        return { def_id: defId, path: relative, sha256: crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex") };
+    }
+    catch (error) {
+        throw new AppError("deferral_invalid", "A deferral requires a readable project-local named DEF", 2, { def_id: defId ?? null, cause: error instanceof Error ? error.message : String(error) });
+    }
+}

package/dist/services/run-control-receipt.js

@@ -50,7 +50,7 @@ export function controlStatusReceipt(observation, request = {}) {
         requested_mode: request.action ?? (activeResume ? "resume" : text(control.requested_mode)),
         applied_mode: settled ? control.requested_mode : state === "running" ? "resume" : null,
         state, settled, nodes, unsettled_reasons: reasons,
-        resumability: settled || state === "recovery_required" ? "recovery_required" : ["running", "not_requested", "terminal_noop", "superseded"].includes(state) ? "not_applicable" : "pending",
+        resumability: ["done", "failed", "cancelled"].includes(String(observation.run_status)) ? "not_applicable" : settled || state === "recovery_required" ? "recovery_required" : ["running", "not_requested", "terminal_noop", "superseded"].includes(state) ? "not_applicable" : "pending",
         recovery_id: text(control.recovery_id)
     };
 }

package/dist/services/run-control-worker.js

@@ -5,11 +5,12 @@ import os from "node:os";
 import { spawn } from "node:child_process";
 import { fileURLToPath } from "node:url";
 import { setTimeout as delay } from "node:timers/promises";
-import { AppError } from "../shared/errors.js";
+import { AppError, errorRecord } from "../shared/errors.js";
 import { resolveProjectRoot } from "../storage/paths.js";
 import { requireProjectByRoot } from "./projects.js";
 import { getResourceDatabase } from "../storage/database.js";
-import { confirmManagedProcess, finishManagedProcess, heartbeatManagedProcess, managedProcessRuntimeHome, processIsAlive, processStartedAt, processTreeIsAlive, registerManagedProcess, resourceHome } from "./managed-processes.js";
+import { assertManagedProcessLease, confirmManagedProcess, finishManagedProcess, renewManagedProcess, managedProcessRuntimeHome, processIsAlive, processStartedAt, processTreeIsAlive, registerManagedProcess, resourceHome } from "./managed-processes.js";
+import { managedLeaseMonitor } from "./managed-lease-monitor.js";
 import { interruptControlledSessions, observeControlledSessions, recordRunControlIntent, runControlStatus } from "./run-control.js";
 import { recoveryGuard, recordRecoveryCapture, sealRunRecovery } from "./run-recovery.js";
 import { createEvalRunSnapshot, verifyRecoverySnapshot } from "./eval-snapshots.js";
@@ -106,13 +107,13 @@ export async function waitForControlWorkerAdmission(context, controlId, ownerTok
         if (!worker || worker.owner_token !== ownerToken)
             return "admitted";
         // A running worker without a resource lease is still in the registration
-        // gap; only a physically live retained child can remain pending at expiry.
+        // gap; a known child not proven dead can remain pending, not admitted.
         if (worker.process_id || ["failed", "completed", "superseded"].includes(worker.status))
             return "admitted";
         if (Date.now() >= deadline) {
-            if (launched?.pid_started_at && processIsAlive(launched))
+            if (launched && processIsAlive(launched))
                 return "pending";
-            throw new AppError("run_control_worker_start_timeout", "Control worker did not publish its managed-process lease and is not alive", 1, { control_id: controlId, timeout_ms: timeoutMs });
+            throw new AppError("run_control_worker_start_timeout", "Control worker did not publish its managed-process lease and is not alive", 1, { control_id: controlId, timeout_ms: timeoutMs, launched });
         }
         await delay(25);
     }
@@ -128,8 +129,8 @@ function reclaimControlWorker(context, input, projectId, token, home) {
         return false;
     const saved = JSON.parse(previous.snapshot_json ?? "{}");
     const requested = input.reconcileId ? renewRecoveryObservation(saved, input.reconcileId) : saved;
-    if (recoveryObservationBudget(requested).state().remaining_ms === 0)
-        return false;
+    const observation = recoveryObservationBudget(requested);
+    const exhausted = observation.state().remaining_ms === 0;
     const registry = fs.existsSync(path.join(home, "runtime.sqlite")) ? getResourceDatabase(home) : null;
     let record = previous.process_id ? registry?.get("SELECT * FROM managed_processes WHERE id = ?", [previous.process_id]) : null;
     const operationId = launchOperationId(input.controlId, previous.owner_token);
@@ -180,17 +181,30 @@ function reclaimControlWorker(context, input, projectId, token, home) {
     }
     // The resource store and RUN store are separate: retirement is idempotent,
     // and a crash before the RUN CAS leaves the same dead owner reconcilable.
+    if (exhausted && previous.status === "failed" && saved.status === "recovery_blocked"
+        && saved.recovery_observation?.remaining_ms === 0)
+        return false;
     if (record && !finishManagedProcess(context, { id: record.id, leaseToken: record.lease_token, state: "stopped", reason: "control_owner_crash_reconciled" }))
         return false;
     const launcherBirth = processStartedAt(process.pid);
     return context.db.writeTransaction(() => {
-        const changed = context.db.run(`UPDATE run_control_workers SET owner_token = ?, owner_pid = NULL, status = 'starting', process_id = NULL, process_lease_token = NULL, updated_at = ?
+        // An exhausted episode still needs an operator-visible receipt, but only
+        // after the same exact dead-owner proof and generation CAS as replacement.
+        const blockedError = { code: "recovery_observation_budget_exhausted", message: "Control settlement has no confirmed result within the active observation budget", ...(previous.error_json ? { cause: JSON.parse(previous.error_json) } : {}) };
+        const blocked = { ...saved, status: "recovery_blocked", settled: false,
+            pending_reasons: [...new Set([...(Array.isArray(saved.pending_reasons) ? saved.pending_reasons : []), "recovery_observation_budget_exhausted"])],
+            error: blockedError, recovery_observation: observation.state() };
+        const fields = exhausted ? "status = 'failed', snapshot_json = ?, error_json = ?, updated_at = ?"
+            : "owner_token = ?, owner_pid = NULL, status = 'starting', process_id = NULL, process_lease_token = NULL, updated_at = ?";
+        const values = exhausted ? [JSON.stringify(blocked), previous.error_json ?? JSON.stringify(blockedError), context.now()] : [token, context.now()];
+        const changed = context.db.run(`UPDATE run_control_workers SET ${fields}
       WHERE control_id = ? AND owner_token = ? AND process_id IS ? AND owner_pid IS ? AND status = ?
+        AND snapshot_json IS ? AND error_json IS ?
         AND (SELECT receipt_json FROM run_control_operations WHERE operation_id = ?) IS ?
         AND EXISTS (SELECT 1 FROM run_controls c JOIN run_recovery_guards g ON g.recovery_id = c.recovery_id
           WHERE c.control_id = run_control_workers.control_id AND c.status = 'requested' AND g.status IN ('draining','sealed')
-            AND g.generation = (SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = g.project_id AND run_id = g.run_id))`, [token, context.now(), input.controlId, previous.owner_token, previous.process_id, previous.owner_pid, previous.status, operationId, launch?.receipt_json ?? null]);
-        if (changed.changes !== 1)
+            AND g.generation = (SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = g.project_id AND run_id = g.run_id))`, [...values, input.controlId, previous.owner_token, previous.process_id, previous.owner_pid, previous.status, previous.snapshot_json ?? null, previous.error_json, operationId, launch?.receipt_json ?? null]);
+        if (changed.changes !== 1 || exhausted)
             return false;
         if (input.reconcileId) {
             const snapshot = renewRecoveryObservation(JSON.parse(previous.snapshot_json ?? "{}"), input.reconcileId);
@@ -223,129 +237,174 @@ export async function serveRunControl(context, input) {
         context.db.run("INSERT INTO run_control_operations (operation_id, control_id, target_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, 'completed', ?, ?, ?)", [`${launchOperationId(input.controlId, worker.owner_token)}:worker-claim`, input.controlId, JSON.stringify({ kind: "control_worker_self_claim" }), JSON.stringify({ pid: process.pid, pid_started_at: workerBirth }), now, now]);
     });
     const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: worker.resource_home } };
-    let leaseLost = false;
+    const lease = managedLeaseMonitor(attemptMs => renewManagedProcess({ ...resourceContext, resourceDatabaseMode: "prepared_existing", databaseDeadlineAt: Date.now() + attemptMs }, { id: worker.process_id, leaseToken: worker.process_lease_token }));
+    let timerFailure;
+    let primaryFailure;
+    const assertControlCurrent = () => {
+        const latest = recoveryGuard(context, project.id, input.runId);
+        if (latest?.recovery_id !== intent.recovery_id || !["draining", "sealed"].includes(latest.status))
+            throw new AppError("run_recovery_generation_stale", "Control intent was superseded during renewal", 1);
+    };
+    const ownedContext = { ...context, assertControlOwner: () => {
+            lease.assertNoRenewal();
+            assertManagedProcessLease(resourceContext, { id: worker.process_id, leaseToken: worker.process_lease_token, pid: process.pid });
+            assertControlCurrent();
+        } };
     const heartbeat = setInterval(() => {
-        try {
-            if (!heartbeatManagedProcess(resourceContext, { id: worker.process_id, leaseToken: worker.process_lease_token }))
-                leaseLost = true;
-        }
-        catch {
-            leaseLost = true;
-        }
+        if (!worker.process_id)
+            return;
+        lease.background(assertControlCurrent, error => { timerFailure = error; });
     }, 10_000);
     heartbeat.unref();
-    try {
-        if (!worker.process_id) {
-            const log = path.join(worker.state_dir, "control.log");
-            const record = registerManagedProcess(resourceContext, { kind: "run-control", ownerId: input.controlId, projectId: project.id, runId: input.runId, operationId: `control:${input.controlId}`, uniqueActiveOperation: true, ownerPid: process.pid, stdoutPath: log, stderrPath: log, metadata: { control_id: input.controlId } });
-            worker.process_id = record.id;
-            worker.process_lease_token = record.lease_token;
-            context.db.run("UPDATE run_control_workers SET process_id = ?, process_lease_token = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND owner_pid = ? AND status = 'running'", [record.id, record.lease_token, context.now(), input.controlId, worker.owner_token, process.pid]);
-        }
-        if (!worker.process_lease_token)
-            throw new AppError("run_control_owner_mismatch", "Control owner has no resource lease", 1);
-        confirmManagedProcess(resourceContext, { id: worker.process_id, leaseToken: worker.process_lease_token, pid: process.pid, processGroupId: process.pid, ownerPid: process.pid });
-        let priorObservation = null;
+    let cleanupFailure;
+    const result = await (async () => {
         try {
-            priorObservation = worker.snapshot_json ? JSON.parse(worker.snapshot_json) : null;
-        }
-        catch { /* A malformed prior snapshot cannot grant a new budget. */
-            priorObservation = { recovery_observation: { remaining_ms: 0 } };
-        }
-        const observation = recoveryObservationBudget(priorObservation);
-        for (;;) {
-            if (leaseLost)
-                throw new AppError("run_control_lease_lost", "Control worker lost its physical ownership lease", 1);
-            const current = context.db.get("SELECT * FROM run_controls WHERE control_id = ?", [input.controlId]);
-            const guard = recoveryGuard(context, project.id, input.runId);
-            if (current.status !== "requested" || guard?.recovery_id !== intent.recovery_id || !["draining", "sealed"].includes(guard.status)) {
-                const status = current.status === "settled" ? "completed" : "superseded";
-                context.db.run("UPDATE run_control_workers SET status = ?, updated_at = ? WHERE control_id = ? AND owner_token = ?", [status, context.now(), input.controlId, worker.owner_token]);
-                return { ok: true, control_id: input.controlId, status };
+            if (!worker.process_id) {
+                const log = path.join(worker.state_dir, "control.log");
+                const record = registerManagedProcess(resourceContext, { kind: "run-control", ownerId: input.controlId, projectId: project.id, runId: input.runId, operationId: `control:${input.controlId}`, uniqueActiveOperation: true, ownerPid: process.pid, stdoutPath: log, stderrPath: log, metadata: { control_id: input.controlId } });
+                worker.process_id = record.id;
+                worker.process_lease_token = record.lease_token;
+                context.db.run("UPDATE run_control_workers SET process_id = ?, process_lease_token = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND owner_pid = ? AND status = 'running'", [record.id, record.lease_token, context.now(), input.controlId, worker.owner_token, process.pid]);
             }
-            if (observation.exhausted()) {
-                const latest = context.db.get("SELECT snapshot_json FROM run_control_workers WHERE control_id = ?", [input.controlId]);
-                let retained = {};
-                try {
-                    retained = JSON.parse(latest?.snapshot_json ?? "{}") ?? {};
-                }
-                catch { /* Retain a blocked status even when the old projection is malformed. */ }
-                const snapshot = { ...retained, status: "recovery_blocked", settled: false, pending_reasons: ["recovery_observation_budget_exhausted"],
-                    error: { code: "recovery_observation_budget_exhausted", message: "Control settlement has no confirmed result within the active observation budget" }, recovery_observation: observation.state() };
-                context.db.run("UPDATE run_control_workers SET status = 'failed', snapshot_json = ?, error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify(snapshot), JSON.stringify(snapshot.error), context.now(), input.controlId, worker.owner_token]);
-                return { ok: true, control_id: input.controlId, ...snapshot };
+            if (!worker.process_lease_token)
+                throw new AppError("run_control_owner_mismatch", "Control owner has no resource lease", 1);
+            confirmManagedProcess(resourceContext, { id: worker.process_id, leaseToken: worker.process_lease_token, pid: process.pid, processGroupId: process.pid, ownerPid: process.pid });
+            let priorObservation = null;
+            try {
+                priorObservation = worker.snapshot_json ? JSON.parse(worker.snapshot_json) : null;
             }
-            if (guard.status === "sealed") {
-                // Capture retries are local evidence work. Never dispatch another
-                // native interrupt/inspect merely because publication previously failed.
-                const settlement = JSON.parse(guard.settlement_json ?? "{}");
-                try {
-                    if (settlement.settled !== true)
-                        throw new AppError("run_control_settlement_missing", "Sealed control has no settled tree receipt", 1);
-                    const root = path.resolve(context.env.DD_FLOW_RECOVERY_HOME ?? path.join(os.homedir(), ".dd-flow-recovery"));
-                    const capture = guard.capture_path ?? path.join(root, crypto.createHash("sha256").update(context.ddFlowHome).digest("hex"), project.id, input.runId, guard.recovery_id);
-                    const scope = { projectRoot: project.root, runId: input.runId, recoveryId: guard.recovery_id };
-                    if (!fs.existsSync(capture))
-                        createEvalRunSnapshot(context, { ...scope, output: capture });
-                    if (!guard.capture_path)
-                        recordRecoveryCapture(context, { ...scope, capturePath: capture });
-                    else {
-                        const run = context.db.get("SELECT workspace_root FROM runs WHERE project_id = ? AND id = ?", [project.id, input.runId]);
-                        verifyRecoverySnapshot(capture, { ...scope, workspaceRoot: run.workspace_root });
+            catch { /* A malformed prior snapshot cannot grant a new budget. */
+                priorObservation = { recovery_observation: { remaining_ms: 0 } };
+            }
+            const observation = recoveryObservationBudget(priorObservation);
+            context.db.run("UPDATE run_control_workers SET snapshot_json = json_set(COALESCE(snapshot_json, '{}'), '$.recovery_observation', json(?)), updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify(observation.state()), context.now(), input.controlId, worker.owner_token]);
+            for (;;) {
+                if (timerFailure)
+                    throw timerFailure;
+                await lease.ensureConfirmed(assertControlCurrent);
+                const current = context.db.get("SELECT * FROM run_controls WHERE control_id = ?", [input.controlId]);
+                const guard = recoveryGuard(context, project.id, input.runId);
+                if (current.status !== "requested" || guard?.recovery_id !== intent.recovery_id || !["draining", "sealed"].includes(guard.status)) {
+                    const status = current.status === "settled" ? "completed" : "superseded";
+                    context.db.run("UPDATE run_control_workers SET status = ?, updated_at = ? WHERE control_id = ? AND owner_token = ?", [status, context.now(), input.controlId, worker.owner_token]);
+                    return { ok: true, control_id: input.controlId, status };
+                }
+                observation.observe({ control_id: input.controlId, status: guard.status, ...(guard.capture_path ? { published: true } : {}) });
+                if (observation.exhausted()) {
+                    const final = context.db.get("SELECT * FROM run_controls WHERE control_id = ?", [input.controlId]);
+                    const finalGuard = recoveryGuard(context, project.id, input.runId);
+                    if (final.status !== "requested" || finalGuard?.recovery_id !== intent.recovery_id || !["draining", "sealed"].includes(finalGuard.status)) {
+                        const status = final.status === "settled" ? "completed" : "superseded";
+                        context.db.run("UPDATE run_control_workers SET status = ?, updated_at = ? WHERE control_id = ? AND owner_token = ?", [status, context.now(), input.controlId, worker.owner_token]);
+                        return { ok: true, control_id: input.controlId, status };
+                    }
+                    const latest = context.db.get("SELECT snapshot_json FROM run_control_workers WHERE control_id = ?", [input.controlId]);
+                    let retained = {};
+                    try {
+                        retained = JSON.parse(latest?.snapshot_json ?? "{}") ?? {};
                     }
-                    const applied = context.db.run(`UPDATE run_controls SET status = 'settled', updated_at = ?
-            WHERE control_id = ? AND status = 'requested' AND recovery_id = ?
-              AND EXISTS (SELECT 1 FROM run_control_workers w WHERE w.control_id = run_controls.control_id AND w.owner_token = ? AND w.status = 'running')
-              AND EXISTS (SELECT 1 FROM run_recovery_guards g WHERE g.recovery_id = run_controls.recovery_id AND g.status = 'sealed' AND g.capture_path = ? AND g.generation = (SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = g.project_id AND run_id = g.run_id))`, [context.now(), input.controlId, guard.recovery_id, worker.owner_token, capture]);
-                    if (applied.changes !== 1)
-                        throw new AppError("run_control_state_changed", "Control changed before its capture could be acknowledged", 1);
-                    const snapshot = { ...settlement, status: "settled", capture_path: capture, recovery_observation: observation.state() };
-                    context.db.run("UPDATE run_control_workers SET status = 'completed', snapshot_json = ?, error_json = NULL, updated_at = ? WHERE control_id = ? AND owner_token = ?", [JSON.stringify(snapshot), context.now(), input.controlId, worker.owner_token]);
+                    catch { /* Retain a blocked status even when the old projection is malformed. */ }
+                    const snapshot = { ...retained, status: "recovery_blocked", settled: false, pending_reasons: ["recovery_observation_budget_exhausted"],
+                        error: { code: "recovery_observation_budget_exhausted", message: "Control settlement has no confirmed result within the active observation budget" }, recovery_observation: observation.state() };
+                    context.db.run("UPDATE run_control_workers SET status = 'failed', snapshot_json = ?, error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify(snapshot), JSON.stringify(snapshot.error), context.now(), input.controlId, worker.owner_token]);
                     return { ok: true, control_id: input.controlId, ...snapshot };
                 }
+                if (guard.status === "sealed") {
+                    // Capture retries are local evidence work. Never dispatch another
+                    // native interrupt/inspect merely because publication previously failed.
+                    const settlement = JSON.parse(guard.settlement_json ?? "{}");
+                    try {
+                        if (settlement.settled !== true)
+                            throw new AppError("run_control_settlement_missing", "Sealed control has no settled tree receipt", 1);
+                        const root = path.resolve(context.env.DD_FLOW_RECOVERY_HOME ?? path.join(os.homedir(), ".dd-flow-recovery"));
+                        const capture = guard.capture_path ?? path.join(root, crypto.createHash("sha256").update(context.ddFlowHome).digest("hex"), project.id, input.runId, guard.recovery_id);
+                        const scope = { projectRoot: project.root, runId: input.runId, recoveryId: guard.recovery_id };
+                        if (!fs.existsSync(capture))
+                            createEvalRunSnapshot(context, { ...scope, output: capture });
+                        await lease.ensureConfirmed(assertControlCurrent);
+                        if (!guard.capture_path)
+                            recordRecoveryCapture(context, { ...scope, capturePath: capture });
+                        else {
+                            const run = context.db.get("SELECT workspace_root FROM runs WHERE project_id = ? AND id = ?", [project.id, input.runId]);
+                            verifyRecoverySnapshot(capture, { ...scope, workspaceRoot: run.workspace_root });
+                        }
+                        await lease.ensureConfirmed(assertControlCurrent);
+                        const applied = context.db.run(`UPDATE run_controls SET status = 'settled', updated_at = ?
+              WHERE control_id = ? AND status = 'requested' AND recovery_id = ?
+                AND EXISTS (SELECT 1 FROM run_control_workers w WHERE w.control_id = run_controls.control_id AND w.owner_token = ? AND w.status = 'running')
+                AND EXISTS (SELECT 1 FROM run_recovery_guards g WHERE g.recovery_id = run_controls.recovery_id AND g.status = 'sealed' AND g.capture_path = ? AND g.generation = (SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = g.project_id AND run_id = g.run_id))`, [context.now(), input.controlId, guard.recovery_id, worker.owner_token, capture]);
+                        if (applied.changes !== 1)
+                            throw new AppError("run_control_state_changed", "Control changed before its capture could be acknowledged", 1);
+                        const snapshot = { ...settlement, status: "settled", capture_path: capture, recovery_observation: observation.state() };
+                        context.db.run("UPDATE run_control_workers SET status = 'completed', snapshot_json = ?, error_json = NULL, updated_at = ? WHERE control_id = ? AND owner_token = ?", [JSON.stringify(snapshot), context.now(), input.controlId, worker.owner_token]);
+                        return { ok: true, control_id: input.controlId, ...snapshot };
+                    }
+                    catch (error) {
+                        observation.exhausted();
+                        const snapshot = { ...settlement, status: "requested", capture: { status: "pending", error: failure(error) }, pending_reasons: ["recovery_capture_required"], recovery_observation: observation.state() };
+                        context.db.run("UPDATE run_control_workers SET snapshot_json = ?, error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify(snapshot), JSON.stringify(failure(error)), context.now(), input.controlId, worker.owner_token]);
+                        await delay(observation.nextDelay());
+                        continue;
+                    }
+                }
+                const request = JSON.parse(current.request_json);
+                const interrupt = current.action === "stop" || request.escalate === "interrupt" && Date.now() >= Date.parse(current.created_at) + (request.grace_ms ?? 30_000);
+                let snapshot;
+                try {
+                    snapshot = await withRecoveryObservation(observation, async () => {
+                        const cancellation = interrupt ? await interruptControlledSessions(ownedContext, input) : null;
+                        await lease.ensureConfirmed(assertControlCurrent);
+                        return { ...await observeControlledSessions(ownedContext, input), ...(cancellation ? { cancellation } : {}) };
+                    });
+                }
                 catch (error) {
-                    observation.exhausted();
-                    const snapshot = { ...settlement, status: "requested", capture: { status: "pending", error: failure(error) }, pending_reasons: ["recovery_capture_required"], recovery_observation: observation.state() };
-                    context.db.run("UPDATE run_control_workers SET snapshot_json = ?, error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify(snapshot), JSON.stringify(failure(error)), context.now(), input.controlId, worker.owner_token]);
-                    await delay(observation.nextDelay());
-                    continue;
+                    snapshot = { ok: true, control_id: input.controlId, status: "requested", settled: false, pending_reasons: ["control_observation_failed"], error: failure(error) };
+                }
+                await lease.ensureConfirmed(assertControlCurrent);
+                observation.observe(snapshot);
+                snapshot = { ...snapshot, recovery_observation: observation.state() };
+                context.db.run("UPDATE run_control_workers SET snapshot_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify(snapshot), context.now(), input.controlId, worker.owner_token]);
+                if (snapshot.settled === true) {
+                    try {
+                        sealRunRecovery(context, { projectRoot: project.root, runId: input.runId, recoveryId: guard.recovery_id, settlement: JSON.stringify(snapshot) });
+                        continue;
+                    }
+                    catch (error) {
+                        context.db.run("UPDATE run_control_workers SET snapshot_json = ?, error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify({ ...snapshot, pending_reasons: ["recovery_seal_required"], error: failure(error) }), JSON.stringify(failure(error)), context.now(), input.controlId, worker.owner_token]);
+                    }
                 }
+                await delay(observation.nextDelay());
             }
-            const request = JSON.parse(current.request_json);
-            const interrupt = current.action === "stop" || request.escalate === "interrupt" && Date.now() >= Date.parse(current.created_at) + (request.grace_ms ?? 30_000);
-            let snapshot;
+        }
+        catch (error) {
+            primaryFailure = error;
             try {
-                snapshot = await withRecoveryObservation(observation, async () => {
-                    const cancellation = interrupt ? await interruptControlledSessions(context, input) : null;
-                    return { ...await observeControlledSessions(context, input), ...(cancellation ? { cancellation } : {}) };
-                });
+                context.db.run("UPDATE run_control_workers SET status = 'failed', error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ?", [JSON.stringify(failure(error)), context.now(), input.controlId, worker.owner_token]);
             }
-            catch (error) {
-                snapshot = { ok: true, control_id: input.controlId, status: "requested", settled: false, pending_reasons: ["control_observation_failed"], error: failure(error) };
+            catch (secondary) {
+                process.stderr.write(`${JSON.stringify({ primary: failure(error), phase: "persist_control_failure", secondary: failure(secondary) })}\n`);
             }
-            observation.exhausted();
-            snapshot = { ...snapshot, recovery_observation: observation.state() };
-            context.db.run("UPDATE run_control_workers SET snapshot_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify(snapshot), context.now(), input.controlId, worker.owner_token]);
-            if (snapshot.settled === true) {
-                try {
-                    sealRunRecovery(context, { projectRoot: project.root, runId: input.runId, recoveryId: guard.recovery_id, settlement: JSON.stringify(snapshot) });
-                    continue;
-                }
-                catch (error) {
-                    context.db.run("UPDATE run_control_workers SET snapshot_json = ?, error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify({ ...snapshot, pending_reasons: ["recovery_seal_required"], error: failure(error) }), JSON.stringify(failure(error)), context.now(), input.controlId, worker.owner_token]);
-                }
+            throw error;
+        }
+        finally {
+            lease.close();
+            clearInterval(heartbeat);
+            try {
+                if (worker.process_id && worker.process_lease_token)
+                    finishManagedProcess(resourceContext, { id: worker.process_id, leaseToken: worker.process_lease_token, state: "stopped", reason: "control_owner_exited" });
+            }
+            catch (secondary) {
+                cleanupFailure = secondary;
+                if (primaryFailure)
+                    process.stderr.write(`${JSON.stringify({ primary: failure(primaryFailure), phase: "finish_control_owner", secondary: failure(secondary) })}\n`);
             }
-            await delay(observation.nextDelay());
         }
-    }
-    catch (error) {
-        context.db.run("UPDATE run_control_workers SET status = 'failed', error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ?", [JSON.stringify(failure(error)), context.now(), input.controlId, worker.owner_token]);
-        throw error;
-    }
-    finally {
-        clearInterval(heartbeat);
-        if (worker.process_id && worker.process_lease_token)
-            finishManagedProcess(resourceContext, { id: worker.process_id, leaseToken: worker.process_lease_token, state: "stopped", reason: "control_owner_exited" });
-    }
+    })();
+    if (cleanupFailure !== undefined)
+        throw cleanupFailure;
+    return result;
+}
+function failure(error) {
+    const record = errorRecord(error);
+    return { ...record, code: record.code === "unexpected" ? "run_control_failed" : record.code };
 }
-function failure(error) { return { code: error instanceof AppError ? error.code : "run_control_failed", message: error instanceof Error ? error.message : String(error) }; }