@deksden-com/dd-flow-cli 0.9.0-beta.123 → 0.9.0-beta.127

package/dist/services/run-control.js

@@ -14,6 +14,9 @@ import { finishManagedProcess, managedLeaseReleased, managedProcessBinding, mana
 import { reconcileUnfinishedChecks, terminalCheckSettled } from "./code-checks.js";
 import { controlStatusReceipt } from "./run-control-receipt.js";
 import { assertNativeCancellationReceipt, cancelControlledNativeSession, inspectControlledNativeSession } from "./native-session-control.js";
+import { terminalControllerFinalization } from "./run-controller-state.js";
+import { canonicalNativeOperation } from "../harness-runtime/lib/adapter-timeouts.mjs";
+import { requireManagedDaemonBinding } from "./managed-daemon-binding.js";
 /** Durable projection only: status never probes a provider or acquires an owner. */
 export function runControlStatus(context, input) {
     const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
@@ -26,7 +29,7 @@ function readControlStatus(context, input, project) {
     if (!run)
         throw new AppError("not_found", "RUN is not registered", 1);
     const scope = { project_id: project.id, run_id: input.runId };
-    const empty = controlStatusReceipt({ ok: true, scope, run_status: run.status, control: null, operations: [], worker: null, resume_operation: null }, input);
+    const empty = controlStatusReceipt({ ok: true, scope, run_status: run.status, finalization: terminalControllerFinalization(context, project, input.runId, run.status), control: null, operations: [], worker: null, resume_operation: null }, input);
     if (input.controlId === null)
         return empty;
     if (!context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controls'"))
@@ -79,6 +82,7 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
         throw new AppError("run_control_invalid_state", "Interrupt requires the current stop intent or its explicitly elapsed pause escalation", 1);
     const observationId = boundary?.operationId ?? control.control_id;
     const assertCurrent = () => {
+        context.assertControlOwner?.();
         if (boundary) {
             boundary.assertCurrent();
             return;
@@ -88,7 +92,7 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
             throw new AppError("run_control_superseded", "Stop intent no longer owns this RUN", 1);
     };
     assertCurrent();
-    const databaseVersion = () => JSON.stringify([context.db.get("PRAGMA data_version"), context.db.get("SELECT total_changes() AS changes")]);
+    const databaseVersion = () => runSettlementVersion(context, project.id, input.runId, boundary?.operationId);
     const initialDatabaseVersion = databaseVersion();
     const readBindings = () => context.db.all("SELECT project_id, session_id, provider_session_id, harness, parent_session_id, provider_parent_session_id, run_id, status, stop_reason, metadata_json FROM sessions");
     const foreign = (binding) => binding.project_id !== project.id || Boolean(binding.run_id && binding.run_id !== input.runId);
@@ -382,8 +386,8 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
             const args = saved.args;
             if (!Array.isArray(args) || args.some(value => typeof value !== "string") || !saved.state_dir || !saved.executable || saved.owner !== operation.controller_id || conflicts.has(saved.state_dir) || directoryAdapters.get(saved.state_dir) !== saved.executable)
                 throw new AppError("controller_operation_binding_unknown", "Unfinished controller operation has no matching frozen runtime binding", 1);
-            const method = `${args[0]}.${args[1]}`;
-            if (!["session.create", "session.prompt", "session.start", "session.fork", "session.cancel", "session.resume", "daemon.start", "daemon.stop"].includes(method))
+            const method = `${args[0]}.${args[1] === "status" ? "inspect" : args[1]}`;
+            if (!["session.create", "session.prompt", "session.start", "session.fork", "session.cancel", "session.resume", "session.inspect", "daemon.start", "daemon.stop"].includes(method))
                 throw new AppError("controller_operation_receipt_required", "Unfinished controller operation needs its original receipt", 1);
             const target = args.includes("--session-id") ? args[args.indexOf("--session-id") + 1] : null;
             if (!args.includes("--state-dir") || args[args.indexOf("--state-dir") + 1] !== saved.state_dir || args.includes("--session-id") && (!target || !targets.has(JSON.stringify([saved.state_dir, target]))))
@@ -402,6 +406,23 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
             delete env.DD_FLOW_CONTROLLER_TOKEN;
             delete env.DD_FLOW_CONTROL_TOKEN;
             assertCurrent();
+            if (canonicalNativeOperation(harness, method) === "session.inspect") {
+                if (!target)
+                    throw new AppError("controller_operation_binding_unknown", "Native reobservation has no frozen Session target", 1);
+                const physicalContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: retained.owner.resource_home ?? resourceHome(context) } };
+                const native = requireManagedDaemonBinding(physicalContext, { stateDir: saved.state_dir, projectId: project.id, runId: input.runId, ownerId: operation.controller_id });
+                const binding = managedProcessBinding(native.process, resourceHome(physicalContext));
+                const assertNative = () => {
+                    assertCurrent();
+                    const current = requireManagedDaemonBinding(physicalContext, { stateDir: saved.state_dir, projectId: project.id, runId: input.runId, ownerId: operation.controller_id });
+                    if (current.state.daemon_id !== native.state.daemon_id || JSON.stringify(managedProcessBinding(current.process, resourceHome(physicalContext))) !== JSON.stringify(binding) || harnessKey(current.process.kind.replace(/-daemon$/, "")) !== harness || !processIsAlive(current.process) || ["starting", "stopping", "clean", "unclean", "cleanup_failed", "failed"].includes(current.state.shutdown_state ?? ""))
+                        throw new AppError("daemon_owner_mismatch", "Native reobservation lost its live frozen daemon authority", 1);
+                };
+                const receipt = await inspectControlledNativeSession({ executable: saved.executable, sessionId: target, stateDir: saved.state_dir, harness, env, operationId: `${observationId}:observe:${operation.operation_id}`, assertCurrent: assertNative });
+                assertNative();
+                context.db.run("UPDATE run_controller_operations SET status = 'completed', receipt_json = ?, error_json = NULL, updated_at = ? WHERE operation_id = ? AND status IN ('dispatching','outcome_unknown')", [JSON.stringify(receipt), context.now(), operation.operation_id]);
+                return { operation_id: operation.operation_id, status: "completed", receipt, source: "native_reobservation" };
+            }
             const receipt = await runHarnessAdapter({ executable: saved.executable, args: ["daemon", "operation", "--operation-id", operation.operation_id, "--state-dir", saved.state_dir, "--json"], env, operationId: `${observationId}:observe:${operation.operation_id}`, timeoutMs: 30_000 });
             if (receipt.operation_id !== operation.operation_id || receipt.operation !== method || target && receipt.session_id !== target)
                 throw new AppError("controller_operation_mismatch", "Operation observation belongs to another retained dispatch", 1);
@@ -761,8 +782,14 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
                 if (fs.existsSync(path.join(directory, "daemon.json")) || JSON.stringify(retainedDaemonOperations(directory)) !== journal)
                     throw new AppError("owned_inventory_changed", "Rejected startup acquired new runtime evidence", 1);
             const currentProcesses = [...resourceHomes].flatMap(home => managedProcessStatus({ ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: home } }).filter(record => record.project_id === project.id && record.run_id === input.runId && [null, context.ddFlowHome].includes(managedProcessRuntimeHome(record, context))).map(record => ({ home, record })));
-            if (initialDatabaseVersion !== databaseVersion() || processVersion(registeredProcesses) !== processVersion(currentProcesses) || [...nativeVersions].some(([directory, version]) => version !== nativeInventoryVersion(directory)) || [...importVersions].some(([file, version]) => crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex") !== version))
-                throw new AppError("owned_inventory_changed", "Owned writers changed after their settlement observation", 1);
+            const changed = [
+                ...(initialDatabaseVersion !== databaseVersion() ? ["run_bindings"] : []),
+                ...(processVersion(registeredProcesses) !== processVersion(currentProcesses) ? ["process_owners"] : []),
+                ...[...nativeVersions].filter(([directory, version]) => version !== nativeInventoryVersion(directory)).map(([directory]) => `native:${directory}`),
+                ...[...importVersions].filter(([file, version]) => crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex") !== version).map(([file]) => `import:${file}`)
+            ];
+            if (changed.length)
+                throw new AppError("owned_inventory_changed", "Owned writers changed after their settlement observation", 1, { components: changed });
         };
         try {
             verifyInventory();
@@ -792,7 +819,69 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
 export function nativeInventoryVersion(directory, operations = retainedDaemonOperations(directory)) {
     const state = JSON.parse(fs.readFileSync(path.join(directory, "daemon.json"), "utf8"));
     const keys = ["schema_id", "daemon_id", "config", "pid", "provider_pid", "shutdown_state", "active_operation", "active_tree", "sessions", "descendants", "root_session_id", "resource_process", "provider_process", "owned_processes"];
-    return crypto.createHash("sha256").update(JSON.stringify([Object.fromEntries(keys.map(key => [key, state[key]])), operations])).digest("hex");
+    // Only these native verbs are read-only. Unknown verbs remain guarded.
+    const productive = operations.filter(operation => !isObservationOperation(operation.operation));
+    return crypto.createHash("sha256").update(JSON.stringify([Object.fromEntries(keys.map(key => [key, state[key]])), productive])).digest("hex");
+}
+export function isObservationOperation(operation) {
+    return ["session.inspect", "daemon.status"].includes(String(operation));
+}
+/** Compare authority and productive state of this RUN, not writes anywhere
+ * in the shared database. Heartbeat and observer timestamps are not authority. */
+export function runSettlementVersion(context, projectId, runId, observerOperationId) {
+    return context.db.readSnapshot(() => {
+        const exists = (table) => Boolean(context.db.get("SELECT name FROM sqlite_master WHERE type='table' AND name=?", [table]));
+        const rows = (table, where, values) => exists(table) ? context.db.all(`SELECT * FROM ${table} WHERE ${where}`, values) : [];
+        const scoped = [projectId, runId];
+        const sessions = rows("sessions", "1 = 1", []);
+        const identity = (row) => JSON.stringify([row.project_id, row.session_id]);
+        const selected = new Set(sessions.filter(row => row.project_id === projectId && row.run_id === runId).map(identity));
+        // Include newly registered children, even before their RUN id is attached.
+        // ponytail: retained Session inventories are small; index ancestry if scans become costly.
+        let previous = -1;
+        while (previous !== selected.size) {
+            previous = selected.size;
+            for (const row of sessions)
+                if (sessions.some(parent => selected.has(identity(parent)) && parent.harness === row.harness && (parent.project_id === row.project_id && parent.session_id === row.parent_session_id || parent.provider_session_id && parent.provider_session_id === row.provider_parent_session_id)))
+                    selected.add(identity(row));
+        }
+        // Match the protocol relationships retained by recovery-snapshot-database.
+        const protocolIds = "SELECT subject_id FROM runs WHERE project_id = ? AND id = ? UNION SELECT value FROM merge_requests, json_each(protocol_ids_json) WHERE project_id = ? AND run_id = ? UNION SELECT protocol_id FROM flow_jobs WHERE project_id = ? AND run_id = ?";
+        const protocolScope = [projectId, ...scoped, ...scoped, ...scoped];
+        const projection = {
+            runs: rows("runs", "project_id = ? AND id = ?", scoped),
+            project: rows("projects", "id = ?", [projectId]),
+            project_config: rows("project_config", "project_id = ?", [projectId]),
+            protocols: rows("protocols", `project_id = ? AND id IN (${protocolIds})`, protocolScope),
+            plan_bindings: rows("plan_bindings", `project_id = ? AND protocol_id IN (${protocolIds})`, protocolScope),
+            plan_progress: rows("plan_progress", `project_id = ? AND protocol_id IN (${protocolIds})`, protocolScope),
+            worktrees: rows("worktree_records", `project_id = ? AND protocol_id IN (${protocolIds})`, protocolScope),
+            sessions: sessions.filter(row => selected.has(identity(row))),
+            works: rows("works", "project_id = ? AND run_id = ?", scoped),
+            checks: rows("check_receipts", "project_id = ? AND run_id = ?", scoped),
+            merges: rows("merge_requests", "project_id = ? AND run_id = ?", scoped),
+            memberships: rows("stage_work_memberships", "project_id = ? AND run_id = ?", scoped),
+            repairs: rows("repair_intents", "project_id = ? AND run_id = ?", scoped),
+            lifecycle: rows("lifecycle_invocations", "json_extract(scope_json, '$.runId') = ? AND json_extract(scope_json, '$.projectRoot') = (SELECT root FROM projects WHERE id = ?)", [runId, projectId]),
+            packets: rows("lifecycle_packet_assignments", "invocation_id IN (SELECT id FROM lifecycle_invocations WHERE json_extract(scope_json, '$.runId') = ? AND json_extract(scope_json, '$.projectRoot') = (SELECT root FROM projects WHERE id = ?))", [runId, projectId]),
+            flags: rows("flow_run_flag_mutations", "project_id = ? AND run_id = ?", scoped),
+            jobs: rows("flow_jobs", "project_id = ? AND run_id = ?", scoped),
+            work_sessions: rows("work_sessions", "work_id IN (SELECT work_id FROM works WHERE project_id = ? AND run_id = ?)", scoped),
+            work_launches: rows("work_launches", "work_id IN (SELECT work_id FROM works WHERE project_id = ? AND run_id = ?)", scoped),
+            controls: rows("run_controls", "project_id = ? AND run_id = ?", scoped),
+            recovery: rows("run_recovery_guards", "project_id = ? AND run_id = ?", scoped),
+            operations: rows("run_controller_operations", "controller_id IN (SELECT controller_id FROM run_controllers WHERE project_id = ? AND run_id = ?)", scoped).filter(row => {
+                if (row.operation_id === observerOperationId)
+                    return false;
+                const input = JSON.parse(String(row.input_json));
+                const operation = input.args?.slice(0, 2).join(".");
+                // CLI session status is an alias for the native session.inspect read.
+                return operation !== "session.status" && !isObservationOperation(operation);
+            }).map(row => ({ operation_id: row.operation_id, controller_id: row.controller_id, generation: row.generation, input_hash: row.input_hash, input_json: row.input_json, status: row.status, receipt_json: row.receipt_json, error_json: row.error_json })),
+            controllers: rows("run_controllers", "project_id = ? AND run_id = ?", scoped).map(row => ({ controller_id: row.controller_id, generation: row.generation, status: row.status, owner_token: row.owner_token, process_id: row.process_id, process_lease_token: row.process_lease_token, state_dir: row.state_dir, manifest_json: row.manifest_json, sessions: JSON.parse(String(row.state_json)).sessions }))
+        };
+        return JSON.stringify(Object.fromEntries(Object.entries(projection).map(([table, records]) => [table, records.map(row => Object.fromEntries(Object.entries(row).filter(([key]) => !["created_at", "updated_at", "heartbeat_at"].includes(key)))).sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)))])));
+    });
 }
 /** Read the bundled runtime's durable operation journal, including lost replies. */
 export function retainedDaemonOperations(directory) {
@@ -858,7 +947,9 @@ export function recordRunControlIntent(context, input) {
         const run = context.db.get("SELECT status FROM runs WHERE project_id = ? AND id = ?", [project.id, input.runId]);
         if (!run)
             throw new AppError("not_found", "RUN is not registered", 1);
-        if (["done", "failed", "cancelled"].includes(run.status)) {
+        // Terminal subject state does not prove that its native tree is closed.
+        // Stop still needs the same fenced settlement/capture as a running RUN.
+        if (input.action === "pause" && ["done", "failed", "cancelled"].includes(run.status)) {
             return { ok: true, terminal: true, status: run.status, control_id: null };
         }
         const prior = recoveryGuard(context, project.id, input.runId);

package/dist/services/run-controller-adapter.js

@@ -4,7 +4,14 @@ import path from "node:path";
 import { AppError } from "../shared/errors.js";
 import { adapterSessionId, runHarnessAdapter } from "./harness-adapter.js";
 import { runtimeHarness } from "./execution-policy.js";
-import { appendEvent, digest, digestBytes, errorRecord } from "./run-controller-state.js";
+import { appendEvent, digest, digestBytes, errorRecord, requireRun } from "./run-controller-state.js";
+import { managedProcessStatus } from "./managed-processes.js";
+import { CAPACITY_POLICY, parseCapacityOperation } from "../harness-runtime/lib/codex-capacity-policy.mjs";
+import { recoveryObservationBudget } from "./recovery-observation-budget.js";
+import { canonicalNativeOperation } from "../harness-runtime/lib/adapter-timeouts.mjs";
+import { retainedDaemonReply } from "../harness-runtime/lib/daemon-observation.mjs";
+import { readOperationRecord } from "../harness-runtime/lib/operation-progress.mjs";
+import { readNativeDaemonState } from "./managed-daemon-binding.js";
 /** A paused RUN admits only its controller's exact accepted answer Turn. */
 export function controllerHitlAdmission(context, record, input) {
     if (input.operation !== "session.prompt" || !input.operationId || !input.promptSha256)
@@ -15,13 +22,13 @@ export function controllerHitlAdmission(context, record, input) {
     if (!row || row.generation !== JSON.parse(record.metadata_json).run_generation)
         return false;
     const suffix = `:prompt:${row.generation}`;
-    const successor = /^(.*):capacity:([12]):session\.prompt$/.exec(input.operationId);
-    const baseId = successor ? successor[1] : input.operationId;
+    const successor = parseCapacityOperation(input.operationId);
+    const baseId = successor.root;
     if (!baseId.endsWith(suffix))
         return false;
-    if (successor) {
-        const ordinal = Number(successor[2]);
-        const predecessorId = ordinal === 1 ? baseId : `${baseId}:capacity:1:session.prompt`;
+    if (successor.ordinal > 0) {
+        const ordinal = successor.ordinal;
+        const predecessorId = ordinal === 1 ? baseId : `${baseId}:capacity:${ordinal - 1}:session.prompt`;
         const predecessor = context.db.get("SELECT status, receipt_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [predecessorId, row.controller_id, row.generation]);
         if (predecessor?.status !== "failed" || !predecessor.receipt_json)
             return false;
@@ -34,6 +41,8 @@ export function controllerHitlAdmission(context, record, input) {
         }
         if (retained.capacity_intent?.predecessor !== predecessorId || retained.capacity_intent.operation_id !== input.operationId || retained.capacity_intent.ordinal !== ordinal)
             return false;
+        if (retained.capacity_intent.policy !== CAPACITY_POLICY)
+            return false;
         if (!retained.capacity_intent.not_before || !Number.isFinite(Date.parse(retained.capacity_intent.not_before)) || Date.parse(retained.capacity_intent.not_before) > Date.parse(context.now()) || retained.capacity_intent.prompt_sha256 !== input.promptSha256)
             return false;
     }
@@ -47,7 +56,10 @@ export function controllerHitlAdmission(context, record, input) {
     if (!answer || !prompt)
         return false;
     const accepted = JSON.parse(answer.input_json);
-    const dispatch = JSON.parse(prompt.input_json);
+    // The phase is a mutable journal projection, not part of the frozen input.
+    const { dispatch_phase, ...dispatch } = JSON.parse(prompt.input_json);
+    if (dispatch_phase !== "external_dispatch_possible")
+        return false;
     const run = context.db.get("SELECT index_json FROM runs WHERE project_id = ? AND id = ?", [record.project_id, record.run_id]);
     const stage = run && JSON.parse(run.index_json).stage_runs.find(item => item.stage === accepted.stage);
     if (accepted.action !== "user_answer" || state.stage !== accepted.stage || stage?.status !== "paused" || !stage.pause || stage.pause.answer_path || stage.pause.id !== accepted.pause_id || stage.pause.work_id !== accepted.work_id)
@@ -99,8 +111,47 @@ function operationInspectArgs(args, operationId) {
         throw new AppError("controller_operation_binding_unknown", "Harness operation has no state directory", 1, { operation_id: operationId });
     return ["daemon", "operation", "--state-dir", stateDir, "--operation-id", operationId, "--json"];
 }
-function expectedMethod(args) {
-    return `${args[0]}.${args[1] === "status" ? "inspect" : args[1]}`;
+function expectedMethod(args, session) {
+    return canonicalNativeOperation(runtimeHarness(session.profile), `${args[0]}.${args[1] === "status" ? "inspect" : args[1]}`);
+}
+function recoveryAuthorityError(error) {
+    return ["session_identity_mismatch", "profile_integrity_violation", "controller_operation_mismatch", "operation_conflict", "operation_cancelled", "settlement_recovery_required"].includes(String(error?.code));
+}
+/** A retained native effect is not permission to continue under a new daemon. */
+async function recoveredSessionReceipt(context, session, args, operationId, receipt, binding = {}, signal, inspection) {
+    const method = expectedMethod(args, session);
+    if (!["session.create", "session.prompt", "session.start", "session.fork", "session.resume"].includes(method))
+        return receipt;
+    const requested = await readOperationRecord(session.state_dir, operationId, "requested.json");
+    signal?.throwIfAborted();
+    const daemon = fs.existsSync(path.join(session.state_dir, "daemon.json")) ? readNativeDaemonState(session.state_dir) : null;
+    // Old minimal adapters have neither owned operation records nor daemon
+    // ownership metadata. Missing a modern record cannot downgrade to that path.
+    const modern = requested?.owner_id || requested && Object.hasOwn(requested, "params_sha256") || requested?.generation != null || daemon?.config?.runtime_owner?.owner_id
+        || managedProcessStatus({ ...context, env: { ...context.env, ...(session.owner.resource_home ? { DD_FLOW_RESOURCE_HOME: session.owner.resource_home } : {}) } }).some(record => {
+            const metadata = JSON.parse(record.metadata_json);
+            return record.kind.endsWith("-daemon") && record.owner_id === session.owner.owner_id && record.project_id === session.owner.project_id && record.run_id === session.owner.run_id
+                && metadata.state_dir === session.state_dir && metadata.dd_flow_home === session.owner.dd_flow_home && metadata.adapter_executable === session.owner.adapter_executable;
+        });
+    if (!modern)
+        return receipt;
+    const target = args.includes("--session-id") ? args[args.indexOf("--session-id") + 1] : undefined;
+    const retained = await retainedDaemonReply(session.state_dir, operationId, { ...binding, operation: method, owner_id: session.owner.owner_id, ...(target ? { session_id: target } : {}) });
+    signal?.throwIfAborted();
+    if (!retained || retained.reason)
+        throw new AppError("settlement_recovery_required", "Retained native outcome has no current dispatch authority", 1, { operation_id: operationId, native_outcome: "completed", native_result: retained?.outcome ?? receipt, settlement: retained?.settlement, reason: retained?.reason ?? "native_receipt_unavailable" });
+    if (inspection && settlement(inspection)?.state === "settled" && !retained.ready) {
+        const proof = inspection.settlement;
+        if (proof.source !== "inspection" || !Array.isArray(proof.operation_ids) || !proof.operation_ids.includes(operationId))
+            throw new AppError("settlement_recovery_required", "Fresh inspection does not settle the original retained operation", 1, { operation_id: operationId, native_outcome: "completed", native_result: retained.outcome, reason: "inspection_settlement_unbound", settlement: proof });
+    }
+    const result = retained.ready ? retained.value : retained.outcome;
+    if (!result || typeof result !== "object" || Array.isArray(result))
+        throw new AppError("controller_operation_mismatch", "Retained native outcome has no Session result", 1, { operation_id: operationId });
+    validateReceipt(result, session, args);
+    if (method === "session.create" && !adapterSessionId(result))
+        throw new AppError("session_identity_mismatch", "Retained creation outcome has no native Session identity", 1);
+    return { ...receipt, ...result, settlement: retained.ready ? retained.settlement ?? { state: "not_required" } : ["pending", "blocked"].includes(String(retained.settlement?.state)) ? retained.settlement : { ...retained.settlement, state: "pending", reason: retained.settlement?.reason ?? "settlement_evidence_required" } };
 }
 function validateReceipt(receipt, session, args) {
     // `daemon start --session-id` names the retained state being reopened; the
@@ -119,11 +170,9 @@ function validateReceipt(receipt, session, args) {
     }
 }
 async function recoverCompletedOperation(context, manifest, session, executable, args, operationId, signal) {
-    // Creation has no stable client Session id yet. Its durable daemon journal
-    // is reconciled by run-control after the owner is fenced; probing it from the
-    // failing transport path would be a second adapter invocation and can itself
-    // be mistaken for replay.
-    if (args[0] === "session" && ["create", "inspect", "status"].includes(args[1]))
+    // Creation has no Session ID yet, but its original operation ID is already
+    // retained. Inspect that receipt, never invoke native creation a second time.
+    if (expectedMethod(args, session) === "session.inspect")
         return null;
     // A detached adapter without a native daemon has no durable operation
     // journal to inspect. Avoid a second adapter invocation in that case; the
@@ -139,8 +188,8 @@ async function recoverCompletedOperation(context, manifest, session, executable,
     }
     const target = args.includes("--session-id") ? args[args.indexOf("--session-id") + 1] : null;
     const result = observed.result;
-    const primary = observed.primary_error;
-    if (observed.operation_id === operationId && observed.operation === expectedMethod(args)
+    const primary = observed.primary_error ?? (["failed", "interrupted"].includes(String(observed.state)) ? observed.error : undefined);
+    if (observed.operation_id === operationId && observed.operation === expectedMethod(args, session)
         && observed.state !== "completed" && (!target || observed.session_id === target)
         && primary && typeof primary === "object" && !Array.isArray(primary)) {
         const failure = primary;
@@ -148,69 +197,105 @@ async function recoverCompletedOperation(context, manifest, session, executable,
             throw new AppError(failure.code, failure.message, 1, { ...(failure.details && typeof failure.details === "object" && !Array.isArray(failure.details) ? failure.details : {}), operation_id: operationId, effect: "unknown" });
         }
     }
-    if (observed.operation_id !== operationId || observed.operation !== expectedMethod(args) || observed.state !== "completed"
+    if (observed.operation_id !== operationId || observed.operation !== expectedMethod(args, session) || observed.state !== "completed"
         || !result || typeof result !== "object" || Array.isArray(result)
         || target && (observed.session_id !== target || adapterSessionId(result) !== target))
         return null;
-    return { ...result, ...(settlement(observed) ? { settlement: settlement(observed) } : {}) };
+    return await recoveredSessionReceipt(context, session, args, operationId, { ...result, ...(settlement(observed) ? { settlement: settlement(observed) } : {}) }, Object.fromEntries(["generation", "params_sha256"].filter(key => observed[key] !== undefined).map(key => [key, observed[key]])), signal);
 }
 async function awaitSettlement(context, row, manifest, session, executable, args, operationId, initial, assertOwner, signal) {
     let current = initial, delay = 250;
-    const deadline = Date.now() + 120_000;
+    const observation = recoveryObservationBudget(initial);
+    observation.observe(initial);
+    const retainObservation = () => {
+        assertOwner?.();
+        const receipt = { ...initial, ...current, recovery_observation: observation.state() };
+        const changed = context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [JSON.stringify(receipt), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
+        if (changed.changes !== 1)
+            throw new AppError("controller_owner_mismatch", "Controller lost ownership before settlement observation retention", 1);
+    };
     while (["pending", "blocked"].includes(String(settlement(current)?.state))) {
         if (signal?.aborted)
             throw new AppError("harness_settlement_interrupted", "Harness settlement was interrupted", 1, { operation_id: operationId });
         // A completed parent with live children is normal controller waiting, not
         // failed bookkeeping. Keep its budget reserved until a later inspection.
-        if (args[0] === "session" && ["prompt", "inspect", "status"].includes(args[1]) && settlement(current)?.reason === "tree_unsettled")
+        if (["session.prompt", "session.inspect"].includes(expectedMethod(args, session)) && settlement(current)?.reason === "tree_unsettled")
             return { ...initial, ...current, ...(initial.assistant_text !== undefined ? { assistant_text: initial.assistant_text } : {}) };
-        if (Date.now() >= deadline)
-            throw new AppError("settlement_recovery_required", "Completed provider operation requires settlement recovery", 1, { operation_id: operationId, settlement: settlement(current) });
-        await new Promise(resolve => setTimeout(resolve, Math.min(delay, Math.max(0, deadline - Date.now()))));
+        const exhausted = observation.exhausted();
+        retainObservation();
+        if (exhausted) {
+            // Final bounded read is observation, never another settlement mutation.
+            if (args.includes("--session-id")) {
+                assertOwner?.();
+                try {
+                    const terminal = await runHarnessAdapter({ executable, args: settlementInspectArgs(args), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement-final-inspect`, runtimeOwner: session.owner, timeoutMs: 30_000, ...(signal ? { signal } : {}) });
+                    validateReceipt(terminal, session, args);
+                    if (settlement(terminal)?.state === "settled") {
+                        await recoveredSessionReceipt(context, session, args, operationId, initial, {}, signal, terminal);
+                        assertOwner?.();
+                        current = terminal;
+                        break;
+                    }
+                }
+                catch (error) {
+                    if (recoveryAuthorityError(error))
+                        throw error;
+                }
+            }
+            throw new AppError("settlement_recovery_required", "Completed provider operation requires settlement recovery", 1, { operation_id: operationId, settlement: settlement(current), recovery_observation: observation.state() });
+        }
+        await new Promise(resolve => setTimeout(resolve, Math.min(delay, observation.timeout())));
         delay = Math.min(delay * 2, 8_000);
         assertOwner?.();
-        if (Date.now() >= deadline)
+        if (observation.exhausted())
             continue;
         try {
             // This only retries durable bookkeeping against the same operation
             // receipt.  It cannot replay the provider request.
-            const inspect = args[0] === "session" && ["inspect", "status"].includes(args[1]);
-            current = await runHarnessAdapter({ executable, args: inspect ? settlementInspectArgs(args) : settlementRetryArgs(args, operationId), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement`, runtimeOwner: session.owner, timeoutMs: Math.min(30_000, Math.max(1, deadline - Date.now())), ...(signal ? { signal } : {}) });
+            const inspect = expectedMethod(args, session) === "session.inspect";
+            current = await runHarnessAdapter({ executable, args: inspect ? settlementInspectArgs(args) : settlementRetryArgs(args, operationId), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement`, runtimeOwner: session.owner, timeoutMs: observation.timeout(), ...(signal ? { signal } : {}) });
             if (inspect)
                 validateReceipt(current, session, args);
+            else
+                current = await recoveredSessionReceipt(context, session, args, operationId, current, Object.fromEntries(["generation", "params_sha256"].filter(key => current[key] !== undefined).map(key => [key, current[key]])), signal);
         }
         catch (error) {
-            if (error instanceof AppError && ["session_identity_mismatch", "profile_integrity_violation"].includes(error.code))
+            if (recoveryAuthorityError(error))
                 throw error;
             current = { settlement: { state: "pending", reason: "inspection_error", error: errorRecord(error) } };
         }
+        observation.observe(current);
         if (!(["pending", "blocked"].includes(String(settlement(current)?.state))))
             continue;
         // A receipt without final tree proof needs a fresh native inspection.  Do
         // it only after the receipt-only retry, and only when this is a session.
-        if (!args.includes("--session-id") || !["tree_unsettled", "settlement_evidence_required"].includes(String(settlement(current)?.reason)) || Date.now() >= deadline)
+        if (!args.includes("--session-id") || !["tree_unsettled", "settlement_evidence_required"].includes(String(settlement(current)?.reason)) || observation.exhausted())
             continue;
         try {
-            current = await runHarnessAdapter({ executable, args: settlementInspectArgs(args), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement-inspect`, runtimeOwner: session.owner, timeoutMs: Math.min(30_000, Math.max(1, deadline - Date.now())), ...(signal ? { signal } : {}) });
+            current = await runHarnessAdapter({ executable, args: settlementInspectArgs(args), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement-inspect`, runtimeOwner: session.owner, timeoutMs: observation.timeout(), ...(signal ? { signal } : {}) });
             validateReceipt(current, session, args);
+            await recoveredSessionReceipt(context, session, args, operationId, initial, {}, signal, current);
         }
         catch (error) {
-            if (error instanceof AppError && ["session_identity_mismatch", "profile_integrity_violation"].includes(error.code))
+            if (recoveryAuthorityError(error))
                 throw error;
             current = { settlement: { state: "pending", reason: "inspection_error", error: errorRecord(error) } };
         }
+        observation.observe(current);
     }
     return adapterSessionId(current) ? { ...initial, ...current, ...(initial.assistant_text !== undefined ? { assistant_text: initial.assistant_text } : {}) } : { ...initial, settlement: settlement(current) };
 }
-export async function controllerAdapter(context, row, manifest, session, args, assertOwner, retainedOperationId, signal) {
-    assertOwner?.();
-    const operationId = retainedOperationId ?? `${row.controller_id}:${crypto.randomUUID()}`;
+export async function controllerAdapter(context, row, manifest, session, args, assertOwner, retainedOperationId, signal, reconfirmOwner) {
     const executable = manifest.adapters[runtimeHarness(session.profile)].executable;
     const promptFile = args.indexOf("--prompt-file"), inlinePrompt = args.indexOf("--prompt");
     const promptSha256 = promptFile >= 0 ? digestBytes(fs.readFileSync(args[promptFile + 1])) : inlinePrompt >= 0 ? digestBytes(args[inlinePrompt + 1] ?? "") : undefined;
     const input = { executable, args, owner: session.owner.owner_id, state_dir: session.state_dir, ...(args[0] === "session" && args[1] === "prompt" && promptSha256 ? { prompt_sha256: promptSha256 } : {}) };
+    const prepared = retainedOperationId ? [] : context.db.all("SELECT operation_id FROM run_controller_operations WHERE controller_id = ? AND generation = ? AND input_hash = ? AND status = 'dispatching' AND json_extract(input_json, '$.dispatch_phase') = 'prepared'", [row.controller_id, row.generation, digest(input)]);
+    if (prepared.length > 1)
+        throw new AppError("controller_operation_conflict", "Multiple prepared intents require explicit reconciliation", 1);
+    const operationId = retainedOperationId ?? prepared[0]?.operation_id ?? `${row.controller_id}:${crypto.randomUUID()}`;
     const now = context.now();
-    const existing = context.db.get("SELECT input_hash,status,receipt_json,error_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [operationId, row.controller_id, row.generation]);
+    const existing = context.db.get("SELECT input_hash,input_json,status,receipt_json,error_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [operationId, row.controller_id, row.generation]);
     if (existing?.input_hash && existing.input_hash !== digest(input))
         throw new AppError("controller_operation_conflict", "Retained operation has different dispatch parameters", 1, { operation_id: operationId });
     if (existing?.status === "completed" && existing.receipt_json) {
@@ -219,29 +304,63 @@ export async function controllerAdapter(context, row, manifest, session, args, a
         return receipt;
     }
     if (!existing) {
-        context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'dispatching', ?, ?)", [operationId, row.controller_id, row.generation, digest(input), JSON.stringify(input), now, now]);
+        context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, created_at, updated_at) SELECT ?, ?, ?, ?, ?, 'dispatching', ?, ? WHERE EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [operationId, row.controller_id, row.generation, digest(input), JSON.stringify({ ...input, dispatch_phase: "prepared" }), now, now, row.controller_id, row.owner_token, row.generation]);
         appendEvent(context, row.controller_id, "operation_requested", { operation_id: operationId, operation: args.slice(0, 2), session_id: session.id || null });
     }
+    let phase = existing ? JSON.parse(existing.input_json).dispatch_phase : "prepared";
+    const retainPhase = (next) => {
+        const changed = context.db.run("UPDATE run_controller_operations SET input_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status <> 'completed' AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [JSON.stringify({ ...input, dispatch_phase: next }), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
+        if (changed.changes !== 1)
+            throw new AppError("controller_owner_mismatch", "Controller lost ownership before durable dispatch transition", 1, { operation_id: operationId });
+        phase = next;
+    };
     try {
+        await reconfirmOwner?.();
         assertOwner?.();
         let receipt;
         let recoveredOperation = false;
         try {
-            const timeoutMs = args[0] === "session" && args[1] === "prompt" ? null
-                : args[0] === "session" && args[1] === "create" && runtimeHarness(session.profile) === "zcode-acp" ? 180_000 : 120_000;
-            if (existing) {
-                const observed = await recoverCompletedOperation(context, manifest, session, executable, args, operationId, signal);
-                if (!observed) {
-                    const cause = existing.error_json ? JSON.parse(existing.error_json) : { code: "controller_operation_outcome_unknown", message: "Retained dispatch has no observed terminal result", details: { operation_id: operationId } };
-                    throw new AppError(cause.code, cause.message, 1, cause.details);
-                }
-                receipt = observed;
+            const timeoutMs = 120_000;
+            if (existing && phase === "receipt_observed" && existing.receipt_json) {
+                // The exact input hash and committed receipt survive a controller
+                // crash. Complete settlement without replaying productive native IO.
+                receipt = JSON.parse(existing.receipt_json);
+                validateReceipt(receipt, session, args);
+                if (args[0] === "session" && args[1] === "create" && !adapterSessionId(receipt))
+                    throw new AppError("session_identity_mismatch", "Retained creation receipt has no native Session identity", 1);
+                receipt = await recoveredSessionReceipt(context, session, args, operationId, receipt, {}, signal);
                 recoveredOperation = true;
             }
-            else
-                receipt = await runHarnessAdapter({ executable, args, env: controllerEnvironment(context, manifest, session), operationId, runtimeOwner: session.owner, timeoutMs, ...(signal ? { signal } : {}) });
+            else if (existing && phase !== "prepared") {
+                if (expectedMethod(args, session) === "session.inspect") {
+                    // A failed read may be explicitly reobserved. Never do this for a
+                    // materializing resume, prompt, create or fork.
+                    retainPhase("external_dispatch_possible");
+                    receipt = await runHarnessAdapter({ executable, args: settlementInspectArgs(args), harness: runtimeHarness(session.profile), env: controllerEnvironment(context, manifest, session), operationId, runtimeOwner: session.owner, timeoutMs, ...(signal ? { signal } : {}) });
+                }
+                else {
+                    const observed = await recoverCompletedOperation(context, manifest, session, executable, args, operationId, signal);
+                    if (!observed) {
+                        const cause = existing.error_json ? JSON.parse(existing.error_json) : { code: "controller_operation_outcome_unknown", message: "Retained dispatch has no observed terminal result", details: { operation_id: operationId } };
+                        throw new AppError(cause.code, cause.message, 1, cause.details);
+                    }
+                    receipt = observed;
+                    recoveredOperation = true;
+                }
+            }
+            else {
+                // Keep observation/settlement of retained operations available after
+                // terminal RUN, but never admit fresh native productive dispatch.
+                const productive = ["daemon.start", "session.create", "session.start", "session.fork", "session.prompt", "session.resume"].includes(expectedMethod(args, session));
+                if (productive && ["done", "failed", "cancelled"].includes(requireRun(context, row.project_id, row.run_id).status))
+                    throw new AppError("invalid_run_state", "A terminal RUN cannot admit a new native controller operation", 1);
+                retainPhase("external_dispatch_possible");
+                receipt = await runHarnessAdapter({ executable, args, harness: runtimeHarness(session.profile), env: controllerEnvironment(context, manifest, session), operationId, runtimeOwner: session.owner, timeoutMs, ...(signal ? { signal } : {}) });
+            }
         }
         catch (error) {
+            if (phase === "prepared" || phase === "receipt_observed" && existing?.receipt_json)
+                throw error;
             const retained = await recoverCompletedOperation(context, manifest, session, executable, args, operationId, signal);
             if (!retained)
                 throw error;
@@ -249,21 +368,28 @@ export async function controllerAdapter(context, row, manifest, session, args, a
             recoveredOperation = true;
         }
         validateReceipt(receipt, session, args);
-        context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(receipt), context.now(), operationId]);
+        retainPhase("receipt_observed");
+        const persistReceipt = () => {
+            const changed = context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [JSON.stringify(receipt), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
+            if (changed.changes !== 1)
+                throw new AppError("controller_owner_mismatch", "Controller lost ownership before receipt retention", 1);
+        };
+        persistReceipt();
         if (["pending", "blocked"].includes(String(settlement(receipt)?.state))) {
             const recovered = await awaitSettlement(context, row, manifest, session, executable, args, operationId, receipt, assertOwner, signal);
             receipt = recovered;
         }
-        context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(receipt), context.now(), operationId]);
+        persistReceipt();
         validateReceipt(receipt, session, args);
-        context.db.run("UPDATE run_controller_operations SET status = 'completed', updated_at = ? WHERE operation_id = ?", [context.now(), operationId]);
-        assertOwner?.();
+        const completed = context.db.run("UPDATE run_controller_operations SET status = 'completed', updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
+        if (completed.changes !== 1)
+            throw new AppError("controller_owner_mismatch", "Controller lost ownership before operation completion", 1);
         appendEvent(context, row.controller_id, recoveredOperation ? "operation_recovered" : "operation_completed", { operation_id: operationId, operation: args.slice(0, 2), session_id: adapterSessionId(receipt) });
         return receipt;
     }
     catch (error) {
         try {
-            context.db.run("UPDATE run_controller_operations SET status = CASE WHEN status = 'dispatching' THEN 'outcome_unknown' ELSE status END, error_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(errorRecord(error)), context.now(), operationId]);
+            context.db.run("UPDATE run_controller_operations SET status = CASE WHEN status = 'dispatching' AND ? <> 'prepared' THEN 'outcome_unknown' ELSE status END, error_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [phase ?? "legacy_unknown", JSON.stringify(errorRecord(error)), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
         }
         catch (persistence) {
             process.stderr.write(`${JSON.stringify({ operation_id: operationId, primary: errorRecord(error), persistence_error: errorRecord(persistence) })}\n`);

package/dist/services/run-controller-capture.js

@@ -83,7 +83,8 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
             throw new AppError("controller_boundary_changed", "Stage identity changed after its capture fence", 1);
         if (physical) {
             const continuation = getFlowRunStatus(context, scope).continuation;
-            if (nextStage ? continuation.kind !== "start_stage" || continuation.stage !== nextStage : continuation.kind !== "terminal")
+            const stoppedAtTarget = nextStage === null && manifest.request.stop_after === stage.stage && continuation.kind === "start_stage";
+            if (nextStage ? continuation.kind !== "start_stage" || continuation.stage !== nextStage : continuation.kind !== "terminal" && !stoppedAtTarget)
                 throw new AppError("controller_boundary_changed", "Stage successor changed after its capture fence", 1);
         }
     };
@@ -105,11 +106,22 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
             let lastPending = "";
             let lastCaptureMismatch = null;
             for (;;) {
+                assertCurrent();
+                if (fs.existsSync(output)) {
+                    receipt = verifyControllerBoundarySnapshot(output, expected);
+                    break;
+                }
                 if (observationBudget.exhausted()) {
+                    assertCurrent();
+                    if (fs.existsSync(output)) {
+                        receipt = verifyControllerBoundarySnapshot(output, expected);
+                        break;
+                    }
                     context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)), '$.last_capture_mismatch', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), JSON.stringify(lastCaptureMismatch), context.now(), operationId]);
                     throw new AppError("recovery_observation_budget_exhausted", "Boundary capture has no stable settlement within the active observation budget", 1, { recovery_observation: observationBudget.state(), last_capture_mismatch: lastCaptureMismatch });
                 }
                 assertCurrent();
+                context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), context.now(), operationId]);
                 let verify = () => { throw new AppError("controller_boundary_unsettled", "Boundary has no stable writer observation", 1); };
                 const observation = await withRecoveryObservation(observationBudget, () => observeControllerBoundary(context, { ...scope, controllerId: row.controller_id, operationId, assertCurrent, retainVerification: value => { verify = value; } }));
                 if (observation.settled === true) {
@@ -126,7 +138,7 @@ export async function captureControllerBoundary(context, row, manifest, stage, n
                         lastCaptureMismatch = { code: error.code, components: error.details?.components ?? [], changes: error.details?.changes ?? [] };
                     }
                 }
-                observationBudget.exhausted();
+                observationBudget.observe(observation);
                 const published = { ...observation, recovery_observation: observationBudget.state(), last_capture_mismatch: lastCaptureMismatch };
                 const pending = digest(observation.pending_reasons);
                 context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(published), context.now(), operationId]);