@deksden-com/dd-flow-cli 0.9.0-beta.16 → 0.9.0-beta.160

package/dist/services/run-controller-adapter.js

@@ -0,0 +1,434 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { AppError } from "../shared/errors.js";
+import { adapterSessionId, runHarnessAdapter } from "./harness-adapter.js";
+import { runtimeHarness } from "./execution-policy.js";
+import { appendEvent, digest, digestBytes, errorRecord, requireRun } from "./run-controller-state.js";
+import { managedProcessStatus } from "./managed-processes.js";
+import { RECOVERY_POLICY as CAPACITY_POLICY, parseCapacityOperation } from "../harness-runtime/lib/provider-overload-policy.mjs";
+import { recoveryObservationBudget } from "./recovery-observation-budget.js";
+import { canonicalNativeOperation } from "../harness-runtime/lib/adapter-timeouts.mjs";
+import { retainedDaemonReply } from "../harness-runtime/lib/daemon-observation.mjs";
+import { readOperationRecord } from "../harness-runtime/lib/operation-progress.mjs";
+import { readNativeDaemonState } from "./managed-daemon-binding.js";
+import { validateCheckEnvironment } from "./check-environment.js";
+import { controllerObligationAccepted, lateNativeFailure } from "./controller-result-authority.js";
+import { recordTechnicalNote } from "./technical-notes.js";
+/** A paused RUN admits only its controller's exact accepted answer Turn. */
+export function controllerHitlAdmission(context, record, input) {
+    if (input.operation !== "session.prompt" || !input.operationId || !input.promptSha256)
+        return false;
+    if (!context.db.get("SELECT 1 FROM sqlite_schema WHERE name = 'run_controller_operations'"))
+        return false;
+    const row = context.db.get("SELECT * FROM run_controllers WHERE controller_id = ? AND project_id = ? AND run_id = ? AND status = 'waiting_for_user'", [record.owner_id, record.project_id, record.run_id]);
+    if (!row || row.generation !== JSON.parse(record.metadata_json).run_generation)
+        return false;
+    const suffix = `:prompt:${row.generation}`;
+    const successor = parseCapacityOperation(input.operationId);
+    const baseId = successor.root;
+    if (!baseId.endsWith(suffix))
+        return false;
+    if (successor.ordinal > 0) {
+        const ordinal = successor.ordinal;
+        const predecessorId = ordinal === 1 ? baseId : `${baseId}:capacity:${ordinal - 1}:session.prompt`;
+        const predecessor = context.db.get("SELECT status, receipt_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [predecessorId, row.controller_id, row.generation]);
+        if (predecessor?.status !== "failed" || !predecessor.receipt_json)
+            return false;
+        let retained;
+        try {
+            retained = JSON.parse(predecessor.receipt_json);
+        }
+        catch {
+            return false;
+        }
+        if (retained.capacity_intent?.predecessor !== predecessorId || retained.capacity_intent.operation_id !== input.operationId || retained.capacity_intent.ordinal !== ordinal)
+            return false;
+        if (retained.capacity_intent.policy !== CAPACITY_POLICY)
+            return false;
+        if (!retained.capacity_intent.not_before || !Number.isFinite(Date.parse(retained.capacity_intent.not_before)) || Date.parse(retained.capacity_intent.not_before) > Date.parse(context.now()) || retained.capacity_intent.prompt_sha256 !== input.promptSha256)
+            return false;
+    }
+    const state = JSON.parse(row.state_json);
+    const session = state.current_session === null ? undefined : state.sessions[state.current_session];
+    if (!session || session.stopped || session.id !== input.nativeSessionId || session.state_dir !== input.stateDir || session.owner.owner_id !== record.owner_id)
+        return false;
+    const operation = (id) => context.db.get("SELECT input_json, input_hash FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'dispatching'", [id, row.controller_id, row.generation]);
+    const answer = context.db.get("SELECT input_json, input_hash FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation <= ? AND status = 'accepted'", [baseId.slice(0, -suffix.length), row.controller_id, row.generation]);
+    const prompt = operation(input.operationId);
+    if (!answer || !prompt)
+        return false;
+    const accepted = JSON.parse(answer.input_json);
+    // The phase is a mutable journal projection, not part of the frozen input.
+    const { dispatch_phase, ...dispatch } = JSON.parse(prompt.input_json);
+    if (dispatch_phase !== "external_dispatch_possible")
+        return false;
+    const run = context.db.get("SELECT index_json FROM runs WHERE project_id = ? AND id = ?", [record.project_id, record.run_id]);
+    const stage = run && JSON.parse(run.index_json).stage_runs.find(item => item.stage === accepted.stage);
+    if (accepted.action !== "user_answer" || state.stage !== accepted.stage || stage?.status !== "paused" || !stage.pause || stage.pause.answer_path || stage.pause.id !== accepted.pause_id || stage.pause.work_id !== accepted.work_id)
+        return false;
+    if (answer.input_hash !== digest({ pause_id: accepted.pause_id, answer_sha256: accepted.sha256 }) || prompt.input_hash !== digest(dispatch))
+        return false;
+    const value = (option) => dispatch.args.includes(option) ? dispatch.args[dispatch.args.indexOf(option) + 1] : undefined;
+    if (dispatch.owner !== record.owner_id || dispatch.state_dir !== input.stateDir || dispatch.args[0] !== "session" || dispatch.args[1] !== "prompt" || value("--session-id") !== input.nativeSessionId || dispatch.prompt_sha256 !== input.promptSha256 || !value("--prompt-file"))
+        return false;
+    return digestBytes(fs.readFileSync(accepted.file)) === accepted.sha256 && digestBytes(fs.readFileSync(value("--prompt-file"))) === input.promptSha256;
+}
+export function sessionArguments(context, manifest, session) {
+    const profile = session.profile;
+    return ["--dd-flow-bin", manifest.flow_executable, "--dd-flow-home", context.ddFlowHome, "--project-root", manifest.request.project_root, "--state-dir", session.state_dir, "--journal", session.journal, "--cwd", session.cwd, "--provider", profile.provider, "--model", profile.model, "--reasoning", profile.reasoning, "--mode", profile.mode, "--permission", profile.permission, ...manifest.adapters[runtimeHarness(profile)].runtime_args, "--json"];
+}
+export function controllerEnvironment(context, manifest, session) {
+    const env = { ...context.env, ...validateCheckEnvironment(manifest.check_environment), DD_FLOW_HOME: context.ddFlowHome, DD_FLOW_BIN: manifest.flow_executable, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_RUNTIME_BUDGET: manifest.budget ? JSON.stringify(manifest.budget) : undefined };
+    for (const key of ["DD_FLOW_CONTROLLER_TOKEN", "DD_FLOW_CONTROL_TOKEN", "DD_FLOW_DAEMON_ID", "DD_FLOW_RUNTIME_OWNER", "DD_FLOW_WORK_ID", "DD_FLOW_SESSION_ID", "CODEX_THREAD_ID"])
+        delete env[key];
+    if (session?.codex_home)
+        env.CODEX_HOME = session.codex_home;
+    return env;
+}
+function settlement(value) {
+    const state = value.settlement;
+    return state && typeof state === "object" && !Array.isArray(state) ? state : null;
+}
+function settlementInspectArgs(args) {
+    const values = ["session", "inspect"];
+    for (let index = 2; index < args.length; index += 1) {
+        const value = args[index];
+        if (["--prompt", "--prompt-file", "--answers-file", "--target-json"].includes(value)) {
+            index += 1;
+            continue;
+        }
+        values.push(value);
+    }
+    return values;
+}
+function settlementRetryArgs(args, operationId) {
+    const stateDir = args[args.indexOf("--state-dir") + 1];
+    if (!stateDir)
+        throw new AppError("settlement_recovery_required", "Harness settlement has no state directory", 1, { operation_id: operationId });
+    return ["daemon", "operation", "settle", "--state-dir", stateDir, "--operation-id", operationId, "--json"];
+}
+function operationInspectArgs(args, operationId) {
+    const stateDir = args[args.indexOf("--state-dir") + 1];
+    if (!stateDir)
+        throw new AppError("controller_operation_binding_unknown", "Harness operation has no state directory", 1, { operation_id: operationId });
+    return ["daemon", "operation", "--state-dir", stateDir, "--operation-id", operationId, "--json"];
+}
+function expectedMethod(args, session) {
+    return canonicalNativeOperation(runtimeHarness(session.profile), `${args[0]}.${args[1] === "status" ? "inspect" : args[1]}`);
+}
+function recoveryAuthorityError(error) {
+    return ["session_identity_mismatch", "profile_integrity_violation", "controller_operation_mismatch", "operation_conflict", "operation_cancelled", "settlement_recovery_required"].includes(String(error?.code));
+}
+/** A retained native effect is not permission to continue under a new daemon. */
+async function recoveredSessionReceipt(context, session, args, operationId, receipt, binding = {}, signal, inspection) {
+    const method = expectedMethod(args, session);
+    if (!["session.create", "session.prompt", "session.start", "session.fork", "session.resume"].includes(method))
+        return receipt;
+    const requested = await readOperationRecord(session.state_dir, operationId, "requested.json");
+    signal?.throwIfAborted();
+    const daemon = fs.existsSync(path.join(session.state_dir, "daemon.json")) ? readNativeDaemonState(session.state_dir) : null;
+    // Old minimal adapters have neither owned operation records nor daemon
+    // ownership metadata. Missing a modern record cannot downgrade to that path.
+    const modern = requested?.owner_id || requested && Object.hasOwn(requested, "params_sha256") || requested?.generation != null || daemon?.config?.runtime_owner?.owner_id
+        || managedProcessStatus({ ...context, env: { ...context.env, ...(session.owner.resource_home ? { DD_FLOW_RESOURCE_HOME: session.owner.resource_home } : {}) } }).some(record => {
+            const metadata = JSON.parse(record.metadata_json);
+            return record.kind.endsWith("-daemon") && record.owner_id === session.owner.owner_id && record.project_id === session.owner.project_id && record.run_id === session.owner.run_id
+                && metadata.state_dir === session.state_dir && metadata.dd_flow_home === session.owner.dd_flow_home && metadata.adapter_executable === session.owner.adapter_executable;
+        });
+    if (!modern)
+        return receipt;
+    const target = args.includes("--session-id") ? args[args.indexOf("--session-id") + 1] : undefined;
+    const retained = await retainedDaemonReply(session.state_dir, operationId, { ...binding, operation: method, owner_id: session.owner.owner_id, ...(target ? { session_id: target } : {}) });
+    signal?.throwIfAborted();
+    if (!retained || retained.reason)
+        throw new AppError("settlement_recovery_required", "Retained native outcome has no current dispatch authority", 1, { operation_id: operationId, native_outcome: "completed", native_result: retained?.outcome ?? receipt, settlement: retained?.settlement, reason: retained?.reason ?? "native_receipt_unavailable" });
+    if (inspection && settlement(inspection)?.state === "settled" && !retained.ready) {
+        const proof = inspection.settlement;
+        if (proof.source !== "inspection" || !Array.isArray(proof.operation_ids) || !proof.operation_ids.includes(operationId))
+            throw new AppError("settlement_recovery_required", "Fresh inspection does not settle the original retained operation", 1, { operation_id: operationId, native_outcome: "completed", native_result: retained.outcome, reason: "inspection_settlement_unbound", settlement: proof });
+    }
+    const result = retained.ready ? retained.value : retained.outcome;
+    if (!result || typeof result !== "object" || Array.isArray(result))
+        throw new AppError("controller_operation_mismatch", "Retained native outcome has no Session result", 1, { operation_id: operationId });
+    validateReceipt(result, session, args);
+    if (method === "session.create" && !adapterSessionId(result))
+        throw new AppError("session_identity_mismatch", "Retained creation outcome has no native Session identity", 1);
+    return { ...receipt, ...result, settlement: retained.ready ? retained.settlement ?? { state: "not_required" } : ["pending", "blocked"].includes(String(retained.settlement?.state)) ? retained.settlement : { ...retained.settlement, state: "pending", reason: retained.settlement?.reason ?? "settlement_evidence_required" } };
+}
+function validateReceipt(receipt, session, args) {
+    // `daemon start --session-id` names the retained state being reopened; the
+    // start receipt itself is a daemon lifecycle result and need not echo a
+    // provider Session identity.  Only session operations promise that binding.
+    const target = args[0] === "session" && args.includes("--session-id") ? args[args.indexOf("--session-id") + 1] : null;
+    if (target && adapterSessionId(receipt) !== target)
+        throw new AppError("session_identity_mismatch", "Managed adapter reply belongs to another or unidentified Session", 1);
+    if (receipt.harness !== undefined && receipt.harness !== runtimeHarness(session.profile))
+        throw new AppError("profile_integrity_violation", "Managed adapter returned another harness identity", 1);
+    const fork = receipt.fork_receipt;
+    const forkUnproven = (args[0] === "session" && args[1] === "fork" && receipt.fork_continuity_error !== undefined)
+        || (fork && typeof fork === "object" && !Array.isArray(fork) && fork.history_prefix_proven === false);
+    if (forkUnproven) {
+        throw new AppError("fork_continuity_unproven", "Native fork exists but its source-history continuity is not proven", 1);
+    }
+}
+async function recoverCompletedOperation(context, manifest, session, executable, args, operationId, signal) {
+    // Creation has no Session ID yet, but its original operation ID is already
+    // retained. Inspect that receipt, never invoke native creation a second time.
+    if (expectedMethod(args, session) === "session.inspect")
+        return null;
+    // A detached adapter without a native daemon has no durable operation
+    // journal to inspect. Avoid a second adapter invocation in that case; the
+    // original failure remains outcome-unknown and the controller will fence it.
+    if (!fs.existsSync(path.join(session.state_dir, "daemon.json")))
+        return null;
+    let observed;
+    try {
+        observed = await runHarnessAdapter({ executable, args: operationInspectArgs(args, operationId), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:observe`, runtimeOwner: session.owner, timeoutMs: 30_000, ...(signal ? { signal } : {}) });
+    }
+    catch {
+        return null;
+    }
+    const target = args.includes("--session-id") ? args[args.indexOf("--session-id") + 1] : null;
+    const result = observed.result;
+    const primary = observed.primary_error ?? (["failed", "interrupted"].includes(String(observed.state)) ? observed.error : undefined);
+    if (observed.operation_id === operationId && observed.operation === expectedMethod(args, session)
+        && observed.state !== "completed" && (!target || observed.session_id === target)
+        && primary && typeof primary === "object" && !Array.isArray(primary)) {
+        const failure = primary;
+        if (typeof failure.code === "string" && typeof failure.message === "string") {
+            throw new AppError(failure.code, failure.message, 1, { ...(failure.details && typeof failure.details === "object" && !Array.isArray(failure.details) ? failure.details : {}), operation_id: operationId, effect: "unknown" });
+        }
+    }
+    if (observed.operation_id !== operationId || observed.operation !== expectedMethod(args, session) || observed.state !== "completed"
+        || !result || typeof result !== "object" || Array.isArray(result)
+        || target && (observed.session_id !== target || adapterSessionId(result) !== target))
+        return null;
+    return await recoveredSessionReceipt(context, session, args, operationId, { ...result, ...(settlement(observed) ? { settlement: settlement(observed) } : {}) }, Object.fromEntries(["generation", "params_sha256"].filter(key => observed[key] !== undefined).map(key => [key, observed[key]])), signal);
+}
+async function awaitSettlement(context, row, manifest, session, executable, args, operationId, initial, assertOwner, signal) {
+    let current = initial, delay = 250;
+    const observation = recoveryObservationBudget(initial);
+    observation.observe(initial);
+    const retainObservation = () => {
+        assertOwner?.();
+        const receipt = { ...initial, ...current, recovery_observation: observation.state() };
+        const changed = context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [JSON.stringify(receipt), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
+        if (changed.changes !== 1)
+            throw new AppError("controller_owner_mismatch", "Controller lost ownership before settlement observation retention", 1);
+    };
+    while (["pending", "blocked"].includes(String(settlement(current)?.state))) {
+        if (signal?.aborted)
+            throw new AppError("harness_settlement_interrupted", "Harness settlement was interrupted", 1, { operation_id: operationId });
+        // A completed parent with live children is normal controller waiting, not
+        // failed bookkeeping. Keep its budget reserved until a later inspection.
+        if (["session.prompt", "session.inspect"].includes(expectedMethod(args, session)) && settlement(current)?.reason === "tree_unsettled")
+            return { ...initial, ...current, ...(initial.assistant_text !== undefined ? { assistant_text: initial.assistant_text } : {}) };
+        const exhausted = observation.exhausted();
+        retainObservation();
+        if (exhausted) {
+            // Final bounded read is observation, never another settlement mutation.
+            if (args.includes("--session-id")) {
+                assertOwner?.();
+                try {
+                    const terminal = await runHarnessAdapter({ executable, args: settlementInspectArgs(args), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement-final-inspect`, runtimeOwner: session.owner, timeoutMs: 30_000, ...(signal ? { signal } : {}) });
+                    validateReceipt(terminal, session, args);
+                    if (settlement(terminal)?.state === "settled") {
+                        await recoveredSessionReceipt(context, session, args, operationId, initial, {}, signal, terminal);
+                        assertOwner?.();
+                        current = terminal;
+                        break;
+                    }
+                }
+                catch (error) {
+                    if (recoveryAuthorityError(error))
+                        throw error;
+                }
+            }
+            throw new AppError("settlement_recovery_required", "Completed provider operation requires settlement recovery", 1, { operation_id: operationId, settlement: settlement(current), recovery_observation: observation.state() });
+        }
+        await new Promise(resolve => setTimeout(resolve, Math.min(delay, observation.timeout())));
+        delay = Math.min(delay * 2, 8_000);
+        assertOwner?.();
+        if (observation.exhausted())
+            continue;
+        try {
+            // This only retries durable bookkeeping against the same operation
+            // receipt.  It cannot replay the provider request.
+            const inspect = expectedMethod(args, session) === "session.inspect";
+            current = await runHarnessAdapter({ executable, args: inspect ? settlementInspectArgs(args) : settlementRetryArgs(args, operationId), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement`, runtimeOwner: session.owner, timeoutMs: observation.timeout(), ...(signal ? { signal } : {}) });
+            if (inspect)
+                validateReceipt(current, session, args);
+            else
+                current = await recoveredSessionReceipt(context, session, args, operationId, current, Object.fromEntries(["generation", "params_sha256"].filter(key => current[key] !== undefined).map(key => [key, current[key]])), signal);
+        }
+        catch (error) {
+            if (recoveryAuthorityError(error))
+                throw error;
+            current = { settlement: { state: "pending", reason: "inspection_error", error: errorRecord(error) } };
+        }
+        observation.observe(current);
+        if (!(["pending", "blocked"].includes(String(settlement(current)?.state))))
+            continue;
+        // A receipt without final tree proof needs a fresh native inspection.  Do
+        // it only after the receipt-only retry, and only when this is a session.
+        if (!args.includes("--session-id") || !["tree_unsettled", "settlement_evidence_required"].includes(String(settlement(current)?.reason)) || observation.exhausted())
+            continue;
+        try {
+            current = await runHarnessAdapter({ executable, args: settlementInspectArgs(args), env: controllerEnvironment(context, manifest, session), operationId: `${operationId}:settlement-inspect`, runtimeOwner: session.owner, timeoutMs: observation.timeout(), ...(signal ? { signal } : {}) });
+            validateReceipt(current, session, args);
+            await recoveredSessionReceipt(context, session, args, operationId, initial, {}, signal, current);
+        }
+        catch (error) {
+            if (recoveryAuthorityError(error))
+                throw error;
+            current = { settlement: { state: "pending", reason: "inspection_error", error: errorRecord(error) } };
+        }
+        observation.observe(current);
+    }
+    return adapterSessionId(current) ? { ...initial, ...current, ...(initial.assistant_text !== undefined ? { assistant_text: initial.assistant_text } : {}) } : { ...initial, settlement: settlement(current) };
+}
+export async function controllerAdapter(context, row, manifest, session, args, assertOwner, retainedOperationId, signal, reconfirmOwner, obligation) {
+    const executable = manifest.adapters[runtimeHarness(session.profile)].executable;
+    const promptFile = args.indexOf("--prompt-file"), inlinePrompt = args.indexOf("--prompt");
+    const promptSha256 = promptFile >= 0 ? digestBytes(fs.readFileSync(args[promptFile + 1])) : inlinePrompt >= 0 ? digestBytes(args[inlinePrompt + 1] ?? "") : undefined;
+    const priorInput = retainedOperationId ? context.db.get("SELECT input_json FROM run_controller_operations WHERE operation_id=? AND controller_id=? AND generation=?", [retainedOperationId, row.controller_id, row.generation]) : undefined;
+    const effectiveObligation = priorInput ? JSON.parse(priorInput.input_json).obligation : obligation;
+    // Committing an answer resumes a Stage; it does not finish the semantic
+    // work owed by that prompt. Preserve its provider error for same-Session recovery.
+    const acceptedPromptBoundary = (expected) => controllerObligationAccepted(context, row, expected.action === "resume" ? { ...expected, action: "finish" } : expected);
+    const input = { executable, args, owner: session.owner.owner_id, state_dir: session.state_dir, ...(args[0] === "session" && args[1] === "prompt" && promptSha256 ? { prompt_sha256: promptSha256 } : {}), ...(effectiveObligation ? { obligation: effectiveObligation } : {}) };
+    const prepared = retainedOperationId ? [] : context.db.all("SELECT operation_id FROM run_controller_operations WHERE controller_id = ? AND generation = ? AND input_hash = ? AND status = 'dispatching' AND json_extract(input_json, '$.dispatch_phase') = 'prepared'", [row.controller_id, row.generation, digest(input)]);
+    if (prepared.length > 1)
+        throw new AppError("controller_operation_conflict", "Multiple prepared intents require explicit reconciliation", 1);
+    const operationId = retainedOperationId ?? prepared[0]?.operation_id ?? `${row.controller_id}:${crypto.randomUUID()}`;
+    const now = context.now();
+    const existing = context.db.get("SELECT input_hash,input_json,status,receipt_json,error_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [operationId, row.controller_id, row.generation]);
+    if (existing?.input_hash && existing.input_hash !== digest(input))
+        throw new AppError("controller_operation_conflict", "Retained operation has different dispatch parameters", 1, { operation_id: operationId });
+    if (existing?.status === "completed" && existing.receipt_json) {
+        const receipt = JSON.parse(existing.receipt_json);
+        validateReceipt(receipt, session, args);
+        return receipt;
+    }
+    if (!existing) {
+        context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, created_at, updated_at) SELECT ?, ?, ?, ?, ?, 'dispatching', ?, ? WHERE EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [operationId, row.controller_id, row.generation, digest(input), JSON.stringify({ ...input, dispatch_phase: "prepared" }), now, now, row.controller_id, row.owner_token, row.generation]);
+        appendEvent(context, row.controller_id, "operation_requested", { operation_id: operationId, operation: args.slice(0, 2), session_id: session.id || null });
+    }
+    let phase = existing ? JSON.parse(existing.input_json).dispatch_phase : "prepared";
+    const retainPhase = (next) => {
+        const changed = context.db.run("UPDATE run_controller_operations SET input_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status <> 'completed' AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [JSON.stringify({ ...input, dispatch_phase: next }), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
+        if (changed.changes !== 1)
+            throw new AppError("controller_owner_mismatch", "Controller lost ownership before durable dispatch transition", 1, { operation_id: operationId });
+        phase = next;
+    };
+    try {
+        await reconfirmOwner?.();
+        assertOwner?.();
+        let receipt;
+        let recoveredOperation = false;
+        try {
+            const timeoutMs = 120_000;
+            if (existing && phase === "receipt_observed" && existing.receipt_json) {
+                // The exact input hash and committed receipt survive a controller
+                // crash. Complete settlement without replaying productive native IO.
+                receipt = JSON.parse(existing.receipt_json);
+                validateReceipt(receipt, session, args);
+                if (args[0] === "session" && args[1] === "create" && !adapterSessionId(receipt))
+                    throw new AppError("session_identity_mismatch", "Retained creation receipt has no native Session identity", 1);
+                receipt = await recoveredSessionReceipt(context, session, args, operationId, receipt, {}, signal);
+                recoveredOperation = true;
+            }
+            else if (existing && phase !== "prepared") {
+                if (expectedMethod(args, session) === "session.inspect") {
+                    // A failed read may be explicitly reobserved. Never do this for a
+                    // materializing resume, prompt, create or fork.
+                    retainPhase("external_dispatch_possible");
+                    receipt = await runHarnessAdapter({ executable, args: settlementInspectArgs(args), harness: runtimeHarness(session.profile), env: controllerEnvironment(context, manifest, session), operationId, runtimeOwner: session.owner, timeoutMs, ...(signal ? { signal } : {}) });
+                }
+                else {
+                    const observed = await recoverCompletedOperation(context, manifest, session, executable, args, operationId, signal);
+                    if (!observed) {
+                        const cause = existing.error_json ? JSON.parse(existing.error_json) : { code: "controller_operation_outcome_unknown", message: "Retained dispatch has no observed terminal result", details: { operation_id: operationId } };
+                        throw new AppError(cause.code, cause.message, 1, cause.details);
+                    }
+                    receipt = observed;
+                    recoveredOperation = true;
+                }
+            }
+            else {
+                // Keep observation/settlement of retained operations available after
+                // terminal RUN, but never admit fresh native productive dispatch.
+                const productive = ["daemon.start", "session.create", "session.start", "session.fork", "session.prompt", "session.resume"].includes(expectedMethod(args, session));
+                if (productive && ["done", "failed", "cancelled"].includes(requireRun(context, row.project_id, row.run_id).status))
+                    throw new AppError("invalid_run_state", "A terminal RUN cannot admit a new native controller operation", 1);
+                retainPhase("external_dispatch_possible");
+                receipt = await runHarnessAdapter({ executable, args, harness: runtimeHarness(session.profile), env: controllerEnvironment(context, manifest, session), operationId, runtimeOwner: session.owner, timeoutMs, ...(signal ? { signal } : {}) });
+            }
+        }
+        catch (error) {
+            if (phase === "prepared" || phase === "receipt_observed" && existing?.receipt_json)
+                throw error;
+            const retained = await recoverCompletedOperation(context, manifest, session, executable, args, operationId, signal);
+            if (!retained)
+                throw error;
+            receipt = retained;
+            recoveredOperation = true;
+        }
+        validateReceipt(receipt, session, args);
+        const hookDiagnostics = Array.isArray(receipt.native_hook_diagnostics) ? receipt.native_hook_diagnostics : [];
+        if (hookDiagnostics.some(item => item && typeof item === "object" && ["native_hook_timeout", "native_hook_failed"].includes(String(item.code)))) {
+            recordTechnicalNote(context, row.project_id, row.run_id, { code: "native_hook_observation_unavailable",
+                message: "Native hook observation failed; the actual lifecycle command remains authoritative.", phase: "native_observation",
+                generation: row.generation, ...(effectiveObligation?.stage ? { stage: effectiveObligation.stage } : {}),
+                harness: runtimeHarness(session.profile), source_operation_id: operationId, resource_identity: session.state_dir,
+                diagnostic_reference: session.journal, consequence: "none" });
+        }
+        retainPhase("receipt_observed");
+        const persistReceipt = () => {
+            const changed = context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [JSON.stringify(receipt), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
+            if (changed.changes !== 1)
+                throw new AppError("controller_owner_mismatch", "Controller lost ownership before receipt retention", 1);
+        };
+        persistReceipt();
+        if (["pending", "blocked"].includes(String(settlement(receipt)?.state)) && effectiveObligation && acceptedPromptBoundary(effectiveObligation)) {
+            // Required semantic work is already committed. Keep the honest pending
+            // native proof for resource cleanup without holding its accepted result.
+            recordTechnicalNote(context, row.project_id, row.run_id, { code: "accepted_boundary_native_settlement_pending", message: "The assigned result is committed; native settlement remains pending.", phase: "native_observation", generation: row.generation, stage: effectiveObligation.stage, harness: runtimeHarness(session.profile), source_operation_id: operationId, diagnostic_reference: session.state_dir, consequence: "cleanup_pending" });
+            receipt = { ...receipt, accepted_boundary: effectiveObligation };
+        }
+        else if (["pending", "blocked"].includes(String(settlement(receipt)?.state))) {
+            const recovered = await awaitSettlement(context, row, manifest, session, executable, args, operationId, receipt, assertOwner, signal);
+            receipt = recovered;
+        }
+        persistReceipt();
+        validateReceipt(receipt, session, args);
+        const completed = context.db.run("UPDATE run_controller_operations SET status = 'completed', updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
+        if (completed.changes !== 1)
+            throw new AppError("controller_owner_mismatch", "Controller lost ownership before operation completion", 1);
+        appendEvent(context, row.controller_id, recoveredOperation ? "operation_recovered" : "operation_completed", { operation_id: operationId, operation: args.slice(0, 2), session_id: adapterSessionId(receipt) });
+        return receipt;
+    }
+    catch (error) {
+        // A lost provider/transport acknowledgement cannot revoke the exact
+        // boundary already accepted by the engine. Identity/integrity failures
+        // and unfinished siblings remain on the existing recovery path.
+        const expected = effectiveObligation;
+        if (expected && lateNativeFailure(error, session.id) && acceptedPromptBoundary(expected)) {
+            assertOwner?.();
+            const receipt = { adapter_session_id: session.id, accepted_boundary: expected, native_observation_unavailable: true, settlement: { state: "pending", reason: "native_observation_unavailable" } };
+            const completed = context.db.run("UPDATE run_controller_operations SET status='completed',receipt_json=?,error_json=NULL,updated_at=? WHERE operation_id=? AND controller_id=? AND generation=? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id=? AND owner_token=? AND generation=?)", [JSON.stringify(receipt), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
+            if (completed.changes !== 1)
+                throw new AppError("controller_owner_mismatch", "Controller lost ownership before accepting the committed boundary", 1);
+            recordTechnicalNote(context, row.project_id, row.run_id, { code: "accepted_boundary_native_observation_unavailable", message: "The assigned result is committed; the native acknowledgement is unavailable.", phase: "native_observation", generation: row.generation, stage: expected.stage, harness: runtimeHarness(session.profile), source_operation_id: operationId, diagnostic_reference: session.state_dir, consequence: "cleanup_pending" });
+            return receipt;
+        }
+        try {
+            context.db.run("UPDATE run_controller_operations SET status = CASE WHEN status = 'dispatching' AND ? <> 'prepared' THEN 'outcome_unknown' ELSE status END, error_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND EXISTS (SELECT 1 FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?)", [phase ?? "legacy_unknown", JSON.stringify(errorRecord(error)), context.now(), operationId, row.controller_id, row.generation, row.controller_id, row.owner_token, row.generation]);
+        }
+        catch (persistence) {
+            process.stderr.write(`${JSON.stringify({ operation_id: operationId, primary: errorRecord(error), persistence_error: errorRecord(persistence) })}\n`);
+        }
+        throw error;
+    }
+}

package/dist/services/run-controller-capture.js

@@ -0,0 +1,186 @@
+import fs from "node:fs";
+import path from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import { AppError } from "../shared/errors.js";
+import { createEvalRunSnapshot, verifyControllerBoundarySnapshot } from "./eval-snapshots.js";
+import { observeControllerBoundary } from "./run-control.js";
+import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
+import { appendEvent, digest, errorRecord, requireController, requireRun } from "./run-controller-state.js";
+import { getFlowRunStatus, readFlowRunExecutionBundle } from "./runs.js";
+import { recoveryObservationBudget, withRecoveryObservation } from "./recovery-observation-budget.js";
+import { getResourceDatabase } from "../storage/database.js";
+import { managedProcessBinding, resourceHome } from "./managed-processes.js";
+/** Physical probes precede reservation; the accepted publisher still checks
+ * exact owner identity and fresh lease expiry after waiting for the writer. */
+export function prepareControllerPublication(context, row, assertPhysicalOwner) {
+    assertPhysicalOwner();
+    const registry = getResourceDatabase(resourceHome(context));
+    const owner = registry.get("SELECT * FROM managed_processes WHERE id = ?", [row.process_id]);
+    if (!owner || owner.lease_token !== row.process_lease_token || owner.pid !== row.owner_pid || owner.owner_id !== row.controller_id)
+        throw new AppError("controller_owner_mismatch", "Controller publication lacks its physical owner", 1, { process_id: row.process_id, controller_id: row.controller_id, owner_found: Boolean(owner), lease_matches: owner?.lease_token === row.process_lease_token, pid_matches: owner?.pid === row.owner_pid, owner_matches: owner?.owner_id === row.controller_id, resource_home: resourceHome(context) });
+    const binding = managedProcessBinding(owner, resourceHome(context));
+    return () => {
+        const current = requireController(context, row.controller_id), process = registry.get("SELECT * FROM managed_processes WHERE id = ?", [owner.id]);
+        if (current.owner_token !== row.owner_token || current.owner_pid !== row.owner_pid || current.process_id !== row.process_id || current.process_lease_token !== row.process_lease_token || current.generation !== row.generation
+            || !["running", "capturing", "waiting_for_user", "waiting_for_context", "waiting_for_children", "waiting_for_capacity", "waiting_for_merge"].includes(current.status)
+            || !process || process.state !== "running" || process.lease_expires_at <= context.now() || digest(managedProcessBinding(process, resourceHome(context))) !== digest(binding))
+            throw new AppError("controller_owner_mismatch", "Controller publication lost its current physical lease", 1);
+    };
+}
+/** Capture is performed by the productive owner, before it releases the next
+ * Stage. An observer never races the controller to copy a completed boundary. */
+export async function captureControllerBoundary(context, row, manifest, stage, nextStage, assertPhysicalOwner) {
+    const root = manifest.request.capture_root;
+    if (!root)
+        return;
+    const identity = (value) => ({ run_id: row.run_id, stage: value.stage, attempt: value.attempt ?? null, started_at: value.started_at ?? null, completed_at: value.completed_at ?? null });
+    const boundaryKey = digest(identity(stage));
+    const operationId = `${row.controller_id}:boundary:${boundaryKey}`;
+    const output = path.join(root, `${stage.stage}-${boundaryKey}`);
+    const scope = { projectRoot: manifest.request.project_root, runId: row.run_id };
+    const expected = { controllerId: row.controller_id, operationId, boundaryKey, runId: row.run_id, stageEntry: nextStage };
+    const prior = context.db.get("SELECT status,receipt_json FROM run_controller_operations WHERE operation_id = ?", [operationId]);
+    if (prior?.status === "completed") {
+        try {
+            const receipt = JSON.parse(prior.receipt_json ?? "{}");
+            if (typeof receipt.manifest_sha256 !== "string")
+                throw new AppError("controller_boundary_snapshot_invalid", "Completed capture has no immutable manifest receipt", 1);
+            verifyControllerBoundarySnapshot(output, { ...expected, manifestSha256: receipt.manifest_sha256 });
+            return;
+        }
+        catch (error) {
+            try {
+                assertPhysicalOwner();
+                context.db.run("UPDATE run_controllers SET status = 'capture_failed', last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [JSON.stringify(errorRecord(error)), context.now(), row.controller_id, row.owner_token]);
+            }
+            catch (recordError) {
+                if (error instanceof AppError)
+                    error.details.capture_error_recording = errorRecord(recordError);
+            }
+            throw error;
+        }
+    }
+    const preparedBundle = readFlowRunExecutionBundle(context, scope);
+    const assertStartOwner = prepareControllerPublication(context, row, assertPhysicalOwner);
+    context.db.writeTransaction(() => {
+        assertStartOwner();
+        assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation);
+        const now = context.now();
+        context.db.run("UPDATE run_controllers SET status = 'capturing', updated_at = ? WHERE controller_id = ? AND owner_token = ?", [now, row.controller_id, row.owner_token]);
+        context.db.run("INSERT INTO run_controller_operations (operation_id,controller_id,generation,input_hash,input_json,status,created_at,updated_at) VALUES (?,?,?,?,?,'capturing',?,?) ON CONFLICT(operation_id) DO UPDATE SET generation = excluded.generation, status = 'capturing', error_json = NULL, updated_at = excluded.updated_at", [operationId, row.controller_id, row.generation, boundaryKey, JSON.stringify({ action: "boundary_capture", ...identity(stage), stage_entry: nextStage, output }), now, now]);
+        appendEvent(context, row.controller_id, "boundary_capture_requested", { operation_id: operationId, boundary_key: boundaryKey, stage: stage.stage, stage_entry: nextStage });
+    });
+    const assertCurrent = (physical = true) => {
+        if (physical)
+            assertPhysicalOwner();
+        const guard = recoveryGuard(context, row.project_id, row.run_id);
+        if ((guard?.generation ?? 0) !== row.generation || ["draining", "sealed", "resuming"].includes(guard?.status ?? ""))
+            throw new AppError("run_recovery_guarded", "Recovery or operator control superseded boundary capture", 1);
+        if (requireController(context, row.controller_id).status !== "capturing" || context.db.get("SELECT status FROM run_controller_operations WHERE operation_id = ?", [operationId])?.status !== "capturing")
+            throw new AppError("controller_capture_owner_lost", "Controller no longer owns boundary capture", 1);
+        const index = JSON.parse(requireRun(context, row.project_id, row.run_id).index_json);
+        const current = index.stage_runs.find(item => item.stage === stage.stage);
+        if (!current || !["done", "skipped"].includes(current.status) || digest(identity(current)) !== boundaryKey)
+            throw new AppError("controller_boundary_changed", "Stage identity changed after its capture fence", 1);
+        if (physical) {
+            const continuation = getFlowRunStatus(context, scope, preparedBundle ?? undefined).continuation;
+            const stoppedAtTarget = nextStage === null && manifest.request.stop_after === stage.stage && continuation.kind === "start_stage";
+            if (nextStage ? continuation.kind !== "start_stage" || continuation.stage !== nextStage : continuation.kind !== "terminal" && !stoppedAtTarget)
+                throw new AppError("controller_boundary_changed", "Stage successor changed after its capture fence", 1);
+        }
+    };
+    let priorObservation = null;
+    try {
+        priorObservation = prior?.receipt_json ? JSON.parse(prior.receipt_json) : null;
+    }
+    catch {
+        priorObservation = { recovery_observation: { remaining_ms: 0 } };
+    }
+    const observationBudget = recoveryObservationBudget(priorObservation);
+    try {
+        let receipt;
+        if (fs.existsSync(output)) {
+            assertCurrent();
+            receipt = verifyControllerBoundarySnapshot(output, expected);
+        }
+        else {
+            let lastPending = "";
+            let lastCaptureMismatch = null;
+            for (;;) {
+                assertCurrent();
+                if (fs.existsSync(output)) {
+                    receipt = verifyControllerBoundarySnapshot(output, expected);
+                    break;
+                }
+                if (observationBudget.exhausted()) {
+                    assertCurrent();
+                    if (fs.existsSync(output)) {
+                        receipt = verifyControllerBoundarySnapshot(output, expected);
+                        break;
+                    }
+                    context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)), '$.last_capture_mismatch', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), JSON.stringify(lastCaptureMismatch), context.now(), operationId]);
+                    throw new AppError("recovery_observation_budget_exhausted", "Boundary capture has no stable settlement within the active observation budget", 1, { recovery_observation: observationBudget.state(), last_capture_mismatch: lastCaptureMismatch });
+                }
+                assertCurrent();
+                context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), context.now(), operationId]);
+                let verify = () => { throw new AppError("controller_boundary_unsettled", "Boundary has no stable writer observation", 1); };
+                const observation = await withRecoveryObservation(observationBudget, () => observeControllerBoundary(context, { ...scope, controllerId: row.controller_id, operationId, assertCurrent, retainVerification: value => {
+                        verify = () => {
+                            value();
+                            if (readFlowRunExecutionBundle(context, scope)?.sha256 !== preparedBundle?.sha256)
+                                throw new AppError("controller_boundary_changed", "Frozen bundle changed before snapshot publication", 1);
+                        };
+                    } }));
+                if (observation.settled === true) {
+                    try {
+                        createEvalRunSnapshot(context, { ...scope, output, ...(nextStage ? { stageEntry: nextStage } : { candidate: true }), boundary: { controllerId: row.controller_id, operationId, boundaryKey, observation, verify } });
+                        receipt = verifyControllerBoundarySnapshot(output, expected);
+                        break;
+                    }
+                    catch (error) {
+                        if (!(error instanceof AppError) || !["snapshot_source_changed", "owned_inventory_changed"].includes(error.code))
+                            throw error;
+                        observation.settled = false;
+                        observation.pending_reasons = [error.code];
+                        lastCaptureMismatch = { code: error.code, components: error.details?.components ?? [], changes: error.details?.changes ?? [] };
+                    }
+                }
+                observationBudget.observe(observation);
+                const published = { ...observation, recovery_observation: observationBudget.state(), last_capture_mismatch: lastCaptureMismatch };
+                const pending = digest(observation.pending_reasons);
+                context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(published), context.now(), operationId]);
+                if (pending !== lastPending) {
+                    appendEvent(context, row.controller_id, "boundary_capture_pending", { operation_id: operationId, pending_reasons: observation.pending_reasons, last_capture_mismatch: lastCaptureMismatch });
+                    lastPending = pending;
+                }
+                await delay(observationBudget.nextDelay());
+            }
+        }
+        assertCurrent();
+        const assertFinishOwner = prepareControllerPublication(context, row, assertPhysicalOwner);
+        const preparedIndex = requireRun(context, row.project_id, row.run_id).index_json;
+        context.db.writeTransaction(() => {
+            assertFinishOwner();
+            assertCurrent(false);
+            if (requireRun(context, row.project_id, row.run_id).index_json !== preparedIndex)
+                throw new AppError("controller_boundary_changed", "RUN changed before capture publication", 1);
+            context.db.run("UPDATE run_controller_operations SET status = 'completed', receipt_json = ?, error_json = NULL, updated_at = ? WHERE operation_id = ?", [JSON.stringify(receipt), context.now(), operationId]);
+            appendEvent(context, row.controller_id, "boundary_captured", { ...identity(stage), boundary_key: boundaryKey, operation_id: operationId, ...receipt });
+            context.db.run("UPDATE run_controllers SET status = 'running', updated_at = ? WHERE controller_id = ? AND owner_token = ?", [context.now(), row.controller_id, row.owner_token]);
+        });
+    }
+    catch (error) {
+        observationBudget.exhausted();
+        try {
+            context.db.writeTransaction(() => {
+                context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(COALESCE(receipt_json, '{}'), '$.recovery_observation', json(?)) WHERE operation_id = ?", [JSON.stringify(observationBudget.state()), operationId]);
+                context.db.run("UPDATE run_controller_operations SET status = 'failed', error_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(errorRecord(error)), context.now(), operationId]);
+            });
+        }
+        catch (recordError) {
+            if (error instanceof AppError)
+                error.details.capture_error_recording = errorRecord(recordError);
+        }
+        throw error;
+    }
+}