@deksden-com/dd-flow-cli 0.9.0-beta.38 → 0.9.0-beta.39

package/dist/services/run-controller-adapter.js

@@ -0,0 +1,44 @@
+import crypto from "node:crypto";
+import { AppError } from "../shared/errors.js";
+import { adapterSessionId, runHarnessAdapter } from "./harness-adapter.js";
+import { runtimeHarness } from "./execution-policy.js";
+import { appendEvent, digest, errorRecord } from "./run-controller-state.js";
+export function sessionArguments(context, manifest, session) {
+    const profile = session.profile;
+    return ["--dd-flow-bin", manifest.flow_executable, "--dd-flow-home", context.ddFlowHome, "--project-root", manifest.request.project_root, "--state-dir", session.state_dir, "--journal", session.journal, "--cwd", session.cwd, "--provider", profile.provider, "--model", profile.model, "--reasoning", profile.reasoning, "--mode", profile.mode, "--permission", profile.permission, ...manifest.adapters[runtimeHarness(profile)].runtime_args, "--json"];
+}
+export function controllerEnvironment(context, manifest, session) {
+    const env = { ...context.env, DD_FLOW_HOME: context.ddFlowHome, DD_FLOW_BIN: manifest.flow_executable, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_RUNTIME_BUDGET: manifest.budget ? JSON.stringify(manifest.budget) : undefined };
+    for (const key of ["DD_FLOW_CONTROLLER_TOKEN", "DD_FLOW_CONTROL_TOKEN", "DD_FLOW_DAEMON_ID", "DD_FLOW_RUNTIME_OWNER", "DD_FLOW_WORK_ID", "DD_FLOW_SESSION_ID", "CODEX_THREAD_ID"])
+        delete env[key];
+    if (session?.codex_home)
+        env.CODEX_HOME = session.codex_home;
+    return env;
+}
+export async function controllerAdapter(context, row, manifest, session, args, assertOwner, retainedOperationId) {
+    assertOwner?.();
+    const operationId = retainedOperationId ?? `${row.controller_id}:${crypto.randomUUID()}`;
+    const executable = manifest.adapters[runtimeHarness(session.profile)].executable;
+    const input = { executable, args, owner: session.owner.owner_id, state_dir: session.state_dir };
+    const now = context.now();
+    context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'dispatching', ?, ?)", [operationId, row.controller_id, row.generation, digest(input), JSON.stringify(input), now, now]);
+    appendEvent(context, row.controller_id, "operation_requested", { operation_id: operationId, operation: args.slice(0, 2), session_id: session.id || null });
+    try {
+        assertOwner?.();
+        const receipt = await runHarnessAdapter({ executable, args, env: controllerEnvironment(context, manifest, session), operationId, runtimeOwner: session.owner, timeoutMs: args[0] === "session" && args[1] === "prompt" ? 45 * 60_000 : 120_000 });
+        context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(receipt), context.now(), operationId]);
+        const target = args[args.indexOf("--session-id") + 1];
+        if (args[0] === "session" && args.includes("--session-id") && adapterSessionId(receipt) !== target)
+            throw new AppError("session_identity_mismatch", "Managed adapter reply belongs to another or unidentified Session", 1);
+        if (receipt.harness !== undefined && receipt.harness !== runtimeHarness(session.profile))
+            throw new AppError("profile_integrity_violation", "Managed adapter returned another harness identity", 1);
+        context.db.run("UPDATE run_controller_operations SET status = 'completed', updated_at = ? WHERE operation_id = ?", [context.now(), operationId]);
+        assertOwner?.();
+        appendEvent(context, row.controller_id, "operation_completed", { operation_id: operationId, operation: args.slice(0, 2), session_id: adapterSessionId(receipt) });
+        return receipt;
+    }
+    catch (error) {
+        context.db.run("UPDATE run_controller_operations SET status = CASE WHEN status = 'dispatching' THEN 'outcome_unknown' ELSE status END, error_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(errorRecord(error)), context.now(), operationId]);
+        throw error;
+    }
+}

package/dist/services/run-controller-capture.js

@@ -0,0 +1,117 @@
+import fs from "node:fs";
+import path from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import { AppError } from "../shared/errors.js";
+import { createEvalRunSnapshot, verifyControllerBoundarySnapshot } from "./eval-snapshots.js";
+import { observeControllerBoundary } from "./run-control.js";
+import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
+import { appendEvent, digest, errorRecord, requireController, requireRun } from "./run-controller-state.js";
+import { getFlowRunStatus } from "./runs.js";
+/** Capture is performed by the productive owner, before it releases the next
+ * Stage. An observer never races the controller to copy a completed boundary. */
+export async function captureControllerBoundary(context, row, manifest, stage, nextStage, assertPhysicalOwner) {
+    const root = manifest.request.capture_root;
+    if (!root)
+        return;
+    const identity = (value) => ({ run_id: row.run_id, stage: value.stage, attempt: value.attempt ?? null, started_at: value.started_at ?? null, completed_at: value.completed_at ?? null });
+    const boundaryKey = digest(identity(stage));
+    const operationId = `${row.controller_id}:boundary:${boundaryKey}`;
+    const output = path.join(root, `${stage.stage}-${boundaryKey}`);
+    const scope = { projectRoot: manifest.request.project_root, runId: row.run_id };
+    const expected = { controllerId: row.controller_id, operationId, boundaryKey, runId: row.run_id, stageEntry: nextStage };
+    const prior = context.db.get("SELECT status,receipt_json FROM run_controller_operations WHERE operation_id = ?", [operationId]);
+    if (prior?.status === "completed") {
+        try {
+            const receipt = JSON.parse(prior.receipt_json ?? "{}");
+            if (typeof receipt.manifest_sha256 !== "string")
+                throw new AppError("controller_boundary_snapshot_invalid", "Completed capture has no immutable manifest receipt", 1);
+            verifyControllerBoundarySnapshot(output, { ...expected, manifestSha256: receipt.manifest_sha256 });
+            return;
+        }
+        catch (error) {
+            assertPhysicalOwner();
+            context.db.run("UPDATE run_controllers SET status = 'capture_failed', last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [JSON.stringify(errorRecord(error)), context.now(), row.controller_id, row.owner_token]);
+            throw error;
+        }
+    }
+    context.db.exec("BEGIN IMMEDIATE");
+    try {
+        assertPhysicalOwner();
+        assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation);
+        const now = context.now();
+        context.db.run("UPDATE run_controllers SET status = 'capturing', updated_at = ? WHERE controller_id = ? AND owner_token = ?", [now, row.controller_id, row.owner_token]);
+        context.db.run("INSERT INTO run_controller_operations (operation_id,controller_id,generation,input_hash,input_json,status,created_at,updated_at) VALUES (?,?,?,?,?,'capturing',?,?) ON CONFLICT(operation_id) DO UPDATE SET generation = excluded.generation, status = 'capturing', error_json = NULL, updated_at = excluded.updated_at", [operationId, row.controller_id, row.generation, boundaryKey, JSON.stringify({ action: "boundary_capture", ...identity(stage), stage_entry: nextStage, output }), now, now]);
+        appendEvent(context, row.controller_id, "boundary_capture_requested", { operation_id: operationId, boundary_key: boundaryKey, stage: stage.stage, stage_entry: nextStage });
+        context.db.exec("COMMIT");
+    }
+    catch (error) {
+        context.db.exec("ROLLBACK");
+        throw error;
+    }
+    const assertCurrent = () => {
+        assertPhysicalOwner();
+        const guard = recoveryGuard(context, row.project_id, row.run_id);
+        if ((guard?.generation ?? 0) !== row.generation || ["draining", "sealed", "resuming"].includes(guard?.status ?? ""))
+            throw new AppError("run_recovery_guarded", "Recovery or operator control superseded boundary capture", 1);
+        if (requireController(context, row.controller_id).status !== "capturing" || context.db.get("SELECT status FROM run_controller_operations WHERE operation_id = ?", [operationId])?.status !== "capturing")
+            throw new AppError("controller_capture_owner_lost", "Controller no longer owns boundary capture", 1);
+        const index = JSON.parse(requireRun(context, row.project_id, row.run_id).index_json);
+        const current = index.stage_runs.find(item => item.stage === stage.stage);
+        if (!current || !["done", "skipped"].includes(current.status) || digest(identity(current)) !== boundaryKey)
+            throw new AppError("controller_boundary_changed", "Stage identity changed after its capture fence", 1);
+        const continuation = getFlowRunStatus(context, scope).continuation;
+        if (nextStage ? continuation.kind !== "start_stage" || continuation.stage !== nextStage : continuation.kind !== "terminal")
+            throw new AppError("controller_boundary_changed", "Stage successor changed after its capture fence", 1);
+    };
+    try {
+        let receipt;
+        if (fs.existsSync(output)) {
+            assertCurrent();
+            receipt = verifyControllerBoundarySnapshot(output, expected);
+        }
+        else {
+            let lastPending = "";
+            for (;;) {
+                assertCurrent();
+                let verify = () => { throw new AppError("controller_boundary_unsettled", "Boundary has no stable writer observation", 1); };
+                const observation = await observeControllerBoundary(context, { ...scope, controllerId: row.controller_id, operationId, assertCurrent, retainVerification: value => { verify = value; } });
+                if (observation.settled === true) {
+                    try {
+                        createEvalRunSnapshot(context, { ...scope, output, ...(nextStage ? { stageEntry: nextStage } : { candidate: true }), boundary: { controllerId: row.controller_id, operationId, boundaryKey, observation, verify } });
+                        receipt = verifyControllerBoundarySnapshot(output, expected);
+                        break;
+                    }
+                    catch (error) {
+                        if (!(error instanceof AppError) || !["snapshot_source_changed", "owned_inventory_changed"].includes(error.code))
+                            throw error;
+                        observation.settled = false;
+                        observation.pending_reasons = [error.code];
+                    }
+                }
+                const pending = digest(observation.pending_reasons);
+                if (pending !== lastPending) {
+                    context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(observation), context.now(), operationId]);
+                    appendEvent(context, row.controller_id, "boundary_capture_pending", { operation_id: operationId, pending_reasons: observation.pending_reasons });
+                    lastPending = pending;
+                }
+                await delay(1000);
+            }
+        }
+        context.db.exec("BEGIN IMMEDIATE");
+        try {
+            assertCurrent();
+            context.db.run("UPDATE run_controller_operations SET status = 'completed', receipt_json = ?, error_json = NULL, updated_at = ? WHERE operation_id = ?", [JSON.stringify(receipt), context.now(), operationId]);
+            appendEvent(context, row.controller_id, "boundary_captured", { ...identity(stage), boundary_key: boundaryKey, operation_id: operationId, ...receipt });
+            context.db.run("UPDATE run_controllers SET status = 'running', updated_at = ? WHERE controller_id = ? AND owner_token = ?", [context.now(), row.controller_id, row.owner_token]);
+            context.db.exec("COMMIT");
+        }
+        catch (error) {
+            context.db.exec("ROLLBACK");
+            throw error;
+        }
+    }
+    catch (error) {
+        context.db.run("UPDATE run_controller_operations SET status = 'failed', error_json = ?, updated_at = ? WHERE operation_id = ?", [JSON.stringify(errorRecord(error)), context.now(), operationId]);
+        throw error;
+    }
+}

package/dist/services/run-controller-process.js

@@ -0,0 +1,178 @@
+import fs from "node:fs";
+import crypto from "node:crypto";
+import path from "node:path";
+import { spawn } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { AppError } from "../shared/errors.js";
+import { getResourceDatabase } from "../storage/database.js";
+import { finishManagedProcess, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processStartedAt, processTreeIsAlive, registerManagedProcess } from "./managed-processes.js";
+import { assertRunMutationAllowed, recoveryGuard } from "./run-recovery.js";
+import { requireRun, requireController, controllerReceipt, appendEvent, digest, errorRecord } from "./run-controller-state.js";
+const launchId = (row) => `${row.controller_id}:owner-launch:${digest(row.owner_token)}`;
+const processOperation = (context, row) => `controller:${digest({ database: context.db.path, project: row.project_id, run: row.run_id })}`;
+const birth = () => ({ pid: process.pid, pid_started_at: processStartedAt(process.pid) });
+const dead = (value) => Boolean(value && Number.isSafeInteger(value.pid) && value.pid > 0 && typeof value.pid_started_at === "string" && value.pid_started_at && !processIsAlive(value));
+/** Called inside the RUN intent transaction, before the client can disappear. */
+export function recordControllerLauncher(context, row, retired) {
+    const request = { action: "controller_owner_launch", owner_id: launchId(row) };
+    const receipt = { ...birth(), registration: "controller_self@1", ...(retired ? { retired_process: { id: retired.id, lease_token: retired.lease_token, pid: retired.pid, pid_started_at: retired.pid_started_at } } : {}) };
+    const now = context.now();
+    context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'completed', ?, ?, ?)", [launchId(row), row.controller_id, row.generation, digest(request), JSON.stringify(request), JSON.stringify(receipt), now, now]);
+}
+function launchRecord(context, row) {
+    const record = context.db.get("SELECT receipt_json FROM run_controller_operations WHERE operation_id = ?", [launchId(row)]);
+    if (!record)
+        return null;
+    const value = JSON.parse(record.receipt_json);
+    return value?.registration === "controller_self@1" ? { ...record, value } : null;
+}
+function assertLaunchAllowed(context, row) {
+    const run = requireRun(context, row.project_id, row.run_id);
+    if (!["running", "paused", "waiting_for_user"].includes(run.status))
+        throw new AppError("invalid_run_state", "A terminal RUN cannot start a physical controller", 1);
+    const state = JSON.parse(row.state_json);
+    if (!state.recovery) {
+        assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation);
+        return;
+    }
+    const guard = recoveryGuard(context, row.project_id, row.run_id);
+    if (guard?.generation !== row.generation || guard.recovery_id !== state.recovery.recovery_id || !["sealed", "resuming"].includes(guard.status)
+        || !context.db.get("SELECT 1 FROM run_controls WHERE control_id = ? AND recovery_id = ? AND status = 'resuming'", [state.recovery.control_id, state.recovery.recovery_id]))
+        throw new AppError("run_control_superseded", "Controller launch no longer owns the accepted recovery", 1);
+}
+function retire(context, row, identity) {
+    const record = managedProcessStatus(context).find(item => item.id === identity.id);
+    if (!record || record.kind !== "run-controller" || record.owner_id !== row.controller_id || record.project_id !== row.project_id || record.run_id !== row.run_id
+        || managedProcessRuntimeHome(record, context) !== context.ddFlowHome || record.lease_token !== identity.lease_token || record.pid !== identity.pid || record.pid_started_at !== identity.pid_started_at || !dead(identity) || processTreeIsAlive(record)
+        || !finishManagedProcess(context, { id: record.id, leaseToken: record.lease_token, state: "stopped", reason: "controller_launch_owner_retired" }))
+        throw new AppError("controller_owner_unsettled", "Prior controller identity is not proven dead and retired", 1);
+}
+/** Retry only before productive admission, with exact prior birth and journal CAS. */
+function prepareSpawn(context, row) {
+    const launch = launchRecord(context, row);
+    if (!launch || launch.value.dispatch_ready || !["starting", "running", "recovery_required"].includes(row.status))
+        return null;
+    const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: JSON.parse(row.manifest_json).resource_home } };
+    const ours = launch.value.pid === process.pid && launch.value.pid_started_at === processStartedAt(process.pid);
+    let token = row.owner_token;
+    if (!ours) {
+        if (row.owner_pid) {
+            if (launch.value.owner?.pid !== row.owner_pid || !dead(launch.value.owner))
+                return null;
+            const records = managedProcessStatus(resourceContext).filter(item => item.operation_id === processOperation(context, row));
+            const owned = records.filter(item => { const metadata = JSON.parse(item.metadata_json); return metadata.owner_pid === row.owner_pid && metadata.owner_pid_started_at === launch.value.owner.pid_started_at; });
+            if (owned.length > 1 || row.process_id && owned[0]?.id !== row.process_id)
+                return null;
+            const record = owned[0];
+            if (records.some(item => item.id !== record?.id && (item.state !== "stopped" || !item.pid_started_at || processTreeIsAlive(item))))
+                return null;
+            if (record) {
+                if (record.kind !== "run-controller" || record.owner_id !== row.controller_id || record.project_id !== row.project_id || record.run_id !== row.run_id || managedProcessRuntimeHome(record, context) !== context.ddFlowHome || row.process_lease_token && record.lease_token !== row.process_lease_token || record.pid !== null && record.pid !== row.owner_pid || processTreeIsAlive(record))
+                    return null;
+                if (record.pid === null) {
+                    const db = getResourceDatabase(JSON.parse(row.manifest_json).resource_home);
+                    if (db.run("UPDATE managed_processes SET pid = ?, pid_started_at = ? WHERE id = ? AND lease_token = ? AND pid IS NULL", [row.owner_pid, launch.value.owner.pid_started_at, record.id, record.lease_token]).changes !== 1)
+                        return null;
+                }
+                retire(resourceContext, row, { id: record.id, lease_token: record.lease_token, ...launch.value.owner });
+            }
+        }
+        else {
+            if (row.process_id || row.process_lease_token || !dead(launch.value))
+                return null;
+            if (launch.value.child && !dead(launch.value.child))
+                return null;
+            if (managedProcessStatus(resourceContext).some(item => item.operation_id === processOperation(context, row) && item.id !== launch.value.retired_process?.id && (item.state !== "stopped" || !item.pid_started_at || processTreeIsAlive(item))))
+                return null;
+        }
+        token = crypto.randomUUID();
+    }
+    else if (row.owner_pid || launch.value.spawn_started)
+        return null;
+    if (launch.value.retired_process)
+        retire(resourceContext, row, launch.value.retired_process);
+    context.db.exec("BEGIN IMMEDIATE");
+    try {
+        assertLaunchAllowed(context, row);
+        const changed = context.db.run(`UPDATE run_controllers SET owner_token = ?, owner_pid = NULL, process_id = NULL, process_lease_token = NULL, status = 'starting', last_error_json = NULL, updated_at = ?
+      WHERE controller_id = ? AND owner_token = ? AND owner_pid IS ? AND process_id IS ? AND status = ? AND state_json = ?
+        AND (SELECT receipt_json FROM run_controller_operations WHERE operation_id = ?) = ?`, [token, context.now(), row.controller_id, row.owner_token, row.owner_pid, row.process_id, row.status, row.state_json, launchId(row), launch.receipt_json]);
+        if (changed.changes !== 1) {
+            context.db.exec("ROLLBACK");
+            return null;
+        }
+        const current = requireController(context, row.controller_id);
+        if (!ours) {
+            recordControllerLauncher(context, current);
+            appendEvent(context, row.controller_id, "launch_owner_reconciled", { prior_launch: launchId(row), generation: row.generation, prior_process_id: row.process_id });
+        }
+        context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.spawn_started', json('true')), updated_at = ? WHERE operation_id = ?", [context.now(), launchId(current)]);
+        context.db.exec("COMMIT");
+        return current;
+    }
+    catch (error) {
+        context.db.exec("ROLLBACK");
+        throw error;
+    }
+}
+/** The physical child records its own birth before resource registration. */
+export function claimControllerProcess(context, row) {
+    const launch = launchRecord(context, row);
+    if (!launch && (!row.process_id || !row.process_lease_token))
+        throw new AppError("controller_owner_mismatch", "Managed registration is not authorized", 1);
+    context.db.exec("BEGIN IMMEDIATE");
+    try {
+        const claimed = context.db.run("UPDATE run_controllers SET owner_pid = ?, status = 'running', updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status = 'starting' AND owner_pid IS NULL", [process.pid, context.now(), row.controller_id, row.owner_token]);
+        if (claimed.changes !== 1)
+            throw new AppError("controller_already_owned", "Managed owner has already claimed this launch", 1);
+        if (launch)
+            context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.owner', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify(birth()), context.now(), launchId(row)]);
+        context.db.exec("COMMIT");
+    }
+    catch (error) {
+        context.db.exec("ROLLBACK");
+        throw error;
+    }
+    if (!row.process_id) {
+        const manifest = JSON.parse(row.manifest_json);
+        const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
+        const log = path.join(row.state_dir, "controller.log");
+        const record = registerManagedProcess(resourceContext, { kind: "run-controller", ownerId: row.controller_id, projectId: row.project_id, runId: row.run_id, operationId: processOperation(context, row), uniqueActiveOperation: true, ownerPid: process.pid, stdoutPath: log, stderrPath: log, metadata: { controller_id: row.controller_id, state_dir: row.state_dir, role: "controller", budget: manifest.budget } });
+        row.process_id = record.id;
+        row.process_lease_token = record.lease_token;
+        if (context.db.run("UPDATE run_controllers SET process_id = ?, process_lease_token = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND owner_pid = ? AND status = 'running'", [record.id, record.lease_token, context.now(), row.controller_id, row.owner_token, process.pid]).changes !== 1)
+            throw new AppError("controller_owner_mismatch", "Managed owner changed during registration", 1);
+    }
+}
+export function markControllerDispatchReady(context, row) {
+    context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.dispatch_ready', json('true')), updated_at = ? WHERE operation_id = ?", [context.now(), launchId(row)]);
+}
+export async function spawnController(context, row) {
+    const admitted = prepareSpawn(context, row);
+    if (!admitted)
+        return controllerReceipt(requireController(context, row.controller_id));
+    row = admitted;
+    const manifest = JSON.parse(row.manifest_json);
+    const run = requireRun(context, row.project_id, row.run_id);
+    const controllerId = row.controller_id, root = row.state_dir, token = row.owner_token;
+    const entry = fileURLToPath(new URL("../cli.js", import.meta.url));
+    try {
+        if (!fs.existsSync(entry))
+            throw new AppError("harness_runtime_unbuilt", "Build the CLI before launching its detached controller", 1);
+        const output = fs.openSync(path.join(root, "controller.log"), "a", 0o600);
+        try {
+            const child = spawn(process.execPath, [entry, "run", "drive", "serve", "--run", run.id, "--project-root", run.project_root, "--controller-id", controllerId, "--json"], { cwd: run.workspace_root, detached: true, stdio: ["ignore", output, output], env: { ...context.env, DD_FLOW_HOME: context.ddFlowHome, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_ENGINE_MODE: "1", DD_FLOW_CONTROLLER_TOKEN: token } });
+            await new Promise((resolve, reject) => { child.once("spawn", resolve); child.once("error", reject); });
+            context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.child', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify({ pid: child.pid, pid_started_at: child.pid ? processStartedAt(child.pid) : null }), context.now(), launchId(row)]);
+            child.unref();
+        }
+        finally {
+            fs.closeSync(output);
+        }
+        return controllerReceipt(requireController(context, controllerId));
+    }
+    catch (error) {
+        context.db.run("UPDATE run_controllers SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status = 'starting' AND owner_pid IS NULL", [JSON.stringify(errorRecord(error)), context.now(), controllerId, token]);
+        throw error;
+    }
+}

package/dist/services/run-controller-recovery.js

@@ -0,0 +1,219 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import { AppError } from "../shared/errors.js";
+import { resolveProjectRoot } from "../storage/paths.js";
+import { requireProjectByRoot } from "./projects.js";
+import { loadExternalStageContext } from "./stage-context.js";
+import { verifyRecoverySnapshot } from "./eval-snapshots.js";
+import { recoveryGuard, resumeRunRecovery } from "./run-recovery.js";
+import { prepareRunRecoveryDispatch } from "./run-recovery-runtime.js";
+import { runtimeHarness } from "./execution-policy.js";
+import { managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processTreeIsAlive } from "./managed-processes.js";
+import { requireRun, requireController, controllerReceipt, appendEvent, saveState, digest, errorRecord } from "./run-controller-state.js";
+import { controllerAdapter, sessionArguments } from "./run-controller-adapter.js";
+import { spawnController, recordControllerLauncher } from "./run-controller-process.js";
+import { readNativeDaemonState, requireManagedDaemonBinding } from "./managed-daemon-binding.js";
+import { runControlStatus } from "./run-control.js";
+import { runtimeBudgetStatus, waitForRuntimeScopeRecovery } from "./runtime-budget.js";
+/** CLI observation is bounded; the physical controller owns the recovery work. */
+export async function resumeRunController(context, input) {
+    if (input.waitMs !== undefined && (!Number.isSafeInteger(input.waitMs) || input.waitMs < 0 || input.waitMs > 60_000))
+        throw new AppError("usage", "--wait-ms must be between 0 and 60000", 2);
+    const intent = recordControllerResume(context, input);
+    const manifest = JSON.parse(intent.controller.manifest_json);
+    const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
+    await spawnController(resourceContext, intent.controller);
+    const deadline = performance.now() + (input.waitMs ?? 0);
+    const status = () => context.db.get("SELECT status FROM run_controller_operations WHERE operation_id = ?", [intent.operationId]).status;
+    while (status() === "accepted" && performance.now() < deadline)
+        await delay(Math.min(100, Math.max(1, deadline - performance.now())));
+    const observation = runControlStatus(context, { ...input, action: "resume" });
+    return { ...observation, controller: controllerReceipt(requireController(context, intent.controller.controller_id)).controller, operation_id: intent.operationId, status: observation.resume_operation.status, ...(intent.retiredProcess ? {} : { reused: true }) };
+}
+/** Retain the logical controller and its receipts; replace only its dead owner. */
+export function recordControllerResume(context, input) {
+    if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(input.requestId) || !input.controlId.trim())
+        throw new AppError("usage", "Resume requires request-id and the exact source control", 2);
+    const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
+    const run = requireRun(context, project.id, input.runId);
+    if (!context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controllers'"))
+        throw new AppError("controller_attach_required", "RUN has no retained managed controller", 1);
+    const request = { action: "control_resume", request_id: input.requestId, control_id: input.controlId };
+    const hash = digest(request);
+    const findPrior = () => {
+        const prior = context.db.get("SELECT o.controller_id, o.operation_id, o.input_hash FROM run_controller_operations o JOIN run_controllers c ON c.controller_id = o.controller_id WHERE c.project_id = ? AND c.run_id = ? AND json_extract(o.input_json, '$.action') = 'control_resume' AND json_extract(o.input_json, '$.request_id') = ?", [project.id, run.id, input.requestId]);
+        if (!prior)
+            return null;
+        if (prior.input_hash !== hash)
+            throw new AppError("runtime_request_conflict", "Resume request id has different inputs", 1);
+        return { controller: requireController(context, prior.controller_id), operationId: prior.operation_id };
+    };
+    const prior = findPrior();
+    if (prior)
+        return prior;
+    try {
+        const row = context.db.get("SELECT * FROM run_controllers WHERE project_id = ? AND run_id = ? AND status NOT IN ('completed','cancelled','superseded')", [project.id, run.id]);
+        if (!row)
+            throw new AppError("controller_attach_required", "RUN has no retained resumable controller", 1);
+        const operationId = `${row.controller_id}:resume:${input.requestId}`;
+        if (!["running", "paused", "waiting_for_user"].includes(run.status))
+            throw new AppError("invalid_run_state", "Resume cannot resurrect a terminal RUN", 1, { status: run.status });
+        const guard = recoveryGuard(context, project.id, run.id);
+        const control = context.db.get("SELECT recovery_id, status FROM run_controls WHERE control_id = ? AND project_id = ? AND run_id = ?", [input.controlId, project.id, run.id]);
+        if (!guard || guard.recovery_id !== control?.recovery_id || guard.status !== "sealed" || control.status !== "settled" || !guard.capture_path || JSON.parse(guard.settlement_json ?? "{}").settled !== true)
+            throw new AppError("run_control_not_resumable", "Resume requires the current settled control and its immutable recovery capture", 1);
+        const manifest = JSON.parse(row.manifest_json), state = JSON.parse(row.state_json);
+        if (!manifest.execution_profile?.settings.execution || !manifest.execution_profile.agent_profiles || !path.isAbsolute(manifest.resource_home) || !path.isAbsolute(manifest.flow_executable))
+            throw new AppError("controller_manifest_invalid", "Resume requires the original frozen controller manifest", 1);
+        if (state.current_session === null || !state.sessions[state.current_session]?.id)
+            throw new AppError("recovery_owner_missing", "Resume requires the retained coordinator Session", 1);
+        for (const [stage, source] of Object.entries(manifest.contexts))
+            loadExternalStageContext({ file: source.file, expectedSha256: source.sha256, stage });
+        verifyRecoverySnapshot(guard.capture_path, { projectRoot: project.root, runId: run.id, recoveryId: guard.recovery_id, workspaceRoot: run.workspace_root });
+        const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
+        const previousOwner = managedProcessStatus(resourceContext).find(record => record.id === row.process_id);
+        if (!previousOwner || previousOwner.lease_token !== row.process_lease_token || previousOwner.kind !== "run-controller" || previousOwner.owner_id !== row.controller_id || previousOwner.project_id !== project.id || previousOwner.run_id !== run.id || managedProcessRuntimeHome(previousOwner, context) !== context.ddFlowHome || !previousOwner.pid || !previousOwner.pid_started_at || processTreeIsAlive(previousOwner))
+            throw new AppError("controller_owner_unsettled", "Resume requires proof that the previous physical controller has exited", 1);
+        const token = crypto.randomUUID(), now = context.now();
+        context.db.exec("BEGIN IMMEDIATE");
+        try {
+            const duplicate = context.db.get("SELECT input_hash, status FROM run_controller_operations WHERE operation_id = ?", [operationId]);
+            if (duplicate) {
+                if (duplicate.input_hash !== hash)
+                    throw new AppError("runtime_request_conflict", "Resume request id has different inputs", 1);
+                context.db.exec("COMMIT");
+                return { controller: requireController(context, row.controller_id), operationId };
+            }
+            const current = recoveryGuard(context, project.id, run.id);
+            if (current?.recovery_id !== guard.recovery_id || current.generation !== guard.generation || current.status !== "sealed" || current.capture_path !== guard.capture_path)
+                throw new AppError("run_control_superseded", "Control changed before resume could be accepted", 1);
+            if (!["running", "paused", "waiting_for_user"].includes(requireRun(context, project.id, run.id).status))
+                throw new AppError("invalid_run_state", "RUN became terminal before resume could be accepted", 1);
+            const changed = context.db.run("UPDATE run_controls SET status = 'resuming', updated_at = ? WHERE control_id = ? AND recovery_id = ? AND status = 'settled'", [now, input.controlId, guard.recovery_id]);
+            if (changed.changes !== 1)
+                throw new AppError("run_control_superseded", "Control changed before resume could be accepted", 1);
+            state.recovery = { control_id: input.controlId, recovery_id: guard.recovery_id, operation_id: operationId };
+            const claimed = context.db.run("UPDATE run_controllers SET generation = ?, status = 'starting', owner_token = ?, owner_pid = NULL, process_id = NULL, process_lease_token = NULL, state_json = ?, last_error_json = NULL, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND state_json = ? AND manifest_json = ? AND process_id = ?", [guard.generation, token, JSON.stringify(state), now, row.controller_id, row.owner_token, row.state_json, row.manifest_json, row.process_id]);
+            if (claimed.changes !== 1)
+                throw new AppError("controller_owner_mismatch", "Controller changed before resume could claim its next physical owner", 1);
+            context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'accepted', ?, ?)", [operationId, row.controller_id, guard.generation, hash, JSON.stringify(request), now, now]);
+            recordControllerLauncher(context, requireController(context, row.controller_id), previousOwner);
+            appendEvent(context, row.controller_id, "resume_requested", { operation_id: operationId, control_id: input.controlId, recovery_id: guard.recovery_id, generation: guard.generation, retired_process_id: previousOwner.id, retained_session_id: state.sessions[state.current_session].id });
+            context.db.exec("COMMIT");
+        }
+        catch (error) {
+            context.db.exec("ROLLBACK");
+            throw error;
+        }
+        return { controller: requireController(context, row.controller_id), operationId, retiredProcess: previousOwner };
+    }
+    catch (error) {
+        // A concurrent identical client may commit between our first lookup and
+        // validation. Its durable request wins even if our preflight saw the fence.
+        const concurrent = findPrior();
+        if (concurrent)
+            return concurrent;
+        throw error;
+    }
+}
+export async function resumeControllerSession(context, row, manifest, state, assertPhysicalOwner) {
+    const recovery = state.recovery;
+    const session = state.current_session === null ? undefined : state.sessions[state.current_session];
+    if (!session?.id)
+        throw new AppError("recovery_owner_missing", "Recovery lost its retained coordinator Session", 1);
+    const executionContext = { ...context, env: { ...context.env, DD_FLOW_BIN: manifest.flow_executable, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
+    const scopeControl = manifest.budget ? runtimeBudgetStatus(executionContext, manifest.budget.scope_id).control : null;
+    const assertRecoveryOwner = () => {
+        assertPhysicalOwner();
+        const guard = recoveryGuard(context, row.project_id, row.run_id);
+        if (guard?.generation !== row.generation || guard.recovery_id !== recovery.recovery_id)
+            throw new AppError("run_recovery_generation_stale", "A newer control superseded this recovery owner", 1);
+        if (!["sealed", "resuming", "recovered"].includes(guard.status) || !context.db.get("SELECT 1 FROM run_controls WHERE control_id = ? AND recovery_id = ? AND status = 'resuming'", [recovery.control_id, recovery.recovery_id]))
+            throw new AppError("run_recovery_guarded", "Recovery no longer owns this operator intent", 1);
+    };
+    assertRecoveryOwner();
+    const readDaemon = () => readNativeDaemonState(session.state_dir);
+    const { state: retired, process: record } = requireManagedDaemonBinding(executionContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id, ownerId: session.owner.owner_id });
+    const call = (args, operationId) => controllerAdapter(executionContext, row, manifest, session, args, assertRecoveryOwner, operationId);
+    if (processTreeIsAlive(record)) {
+        const stopped = await call(["daemon", "stop", "--state-dir", session.state_dir, "--json"]);
+        if (stopped.stopped !== true || stopped.clean !== true)
+            throw new AppError("recovery_native_shutdown_unproven", "Recovery requires a clean native close before replacing its daemon", 1);
+        const deadline = Date.now() + 30_000;
+        while (processTreeIsAlive(record) && Date.now() < deadline) {
+            await delay(100);
+            assertRecoveryOwner();
+        }
+    }
+    if (processTreeIsAlive(record) || readDaemon().shutdown_state !== "clean")
+        throw new AppError("recovery_native_shutdown_unproven", "The previous native daemon is not proven clean and closed", 1);
+    assertRecoveryOwner();
+    session.stopped = false;
+    saveState(context, row, state);
+    const started = await call(["daemon", "start", "--session-id", session.id, ...sessionArguments(context, manifest, session)]);
+    const daemon = readDaemon(), daemonId = daemon.daemon_id ?? daemon.config?.daemonId;
+    if (!daemonId || daemonId === (retired.daemon_id ?? retired.config?.daemonId) || started.daemon_id !== daemonId || started.already_running === true)
+        throw new AppError("recovery_daemon_stale", "Recovery requires a fresh daemon, not reuse of the retired physical owner", 1);
+    assertRecoveryOwner();
+    const scope = { projectRoot: manifest.request.project_root, runId: row.run_id, recoveryId: recovery.recovery_id };
+    resumeRunRecovery(executionContext, { ...scope, daemonId, harness: runtimeHarness(session.profile), nativeSessionId: session.id });
+    const dispatch = prepareRunRecoveryDispatch(executionContext, { ...scope, stateDir: session.state_dir, operationId: `${recovery.operation_id}:ack` });
+    const inspected = await call(["session", "resume", "--session-id", session.id, ...sessionArguments(context, manifest, session)], dispatch.resume_operation_id);
+    if (inspected.settled !== true)
+        throw new AppError("recovery_session_unsettled", "Retained native Session is not ready for same-Session recovery", 1);
+    assertRecoveryOwner();
+    const prepared = { prepared: true, recovery_id: recovery.recovery_id, session_id: session.id, daemon_id: daemonId, process_id: dispatch.process_id, resume_operation_id: dispatch.resume_operation_id };
+    context.db.exec("BEGIN IMMEDIATE");
+    try {
+        assertRecoveryOwner();
+        const binding = requireManagedDaemonBinding(executionContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id, ownerId: session.owner.owner_id });
+        if (binding.process.id !== dispatch.process_id || binding.state.daemon_id !== daemonId || binding.process.state !== "running" || !processIsAlive(binding.process))
+            throw new AppError("recovery_runtime_mismatch", "Prepared recovery lost its physical Session owner", 1);
+        const recorded = context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'accepted'", [JSON.stringify(prepared), context.now(), recovery.operation_id, row.controller_id, row.generation]);
+        if (recorded.changes !== 1)
+            throw new AppError("run_recovery_generation_stale", "Recovery preparation no longer owns its operation", 1);
+        appendEvent(context, row.controller_id, "recovery_prepared", prepared);
+        context.db.exec("COMMIT");
+    }
+    catch (error) {
+        context.db.exec("ROLLBACK");
+        throw error;
+    }
+    if (scopeControl && manifest.budget) {
+        await waitForRuntimeScopeRecovery(executionContext, { scopeId: manifest.budget.scope_id, generation: scopeControl.generation, controllerId: row.controller_id, operationId: recovery.operation_id }, assertRecoveryOwner);
+    }
+    const file = path.join(row.state_dir, `turn-${String(state.turns + 1).padStart(5, "0")}.md`);
+    fs.writeFileSync(file, dispatch.prompt, { flag: "wx", mode: 0o600 });
+    state.turns += 1;
+    saveState(context, row, state);
+    const receipt = await call(["session", "prompt", "--session-id", session.id, "--prompt-file", file, ...sessionArguments(context, manifest, session)], dispatch.operation_id);
+    assertRecoveryOwner();
+    if (recoveryGuard(context, row.project_id, row.run_id)?.status !== "recovered")
+        throw new AppError("trusted_recovery_acceptance_required", "Recovery Turn ended without its native acknowledgement", 1);
+    context.db.exec("BEGIN IMMEDIATE");
+    try {
+        assertRecoveryOwner();
+        context.db.run("UPDATE run_controls SET status = 'resumed', updated_at = ? WHERE control_id = ? AND status = 'resuming'", [context.now(), recovery.control_id]);
+        context.db.run("UPDATE run_controller_operations SET status = 'completed', receipt_json = ?, updated_at = ? WHERE operation_id = ? AND status = 'accepted'", [JSON.stringify({ resumed: true, recovery_id: recovery.recovery_id, session_id: session.id, daemon_id: daemonId }), context.now(), recovery.operation_id]);
+        state.last_receipt = receipt;
+        state.last_continuation = null;
+        delete state.recovery;
+        saveState(context, row, state);
+        appendEvent(context, row.controller_id, "recovery_resumed", { control_id: recovery.control_id, recovery_id: recovery.recovery_id, session_id: session.id, daemon_id: daemonId, generation: row.generation });
+        context.db.exec("COMMIT");
+    }
+    catch (error) {
+        context.db.exec("ROLLBACK");
+        throw error;
+    }
+}
+export function failControllerRecovery(context, row, state, error) {
+    if (state.recovery) {
+        // Native ACK may already have opened this generation before its Turn lost
+        // observation. Retain the accepted evidence but close further dispatch.
+        context.db.run("UPDATE run_recovery_guards SET status = 'resuming', updated_at = ? WHERE recovery_id = ? AND status = 'recovered' AND generation = (SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = ? AND run_id = ?) AND EXISTS (SELECT 1 FROM run_controls WHERE control_id = ? AND recovery_id = run_recovery_guards.recovery_id AND status = 'resuming')", [context.now(), state.recovery.recovery_id, row.project_id, row.run_id, state.recovery.control_id]);
+        context.db.run("UPDATE run_controller_operations SET status = 'failed', error_json = ?, updated_at = ? WHERE operation_id = ? AND status = 'accepted'", [JSON.stringify(errorRecord(error)), context.now(), state.recovery.operation_id]);
+    }
+}

package/dist/services/run-controller-state.js

@@ -0,0 +1,31 @@
+import crypto from "node:crypto";
+import { AppError } from "../shared/errors.js";
+import { runtimeHarness } from "./execution-policy.js";
+export function requireRun(context, projectId, runId) {
+    const run = context.db.get("SELECT id, project_id, project_root, workspace_root, run_root, status, index_json FROM runs WHERE project_id = ? AND id = ?", [projectId, runId]);
+    if (!run?.run_root)
+        throw new AppError("not_found", "Managed RUN has no materialized root", 1, { run_id: runId });
+    return run;
+}
+export function requireController(context, id) {
+    const row = context.db.get("SELECT * FROM run_controllers WHERE controller_id = ?", [id]);
+    if (!row)
+        throw new AppError("not_found", "Managed controller was not found", 1, { controller_id: id });
+    return row;
+}
+export function controllerReceipt(row) {
+    const state = JSON.parse(row.state_json);
+    const manifest = JSON.parse(row.manifest_json);
+    return { ok: true, schema_id: "dd-flow/run-controller-receipt@1", controller: { controller_id: row.controller_id, request_id: row.request_id, project_id: row.project_id, run_id: row.run_id, generation: row.generation, runtime_budget: manifest.budget ?? null, status: row.status, process_id: row.process_id, owner_pid: row.owner_pid, state_dir: row.state_dir, stage: state.stage, sessions: state.sessions.map(session => ({ session_id: session.id, harness: runtimeHarness(session.profile), profile_id: session.profile.id, state_dir: session.state_dir, stopped: session.stopped === true })), error: row.last_error_json ? JSON.parse(row.last_error_json) : null } };
+}
+export function appendEvent(context, id, type, data) { context.db.run("INSERT INTO run_controller_events (controller_id, event_type, data_json, created_at) VALUES (?, ?, ?, ?)", [id, type, JSON.stringify(data), context.now()]); }
+export function saveState(context, row, state) { context.db.run("UPDATE run_controllers SET state_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ?", [JSON.stringify(state), context.now(), row.controller_id, row.owner_token]); }
+export function digest(value) { return digestBytes(JSON.stringify(value)); }
+export function digestBytes(value) { return crypto.createHash("sha256").update(value).digest("hex"); }
+export function errorRecord(error) {
+    return {
+        code: error instanceof AppError ? error.code : "controller_failed",
+        message: error instanceof Error ? error.message : String(error),
+        ...(error instanceof AppError && Object.keys(error.details).length ? { details: error.details } : {})
+    };
+}