@deksden-com/dd-flow-cli 0.9.0-beta.15 → 0.9.0-beta.151

package/dist/services/run-controller-process.js

@@ -0,0 +1,205 @@
+import fs from "node:fs";
+import crypto from "node:crypto";
+import path from "node:path";
+import { spawn } from "node:child_process";
+import { AppError } from "../shared/errors.js";
+import { validateCheckEnvironment } from "./check-environment.js";
+import { getResourceDatabase } from "../storage/database.js";
+import { finishManagedProcess, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processStartedAt, processTreeIsAlive, registerManagedProcess } from "./managed-processes.js";
+import { setFlowRunRecoveryPaused } from "./runs.js";
+import { assertRunMutationAllowed, assertRunRecoveryAllowed, recoveryGuard } from "./run-recovery.js";
+import { requireRun, requireController, controllerReceipt, appendEvent, digest, errorRecord } from "./run-controller-state.js";
+const launchId = (row) => `${row.controller_id}:owner-launch:${digest(row.owner_token)}`;
+const processOperation = (context, row) => `controller:${digest({ database: context.db.path, project: row.project_id, run: row.run_id })}`;
+const birth = () => ({ pid: process.pid, pid_started_at: processStartedAt(process.pid) });
+const dead = (value) => Boolean(value && Number.isSafeInteger(value.pid) && value.pid > 0 && typeof value.pid_started_at === "string" && value.pid_started_at && !processIsAlive(value));
+/** Called inside the RUN intent transaction, before the client can disappear. */
+export function recordControllerLauncher(context, row, retired, identity = birth()) {
+    const request = { action: "controller_owner_launch", owner_id: launchId(row) };
+    const receipt = { ...identity, registration: "controller_self@1", ...(retired ? { retired_process: { id: retired.id, lease_token: retired.lease_token, pid: retired.pid, pid_started_at: retired.pid_started_at } } : {}) };
+    const now = context.now();
+    context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'completed', ?, ?, ?)", [launchId(row), row.controller_id, row.generation, digest(request), JSON.stringify(request), JSON.stringify(receipt), now, now]);
+}
+function launchRecord(context, row) {
+    const record = context.db.get("SELECT receipt_json FROM run_controller_operations WHERE operation_id = ?", [launchId(row)]);
+    if (!record)
+        return null;
+    const value = JSON.parse(record.receipt_json);
+    return value?.registration === "controller_self@1" ? { ...record, value } : null;
+}
+export function assertLaunchAllowed(context, row) {
+    const run = requireRun(context, row.project_id, row.run_id);
+    if (!["running", "paused", "waiting_for_user"].includes(run.status))
+        throw new AppError("invalid_run_state", "A terminal RUN cannot start a physical controller", 1);
+    if (JSON.parse(row.state_json).recovery)
+        assertControllerRecoveryAllowed(context, row);
+    else
+        assertRunMutationAllowed(context, row.project_id, row.run_id, row.generation);
+}
+/** Existing owners must retain the recovery fence while finishing terminal capture/settlement. */
+export function assertControllerRecoveryAllowed(context, row) {
+    const state = JSON.parse(row.state_json);
+    if (!state.recovery) {
+        assertRunRecoveryAllowed(context, row.project_id, row.run_id, row.generation);
+        return;
+    }
+    const guard = recoveryGuard(context, row.project_id, row.run_id);
+    if (guard?.generation !== row.generation || guard.recovery_id !== state.recovery.recovery_id || !["sealed", "resuming", "recovered"].includes(guard.status)
+        || !context.db.get("SELECT 1 FROM run_controls WHERE control_id = ? AND recovery_id = ? AND status = 'resuming'", [state.recovery.control_id, state.recovery.recovery_id]))
+        throw new AppError("run_control_superseded", "Controller launch no longer owns the accepted recovery", 1);
+}
+/** An admitted physical owner may renew while closing a terminal RUN. This
+ * does not admit a new owner or release a recovery/generation fence. */
+export function assertControllerLeaseAllowed(context, row) {
+    const current = requireController(context, row.controller_id);
+    if (current.owner_token !== row.owner_token || current.generation !== row.generation
+        || current.owner_pid !== process.pid || current.process_id !== row.process_id
+        || current.process_lease_token !== row.process_lease_token || !current.process_id || !current.process_lease_token
+        || !["running", "capturing", "waiting_for_user", "waiting_for_context", "waiting_for_children", "waiting_for_capacity", "waiting_for_merge"].includes(current.status))
+        throw new AppError("controller_owner_mismatch", "Lease renewal no longer owns this physical controller", 1);
+    const run = requireRun(context, current.project_id, current.run_id);
+    if ((["done", "failed", "cancelled"].includes(run.status) || current.status === "capturing") && launchRecord(context, current)?.value.dispatch_ready === true) {
+        assertControllerRecoveryAllowed(context, current);
+        return;
+    }
+    assertLaunchAllowed(context, current);
+}
+function retire(context, row, identity) {
+    const record = managedProcessStatus(context).find(item => item.id === identity.id);
+    if (!record || record.kind !== "run-controller" || record.owner_id !== row.controller_id || record.project_id !== row.project_id || record.run_id !== row.run_id
+        || managedProcessRuntimeHome(record, context) !== context.ddFlowHome || record.lease_token !== identity.lease_token || record.pid !== identity.pid || record.pid_started_at !== identity.pid_started_at || !dead(identity) || processTreeIsAlive(record)
+        || !finishManagedProcess(context, { id: record.id, leaseToken: record.lease_token, state: "stopped", reason: "controller_launch_owner_retired" }))
+        throw new AppError("controller_owner_unsettled", "Prior controller identity is not proven dead and retired", 1);
+}
+/** Retry only before productive admission, with exact prior birth and journal CAS. */
+function prepareSpawn(context, row) {
+    const launch = launchRecord(context, row);
+    if (!launch || launch.value.dispatch_ready || !["starting", "running", "recovery_required"].includes(row.status))
+        return null;
+    const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: JSON.parse(row.manifest_json).resource_home } };
+    const ours = launch.value.pid === process.pid && launch.value.pid_started_at === processStartedAt(process.pid);
+    let token = row.owner_token;
+    if (!ours) {
+        if (row.owner_pid) {
+            if (launch.value.owner?.pid !== row.owner_pid || !dead(launch.value.owner))
+                return null;
+            const records = managedProcessStatus(resourceContext).filter(item => item.operation_id === processOperation(context, row));
+            const owned = records.filter(item => { const metadata = JSON.parse(item.metadata_json); return metadata.owner_pid === row.owner_pid && metadata.owner_pid_started_at === launch.value.owner.pid_started_at; });
+            if (owned.length > 1 || row.process_id && owned[0]?.id !== row.process_id)
+                return null;
+            const record = owned[0];
+            if (records.some(item => item.id !== record?.id && (item.state !== "stopped" || !item.pid_started_at || processTreeIsAlive(item))))
+                return null;
+            if (record) {
+                if (record.kind !== "run-controller" || record.owner_id !== row.controller_id || record.project_id !== row.project_id || record.run_id !== row.run_id || managedProcessRuntimeHome(record, context) !== context.ddFlowHome || row.process_lease_token && record.lease_token !== row.process_lease_token || record.pid !== null && record.pid !== row.owner_pid || processTreeIsAlive(record))
+                    return null;
+                if (record.pid === null) {
+                    const db = getResourceDatabase(JSON.parse(row.manifest_json).resource_home);
+                    if (db.run("UPDATE managed_processes SET pid = ?, pid_started_at = ? WHERE id = ? AND lease_token = ? AND pid IS NULL", [row.owner_pid, launch.value.owner.pid_started_at, record.id, record.lease_token]).changes !== 1)
+                        return null;
+                }
+                retire(resourceContext, row, { id: record.id, lease_token: record.lease_token, ...launch.value.owner });
+            }
+        }
+        else {
+            if (row.process_id || row.process_lease_token || !dead(launch.value))
+                return null;
+            if (launch.value.child && !dead(launch.value.child))
+                return null;
+            if (managedProcessStatus(resourceContext).some(item => item.operation_id === processOperation(context, row) && item.id !== launch.value.retired_process?.id && (item.state !== "stopped" || !item.pid_started_at || processTreeIsAlive(item))))
+                return null;
+        }
+        token = crypto.randomUUID();
+    }
+    else if (row.owner_pid || launch.value.spawn_started)
+        return null;
+    if (launch.value.retired_process)
+        retire(resourceContext, row, launch.value.retired_process);
+    const launcher = birth();
+    return context.db.writeTransaction(() => {
+        assertLaunchAllowed(context, row);
+        const changed = context.db.run(`UPDATE run_controllers SET owner_token = ?, owner_pid = NULL, process_id = NULL, process_lease_token = NULL, status = 'starting', last_error_json = NULL, updated_at = ?
+      WHERE controller_id = ? AND owner_token = ? AND owner_pid IS ? AND process_id IS ? AND status = ? AND state_json = ?
+        AND (SELECT receipt_json FROM run_controller_operations WHERE operation_id = ?) = ?`, [token, context.now(), row.controller_id, row.owner_token, row.owner_pid, row.process_id, row.status, row.state_json, launchId(row), launch.receipt_json]);
+        if (changed.changes !== 1)
+            return null;
+        const current = requireController(context, row.controller_id);
+        if (!ours) {
+            recordControllerLauncher(context, current, undefined, launcher);
+            appendEvent(context, row.controller_id, "launch_owner_reconciled", { prior_launch: launchId(row), generation: row.generation, prior_process_id: row.process_id });
+        }
+        context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.spawn_started', json('true')), updated_at = ? WHERE operation_id = ?", [context.now(), launchId(current)]);
+        return current;
+    });
+}
+/** The physical child records its own birth before resource registration. */
+export function claimControllerProcess(context, row, owner = birth()) {
+    const launch = launchRecord(context, row);
+    if (!launch && (!row.process_id || !row.process_lease_token))
+        throw new AppError("controller_owner_mismatch", "Managed registration is not authorized", 1);
+    context.db.writeTransaction(() => {
+        // Recovery fences are reconciled by serve's retained failure boundary;
+        // only terminal admission must reject before claiming a physical owner.
+        if (["done", "failed", "cancelled"].includes(requireRun(context, row.project_id, row.run_id).status))
+            throw new AppError("invalid_run_state", "A terminal RUN cannot start a physical controller", 1);
+        const claimed = context.db.run("UPDATE run_controllers SET owner_pid = ?, status = 'running', updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status = 'starting' AND owner_pid IS NULL", [process.pid, context.now(), row.controller_id, row.owner_token]);
+        if (claimed.changes !== 1)
+            throw new AppError("controller_already_owned", "Managed owner has already claimed this launch", 1);
+        if (launch)
+            context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.owner', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify(owner), context.now(), launchId(row)]);
+    });
+    row.owner_pid = process.pid;
+    if (!row.process_id) {
+        const manifest = JSON.parse(row.manifest_json);
+        const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
+        const log = path.join(row.state_dir, "controller.log");
+        const record = registerManagedProcess(resourceContext, { kind: "run-controller", ownerId: row.controller_id, projectId: row.project_id, runId: row.run_id, operationId: processOperation(context, row), uniqueActiveOperation: true, ownerPid: process.pid, stdoutPath: log, stderrPath: log, metadata: { controller_id: row.controller_id, state_dir: row.state_dir, role: "controller", budget: manifest.budget } });
+        row.process_id = record.id;
+        row.process_lease_token = record.lease_token;
+        if (context.db.run("UPDATE run_controllers SET process_id = ?, process_lease_token = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND owner_pid = ? AND status = 'running'", [record.id, record.lease_token, context.now(), row.controller_id, row.owner_token, process.pid]).changes !== 1)
+            throw new AppError("controller_owner_mismatch", "Managed owner changed during registration", 1);
+    }
+}
+export function markControllerDispatchReady(context, row) {
+    assertLaunchAllowed(context, row);
+    context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.dispatch_ready', json('true')), updated_at = ? WHERE operation_id = ?", [context.now(), launchId(row)]);
+}
+export async function spawnController(context, row) {
+    const admitted = prepareSpawn(context, row);
+    if (!admitted)
+        return controllerReceipt(requireController(context, row.controller_id));
+    row = admitted;
+    const manifest = JSON.parse(row.manifest_json);
+    const run = requireRun(context, row.project_id, row.run_id);
+    const controllerId = row.controller_id, root = row.state_dir, token = row.owner_token;
+    try {
+        const entry = (await import("./engines.js")).boundRunEngineEntrypoint(context, { projectRoot: run.project_root, runId: run.id });
+        const output = fs.openSync(path.join(root, "controller.log"), "a", 0o600);
+        try {
+            const child = spawn(process.execPath, [entry, "run", "drive", "serve", "--run", run.id, "--project-root", run.project_root, "--controller-id", controllerId, "--json"], { cwd: run.workspace_root, detached: true, stdio: ["ignore", output, output], env: { ...context.env, ...validateCheckEnvironment(manifest.check_environment), DD_FLOW_HOME: context.ddFlowHome, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_ENGINE_MODE: "1", DD_FLOW_CONTROLLER_TOKEN: token } });
+            await new Promise((resolve, reject) => { child.once("spawn", resolve); child.once("error", reject); });
+            context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.child', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify({ pid: child.pid, pid_started_at: child.pid ? processStartedAt(child.pid) : null }), context.now(), launchId(row)]);
+            child.unref();
+        }
+        finally {
+            fs.closeSync(output);
+        }
+        return controllerReceipt(requireController(context, controllerId));
+    }
+    catch (error) {
+        try {
+            context.db.writeTransaction(() => {
+                const changed = context.db.run("UPDATE run_controllers SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND status = 'starting' AND owner_pid IS NULL", [JSON.stringify(errorRecord(error)), context.now(), controllerId, token]);
+                if (changed.changes && (recoveryGuard(context, row.project_id, row.run_id)?.generation ?? 0) === row.generation)
+                    setFlowRunRecoveryPaused(context, row.project_id, row.run_id, true);
+            });
+        }
+        catch (failure) {
+            const secondary = { status_persistence_error: errorRecord(failure) };
+            if (error instanceof AppError)
+                throw new AppError(error.code, error.message, error.exitCode, { ...error.details, ...secondary });
+            throw new AppError("controller_spawn_failed", error instanceof Error ? error.message : String(error), 1, { cause: errorRecord(error), ...secondary });
+        }
+        throw error;
+    }
+}

package/dist/services/run-controller-recovery.js

@@ -0,0 +1,295 @@
+import crypto from "node:crypto";
+import { nativeShutdownConfirmed } from "../harness-runtime/lib/native-children.mjs";
+import fs from "node:fs";
+import path from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import { AppError } from "../shared/errors.js";
+import { resolveProjectRoot } from "../storage/paths.js";
+import { requireProjectByRoot } from "./projects.js";
+import { loadExternalStageContext } from "./stage-context.js";
+import { verifyRecoverySnapshot } from "./eval-snapshots.js";
+import { recoveryGuard, resumeRunRecovery } from "./run-recovery.js";
+import { prepareRunRecoveryDispatch, retainRecoveryCapacityIntent } from "./run-recovery-runtime.js";
+import { runtimeHarness } from "./execution-policy.js";
+import { selectControllerExecution } from "./controller-execution.js";
+import { managedProcessBinding, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processTreeIsAlive, processStartedAt } from "./managed-processes.js";
+import { requireRun, requireController, controllerReceipt, appendEvent, saveState, digest, errorRecord } from "./run-controller-state.js";
+import { controllerAdapter, controllerEnvironment, sessionArguments } from "./run-controller-adapter.js";
+import { runHarnessAdapter } from "./harness-adapter.js";
+import { continueProviderCapacity } from "./provider-capacity.js";
+import { spawnController, recordControllerLauncher } from "./run-controller-process.js";
+import { readNativeDaemonState, requireManagedDaemonBinding } from "./managed-daemon-binding.js";
+import { managedInvocationContext } from "./lifecycle-invocations.js";
+import { runControlStatus } from "./run-control.js";
+import { runtimeBudgetStatus, waitForRuntimeScopeRecovery } from "./runtime-budget.js";
+import { prepareControllerPublication } from "./run-controller-capture.js";
+/** CLI observation is bounded; the physical controller owns the recovery work. */
+export async function resumeRunController(context, input) {
+    if (input.waitMs !== undefined && (!Number.isSafeInteger(input.waitMs) || input.waitMs < 0 || input.waitMs > 60_000))
+        throw new AppError("usage", "--wait-ms must be between 0 and 60000", 2);
+    const intent = recordControllerResume(context, input);
+    const manifest = JSON.parse(intent.controller.manifest_json);
+    const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
+    await spawnController(resourceContext, intent.controller);
+    const deadline = performance.now() + (input.waitMs ?? 0);
+    const status = () => context.db.get("SELECT status FROM run_controller_operations WHERE operation_id = ?", [intent.operationId]).status;
+    while (status() === "accepted" && performance.now() < deadline)
+        await delay(Math.min(100, Math.max(1, deadline - performance.now())));
+    const observation = runControlStatus(context, { ...input, action: "resume" });
+    return { ...observation, controller: controllerReceipt(requireController(context, intent.controller.controller_id)).controller, operation_id: intent.operationId, status: observation.resume_operation.status, ...(intent.retiredProcess ? {} : { reused: true }) };
+}
+/** Retain the logical controller and its receipts; replace only its dead owner. */
+export function recordControllerResume(context, input) {
+    return prepareControllerResume(context, input)();
+}
+/** Verify immutable recovery evidence before a caller reserves the registry
+ * writer. The returned operation rechecks SQL ownership under the RUN CAS. */
+export function prepareControllerResume(context, input) {
+    if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(input.requestId) || !input.controlId.trim())
+        throw new AppError("usage", "Resume requires request-id and the exact source control", 2);
+    const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
+    const run = requireRun(context, project.id, input.runId);
+    if (!context.db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'run_controllers'"))
+        throw new AppError("controller_attach_required", "RUN has no retained managed controller", 1);
+    const request = { action: "control_resume", request_id: input.requestId, control_id: input.controlId };
+    const hash = digest(request);
+    const findPrior = () => {
+        const prior = context.db.get("SELECT o.controller_id, o.operation_id, o.input_hash FROM run_controller_operations o JOIN run_controllers c ON c.controller_id = o.controller_id WHERE c.project_id = ? AND c.run_id = ? AND json_extract(o.input_json, '$.action') = 'control_resume' AND json_extract(o.input_json, '$.request_id') = ?", [project.id, run.id, input.requestId]);
+        if (!prior)
+            return null;
+        if (prior.input_hash !== hash)
+            throw new AppError("runtime_request_conflict", "Resume request id has different inputs", 1);
+        return { controller: requireController(context, prior.controller_id), operationId: prior.operation_id };
+    };
+    const prior = findPrior();
+    if (prior)
+        return () => findPrior() ?? prior;
+    try {
+        const row = context.db.get("SELECT * FROM run_controllers WHERE project_id = ? AND run_id = ? AND status NOT IN ('completed','cancelled','superseded')", [project.id, run.id]);
+        if (!row)
+            throw new AppError("controller_attach_required", "RUN has no retained resumable controller", 1);
+        const operationId = `${row.controller_id}:resume:${input.requestId}`;
+        if (!["running", "paused", "waiting_for_user"].includes(run.status))
+            throw new AppError("invalid_run_state", "Resume cannot resurrect a terminal RUN", 1, { status: run.status });
+        const guard = recoveryGuard(context, project.id, run.id);
+        const control = context.db.get("SELECT recovery_id, status FROM run_controls WHERE control_id = ? AND project_id = ? AND run_id = ?", [input.controlId, project.id, run.id]);
+        if (!guard || guard.recovery_id !== control?.recovery_id || guard.status !== "sealed" || control.status !== "settled" || !guard.capture_path || JSON.parse(guard.settlement_json ?? "{}").settled !== true)
+            throw new AppError("run_control_not_resumable", "Resume requires the current settled control and its immutable recovery capture", 1);
+        const manifest = JSON.parse(row.manifest_json), state = JSON.parse(row.state_json);
+        selectControllerExecution(context, run, manifest);
+        if (!path.isAbsolute(manifest.resource_home) || !path.isAbsolute(manifest.flow_executable))
+            throw new AppError("controller_manifest_invalid", "Resume requires the original frozen controller manifest", 1);
+        if (state.current_session === null || !state.sessions[state.current_session]?.id)
+            throw new AppError("recovery_owner_missing", "Resume requires the retained coordinator Session", 1);
+        for (const [stage, source] of Object.entries(manifest.contexts))
+            loadExternalStageContext({ file: source.file, expectedSha256: source.sha256, stage });
+        verifyRecoverySnapshot(guard.capture_path, { projectRoot: project.root, runId: run.id, recoveryId: guard.recovery_id, workspaceRoot: run.workspace_root });
+        const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
+        const previousOwner = managedProcessStatus(resourceContext).find(record => record.id === row.process_id);
+        if (!previousOwner || previousOwner.lease_token !== row.process_lease_token || previousOwner.kind !== "run-controller" || previousOwner.owner_id !== row.controller_id || previousOwner.project_id !== project.id || previousOwner.run_id !== run.id || managedProcessRuntimeHome(previousOwner, context) !== context.ddFlowHome || !previousOwner.pid || !previousOwner.pid_started_at || processTreeIsAlive(previousOwner))
+            throw new AppError("controller_owner_unsettled", "Resume requires proof that the previous physical controller has exited", 1);
+        const token = crypto.randomUUID(), now = context.now();
+        const launcher = { pid: process.pid, pid_started_at: processStartedAt(process.pid) };
+        return () => context.db.writeTransaction(() => {
+            const duplicate = context.db.get("SELECT input_hash, status FROM run_controller_operations WHERE operation_id = ?", [operationId]);
+            if (duplicate) {
+                if (duplicate.input_hash !== hash)
+                    throw new AppError("runtime_request_conflict", "Resume request id has different inputs", 1);
+                return { controller: requireController(context, row.controller_id), operationId };
+            }
+            const current = recoveryGuard(context, project.id, run.id);
+            if (current?.recovery_id !== guard.recovery_id || current.generation !== guard.generation || current.status !== "sealed" || current.capture_path !== guard.capture_path)
+                throw new AppError("run_control_superseded", "Control changed before resume could be accepted", 1);
+            const currentOwner = managedProcessStatus(resourceContext).find(record => record.id === previousOwner.id);
+            if (!currentOwner || digest(managedProcessBinding(currentOwner, manifest.resource_home)) !== digest(managedProcessBinding(previousOwner, manifest.resource_home)))
+                throw new AppError("controller_owner_unsettled", "Retired controller binding changed while recovery was prepared", 1);
+            if (!["running", "paused", "waiting_for_user"].includes(requireRun(context, project.id, run.id).status))
+                throw new AppError("invalid_run_state", "RUN became terminal before resume could be accepted", 1);
+            const changed = context.db.run("UPDATE run_controls SET status = 'resuming', updated_at = ? WHERE control_id = ? AND recovery_id = ? AND status = 'settled'", [now, input.controlId, guard.recovery_id]);
+            if (changed.changes !== 1)
+                throw new AppError("run_control_superseded", "Control changed before resume could be accepted", 1);
+            state.recovery = { control_id: input.controlId, recovery_id: guard.recovery_id, operation_id: operationId };
+            const claimed = context.db.run("UPDATE run_controllers SET generation = ?, status = 'starting', owner_token = ?, owner_pid = NULL, process_id = NULL, process_lease_token = NULL, state_json = ?, last_error_json = NULL, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND state_json = ? AND manifest_json = ? AND process_id = ?", [guard.generation, token, JSON.stringify(state), now, row.controller_id, row.owner_token, row.state_json, row.manifest_json, row.process_id]);
+            if (claimed.changes !== 1)
+                throw new AppError("controller_owner_mismatch", "Controller changed before resume could claim its next physical owner", 1);
+            context.db.run("INSERT INTO run_controller_operations (operation_id, controller_id, generation, input_hash, input_json, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'accepted', ?, ?)", [operationId, row.controller_id, guard.generation, hash, JSON.stringify(request), now, now]);
+            recordControllerLauncher(context, requireController(context, row.controller_id), previousOwner, launcher);
+            appendEvent(context, row.controller_id, "resume_requested", { operation_id: operationId, control_id: input.controlId, recovery_id: guard.recovery_id, generation: guard.generation, retired_process_id: previousOwner.id, retained_session_id: state.sessions[state.current_session].id });
+            return { controller: requireController(context, row.controller_id), operationId, retiredProcess: previousOwner };
+        });
+    }
+    catch (error) {
+        // A concurrent identical client may commit between our first lookup and
+        // validation. Its durable request wins even if our preflight saw the fence.
+        const concurrent = findPrior();
+        if (concurrent)
+            return () => concurrent;
+        throw error;
+    }
+}
+export async function resumeControllerSession(context, row, manifest, state, assertPhysicalOwner) {
+    const recovery = state.recovery;
+    const session = state.current_session === null ? undefined : state.sessions[state.current_session];
+    if (!session?.id)
+        throw new AppError("recovery_owner_missing", "Recovery lost its retained coordinator Session", 1);
+    let executionContext = { ...context, env: { ...context.env, DD_FLOW_BIN: manifest.flow_executable, DD_FLOW_RESOURCE_HOME: manifest.resource_home } };
+    const scopeControl = manifest.budget ? runtimeBudgetStatus(executionContext, manifest.budget.scope_id).control : null;
+    const assertRecoveryOwner = (physical = true) => {
+        if (physical)
+            assertPhysicalOwner();
+        const guard = recoveryGuard(context, row.project_id, row.run_id);
+        if (guard?.generation !== row.generation || guard.recovery_id !== recovery.recovery_id)
+            throw new AppError("run_recovery_generation_stale", "A newer control superseded this recovery owner", 1);
+        if (!["sealed", "resuming", "recovered"].includes(guard.status) || !context.db.get("SELECT 1 FROM run_controls WHERE control_id = ? AND recovery_id = ? AND status = 'resuming'", [recovery.control_id, recovery.recovery_id]))
+            throw new AppError("run_recovery_guarded", "Recovery no longer owns this operator intent", 1);
+    };
+    assertRecoveryOwner();
+    const readDaemon = () => readNativeDaemonState(session.state_dir);
+    const { state: retired, process: record } = requireManagedDaemonBinding(executionContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id, ownerId: session.owner.owner_id });
+    const call = (args, operationId) => controllerAdapter(executionContext, row, manifest, session, args, assertRecoveryOwner, operationId);
+    if (processTreeIsAlive(record)) {
+        const stopped = await call(["daemon", "stop", "--state-dir", session.state_dir, "--json"]);
+        if (!retired.daemon_id || stopped.stopped !== true || !nativeShutdownConfirmed(stopped, { daemonId: retired.daemon_id }))
+            throw new AppError("recovery_native_shutdown_unproven", "Recovery requires a clean native close before replacing its daemon", 1);
+        const deadline = Date.now() + 30_000;
+        while (processTreeIsAlive(record) && Date.now() < deadline) {
+            await delay(100);
+            assertRecoveryOwner();
+        }
+    }
+    if (processTreeIsAlive(record) || readDaemon().shutdown_state !== "clean")
+        throw new AppError("recovery_native_shutdown_unproven", "The previous native daemon is not proven clean and closed", 1);
+    assertRecoveryOwner();
+    session.stopped = false;
+    saveState(context, row, state);
+    const started = await call(["daemon", "start", "--session-id", session.id, ...sessionArguments(context, manifest, session)]);
+    const daemon = readDaemon(), daemonId = daemon.daemon_id ?? daemon.config?.daemonId;
+    if (!daemonId || daemonId === (retired.daemon_id ?? retired.config?.daemonId) || started.daemon_id !== daemonId || started.already_running === true)
+        throw new AppError("recovery_daemon_stale", "Recovery requires a fresh daemon, not reuse of the retired physical owner", 1);
+    executionContext = managedInvocationContext(executionContext, { harness: runtimeHarness(session.profile), stateDir: session.state_dir, sessionId: session.id,
+        projectRoot: manifest.request.project_root, runId: row.run_id, generation: row.generation });
+    assertRecoveryOwner();
+    const scope = { projectRoot: manifest.request.project_root, runId: row.run_id, recoveryId: recovery.recovery_id };
+    resumeRunRecovery(executionContext, { ...scope, daemonId, harness: runtimeHarness(session.profile), nativeSessionId: session.id });
+    const dispatch = prepareRunRecoveryDispatch(executionContext, { ...scope, stateDir: session.state_dir, operationId: `${recovery.operation_id}:ack` });
+    const inspected = await call(["session", "resume", "--session-id", session.id, ...sessionArguments(context, manifest, session)], dispatch.resume_operation_id);
+    if (inspected.settled !== true)
+        throw new AppError("recovery_session_unsettled", "Retained native Session is not ready for same-Session recovery", 1);
+    assertRecoveryOwner();
+    const prepared = { prepared: true, recovery_id: recovery.recovery_id, session_id: session.id, daemon_id: daemonId, process_id: dispatch.process_id, resume_operation_id: dispatch.resume_operation_id };
+    const binding = requireManagedDaemonBinding(executionContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id, ownerId: session.owner.owner_id });
+    if (binding.process.id !== dispatch.process_id || binding.state.daemon_id !== daemonId || binding.process.state !== "running" || !processIsAlive(binding.process))
+        throw new AppError("recovery_runtime_mismatch", "Prepared recovery lost its physical Session owner", 1);
+    const assertPreparedOwner = prepareControllerPublication(executionContext, row, assertPhysicalOwner);
+    context.db.writeTransaction(() => {
+        assertPreparedOwner();
+        assertRecoveryOwner(false);
+        const currentNative = managedProcessStatus(executionContext).find(item => item.id === binding.process.id);
+        if (!currentNative || currentNative.state !== "running" || currentNative.lease_expires_at <= context.now() || digest(managedProcessBinding(currentNative, manifest.resource_home)) !== digest(managedProcessBinding(binding.process, manifest.resource_home)))
+            throw new AppError("recovery_runtime_mismatch", "Prepared recovery lost its current native lease", 1);
+        const recorded = context.db.run("UPDATE run_controller_operations SET receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'accepted'", [JSON.stringify(prepared), context.now(), recovery.operation_id, row.controller_id, row.generation]);
+        if (recorded.changes !== 1)
+            throw new AppError("run_recovery_generation_stale", "Recovery preparation no longer owns its operation", 1);
+        appendEvent(context, row.controller_id, "recovery_prepared", prepared);
+    });
+    if (scopeControl && manifest.budget) {
+        await waitForRuntimeScopeRecovery(executionContext, { scopeId: manifest.budget.scope_id, generation: scopeControl.generation, controllerId: row.controller_id, operationId: recovery.operation_id }, assertRecoveryOwner);
+    }
+    const file = path.join(row.state_dir, `turn-${String(state.turns + 1).padStart(5, "0")}.md`);
+    fs.writeFileSync(file, dispatch.prompt, { flag: "wx", mode: 0o600 });
+    state.turns += 1;
+    saveState(context, row, state);
+    const sendAck = (operationId, prompt) => {
+        let promptFile = file;
+        if (operationId !== dispatch.operation_id) {
+            promptFile = path.join(row.state_dir, `turn-${String(state.turns + 1).padStart(5, "0")}.md`);
+            fs.writeFileSync(promptFile, prompt, { flag: "wx", mode: 0o600 });
+            state.turns += 1;
+            saveState(context, row, state);
+        }
+        return call(["session", "prompt", "--session-id", session.id, "--prompt-file", promptFile, ...sessionArguments(context, manifest, session)], operationId);
+    };
+    let receipt;
+    try {
+        receipt = await sendAck(dispatch.operation_id, dispatch.prompt);
+    }
+    catch (error) {
+        const continuationText = `Continue the same retained Session and recovery generation. First inspect the current RUN recovery state; only if the exact ACK still awaits acceptance, execute the exact standalone acceptance command below once. Never deliver it again after acceptance. Then stop at the original Stage or Work-graph boundary.\n\n${dispatch.prompt}`;
+        const inspect = (args, operationId) => runHarnessAdapter({ executable: manifest.adapters[runtimeHarness(session.profile)].executable,
+            args, env: controllerEnvironment(executionContext, manifest, session), runtimeOwner: session.owner, operationId, timeoutMs: 30_000 });
+        const resumed = await continueProviderCapacity({ error, sessionId: session.id, operationId: dispatch.operation_id, allowSettledDescendants: true, continuationText,
+            retainedIntents: () => {
+                const binding = context.db.get("SELECT runtime_dispatch_json FROM run_recovery_bindings WHERE recovery_id = ?", [recovery.recovery_id]);
+                return binding?.runtime_dispatch_json ? JSON.parse(binding.runtime_dispatch_json).capacity_intents ?? [] : [];
+            },
+            retainedFailures: () => {
+                const binding = context.db.get("SELECT runtime_dispatch_json FROM run_recovery_bindings WHERE recovery_id = ?", [recovery.recovery_id]);
+                return binding?.runtime_dispatch_json ? JSON.parse(binding.runtime_dispatch_json).capacity_failures ?? [] : [];
+            },
+            retainFailure: (_id, refusal) => {
+                assertRecoveryOwner();
+                context.db.writeTransaction(() => {
+                    const binding = context.db.get("SELECT runtime_dispatch_json FROM run_recovery_bindings WHERE recovery_id = ? AND daemon_id = ? AND status = 'prepared'", [recovery.recovery_id, daemonId]);
+                    if (!binding)
+                        throw new AppError("recovery_capacity_owner_changed", "Recovery binding changed before refusal retention", 1);
+                    const retained = JSON.parse(binding.runtime_dispatch_json);
+                    retained.capacity_failures = [...(retained.capacity_failures ?? []), refusal];
+                    const changed = context.db.run("UPDATE run_recovery_bindings SET runtime_dispatch_json = ? WHERE recovery_id = ? AND daemon_id = ? AND status = 'prepared' AND runtime_dispatch_json = ?", [JSON.stringify(retained), recovery.recovery_id, daemonId, binding.runtime_dispatch_json]);
+                    if (changed.changes !== 1)
+                        throw new AppError("recovery_capacity_owner_changed", "Recovery binding changed during refusal retention", 1);
+                });
+            },
+            inspectOperation: id => inspect(["daemon", "operation", "--state-dir", session.state_dir, "--operation-id", id, "--json"], `${id}:capacity-observe`),
+            inspectSession: turnId => inspect(["session", "inspect", "--session-id", session.id, ...(turnId ? ["--turn-id", turnId] : []), ...sessionArguments(context, manifest, session)], `${dispatch.operation_id}:capacity-inspect`),
+            assertCurrent: () => {
+                assertRecoveryOwner();
+                const guard = recoveryGuard(context, row.project_id, row.run_id);
+                if (guard?.status === "recovered")
+                    return;
+                const binding = context.db.get("SELECT status, daemon_id FROM run_recovery_bindings WHERE recovery_id = ?", [recovery.recovery_id]);
+                const ack = context.db.get("SELECT status FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ?", [recovery.operation_id, row.controller_id, row.generation]);
+                if (guard?.status !== "resuming" || binding?.status !== "prepared" || binding.daemon_id !== daemonId || ack?.status !== "accepted")
+                    throw new AppError("recovery_capacity_owner_changed", "Recovery ACK was accepted or superseded during capacity wait", 1);
+                const current = requireManagedDaemonBinding(executionContext, { stateDir: session.state_dir, projectId: row.project_id, runId: row.run_id, ownerId: session.owner.owner_id });
+                const metadata = JSON.parse(current.process.metadata_json);
+                if (current.process.id !== dispatch.process_id || current.state.daemon_id !== daemonId || current.process.state !== "running"
+                    || current.process.lease_expires_at <= context.now() || metadata.run_generation !== row.generation || !processIsAlive(current.process))
+                    throw new AppError("recovery_capacity_owner_changed", "Replacement native owner changed during capacity wait", 1);
+            },
+            decision: (boundary) => {
+                if (boundary && (boundary.lifecycle_operation !== "recovery_accept" || boundary.recovery_id !== recovery.recovery_id
+                    || !context.db.get("SELECT 1 FROM run_recovery_bindings b JOIN hook_events h ON h.event_key = b.hook_event_key WHERE b.recovery_id = ? AND b.status = 'accepted' AND b.daemon_id = ? AND b.provider_session_id = ? AND h.project_id = ? AND h.status = 'claimed' AND h.event_key = ?", [recovery.recovery_id, daemonId, session.id, row.project_id, boundary.native_hook_event_key])))
+                    return "reconciliation_blocked";
+                return recoveryGuard(context, row.project_id, row.run_id)?.status === "recovered" ? "completed" : "continue";
+            },
+            retainIntent: intent => retainRecoveryCapacityIntent(executionContext, { projectId: row.project_id, runId: row.run_id, recoveryId: recovery.recovery_id,
+                controllerId: row.controller_id, generation: row.generation, daemonId, intent, prompt: continuationText }),
+            prompt: (operationId, prompt) => sendAck(operationId, prompt) });
+        if (!resumed)
+            throw error;
+        receipt = resumed;
+    }
+    assertRecoveryOwner();
+    if (recoveryGuard(context, row.project_id, row.run_id)?.status !== "recovered")
+        throw new AppError("trusted_recovery_acceptance_required", "Recovery Turn ended without its native acknowledgement", 1);
+    const assertAcceptedOwner = prepareControllerPublication(executionContext, row, assertPhysicalOwner);
+    context.db.writeTransaction(() => {
+        assertAcceptedOwner();
+        assertRecoveryOwner(false);
+        context.db.run("UPDATE run_controls SET status = 'resumed', updated_at = ? WHERE control_id = ? AND status = 'resuming'", [context.now(), recovery.control_id]);
+        context.db.run("UPDATE run_controller_operations SET status = 'completed', receipt_json = ?, updated_at = ? WHERE operation_id = ? AND status = 'accepted'", [JSON.stringify({ resumed: true, recovery_id: recovery.recovery_id, session_id: session.id, daemon_id: daemonId }), context.now(), recovery.operation_id]);
+        state.last_receipt = receipt;
+        state.last_continuation = null;
+        delete state.recovery;
+        saveState(context, row, state);
+        appendEvent(context, row.controller_id, "recovery_resumed", { control_id: recovery.control_id, recovery_id: recovery.recovery_id, session_id: session.id, daemon_id: daemonId, generation: row.generation });
+    });
+}
+export function failControllerRecovery(context, row, state, error) {
+    if (state.recovery) {
+        // Native ACK may already have opened this generation before its Turn lost
+        // observation. Retain the accepted evidence but close further dispatch.
+        context.db.run("UPDATE run_recovery_guards SET status = 'resuming', updated_at = ? WHERE recovery_id = ? AND status = 'recovered' AND generation = (SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = ? AND run_id = ?) AND EXISTS (SELECT 1 FROM run_controls WHERE control_id = ? AND recovery_id = run_recovery_guards.recovery_id AND status = 'resuming')", [context.now(), state.recovery.recovery_id, row.project_id, row.run_id, state.recovery.control_id]);
+        context.db.run("UPDATE run_controller_operations SET status = 'failed', error_json = ?, updated_at = ? WHERE operation_id = ? AND status = 'accepted'", [JSON.stringify(errorRecord(error)), context.now(), state.recovery.operation_id]);
+    }
+}

package/dist/services/run-controller-state.js

@@ -0,0 +1,97 @@
+import crypto from "node:crypto";
+import { AppError, errorRecord as structuredErrorRecord } from "../shared/errors.js";
+import { runtimeHarness } from "./execution-policy.js";
+import { managedProcessRuntimeHome, managedProcessStatus, processTreeIsAlive } from "./managed-processes.js";
+export function requireRun(context, projectId, runId) {
+    const run = context.db.get("SELECT id, project_id, flow_kind, project_root, workspace_root, run_root, status, index_json FROM runs WHERE project_id = ? AND id = ?", [projectId, runId]);
+    if (!run?.run_root)
+        throw new AppError("not_found", "Managed RUN has no materialized root", 1, { run_id: runId });
+    return run;
+}
+export function requireController(context, id) {
+    const row = context.db.get("SELECT * FROM run_controllers WHERE controller_id = ?", [id]);
+    if (!row)
+        throw new AppError("not_found", "Managed controller was not found", 1, { controller_id: id });
+    return row;
+}
+export function controllerReceipt(row) {
+    const state = JSON.parse(row.state_json);
+    const manifest = JSON.parse(row.manifest_json);
+    return { ok: true, schema_id: "dd-flow/run-controller-receipt@1", controller: { controller_id: row.controller_id, request_id: row.request_id, project_id: row.project_id, run_id: row.run_id, generation: row.generation, runtime_budget: manifest.budget ?? null, status: row.status, process_id: row.process_id, owner_pid: row.owner_pid, state_dir: row.state_dir, stage: state.stage, sessions: state.sessions.map(session => ({ session_id: session.id, harness: runtimeHarness(session.profile), profile_id: session.profile.id, state_dir: session.state_dir, journal: session.journal, stopped: session.stopped === true })), error: row.last_error_json ? JSON.parse(row.last_error_json) : null } };
+}
+/** Read-only causal diagnostics, not a settlement or capture authorization. */
+export function terminalControllerFinalization(context, project, runId, runStatus) {
+    if (!["done", "failed", "cancelled"].includes(runStatus) || !context.db.get("SELECT 1 FROM sqlite_schema WHERE name = 'run_controllers'"))
+        return null;
+    const row = context.db.get("SELECT * FROM run_controllers WHERE project_id = ? AND run_id = ? ORDER BY created_at DESC, controller_id DESC LIMIT 1", [project.id, runId]);
+    if (!row?.last_error_json || !["recovery_required", "capture_failed"].includes(row.status))
+        return null;
+    const error = JSON.parse(row.last_error_json);
+    if (error.details?.phase !== "terminal_finalization" || error.details.project_root !== project.root || error.details.run_id !== runId || error.details.controller_id !== row.controller_id)
+        return null;
+    let owner = "unknown";
+    try {
+        const manifest = JSON.parse(row.manifest_json);
+        const records = managedProcessStatus({ ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: manifest.resource_home } });
+        const record = records.find(item => item.id === row.process_id);
+        if (record && record.kind === "run-controller" && record.owner_id === row.controller_id && record.project_id === project.id && record.run_id === runId
+            && record.lease_token === row.process_lease_token && record.pid === row.owner_pid && record.pid && record.pid_started_at
+            && managedProcessRuntimeHome(record, context) === context.ddFlowHome) {
+            if (processTreeIsAlive(record))
+                owner = "live";
+            else if (["stopped", "failed"].includes(record.state) && record.finished_at)
+                owner = "absent";
+        }
+        // Any other still-owned controller record makes absence unproven.
+        if (owner === "absent" && records.some(item => item.id !== row.process_id && item.kind === "run-controller" && item.project_id === project.id && item.run_id === runId
+            && managedProcessRuntimeHome(item, context) === context.ddFlowHome && (!item.pid_started_at || !["stopped", "failed"].includes(item.state) || processTreeIsAlive(item))))
+            owner = "unknown";
+    }
+    catch { /* Missing registry or unreadable identity is uncertainty, never death. */ }
+    const control = context.db.get("SELECT 1 FROM sqlite_schema WHERE name = 'run_controls'")
+        && context.db.get("SELECT 1 FROM run_controls WHERE project_id = ? AND run_id = ? AND status IN ('requested','settled','resuming')", [project.id, runId]);
+    const capture = context.db.get("SELECT 1 FROM run_controller_operations WHERE controller_id = ? AND generation = ? AND status = 'completed' AND json_extract(input_json, '$.action') = 'boundary_capture' AND json_type(input_json, '$.stage_entry') = 'null'", [row.controller_id, row.generation]);
+    return { schema_id: "dd-flow/terminal-finalization@1", issuer: "dd-flow", phase: "terminal_finalization", project_root: project.root, run_id: runId,
+        controller_id: row.controller_id, generation: row.generation, controller_status: row.status, run_status: runStatus,
+        status: !control && owner === "absent" ? "blocked" : "pending", owner, capture: capture ? "recorded" : "missing", error };
+}
+export function appendEvent(context, id, type, data) { context.db.run("INSERT INTO run_controller_events (controller_id, event_type, data_json, created_at) VALUES (?, ?, ?, ?)", [id, type, JSON.stringify(data), context.now()]); }
+export function saveState(context, row, state) {
+    const changed = context.db.run("UPDATE run_controllers SET state_json = ?, updated_at = ? WHERE controller_id = ? AND owner_token = ? AND generation = ? AND status NOT IN ('completed','cancelled','superseded')", [JSON.stringify(state), context.now(), row.controller_id, row.owner_token, row.generation]);
+    if (changed.changes !== 1)
+        throw new AppError("controller_owner_mismatch", "Controller state owner or generation changed", 1);
+}
+export function retainControllerCapacityIntent(context, row, state, intent, sessionId, kind, assertCurrent) {
+    const root = intent.root_operation_id;
+    if (!root || intent.session_id !== sessionId || !intent.refusal)
+        throw new AppError("capacity_continuation_conflict", "Capacity intent has no exact controller root/Session/refusal", 1);
+    const pending = { root_operation_id: root, predecessor: intent.predecessor, successor: intent.operation_id, session_id: sessionId, generation: row.generation, kind };
+    context.db.writeTransaction(() => {
+        assertCurrent();
+        const owner = context.db.get("SELECT status FROM run_controllers WHERE controller_id = ? AND owner_token = ? AND generation = ?", [row.controller_id, row.owner_token, row.generation]);
+        if (!owner || !["running", "waiting_for_user", "waiting_for_children", "waiting_for_capacity", "waiting_for_merge"].includes(owner.status))
+            throw new AppError("controller_owner_mismatch", "Controller no longer owns continuation preparation", 1);
+        const changed = context.db.run("UPDATE run_controller_operations SET status = 'failed', receipt_json = ?, updated_at = ? WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'outcome_unknown'", [JSON.stringify({ capacity_root: root, capacity_failure: intent.refusal, capacity_intent: intent }), context.now(), intent.predecessor, row.controller_id, row.generation]);
+        if (changed.changes !== 1)
+            throw new AppError("capacity_continuation_blocked", "Failed native operation changed before continuation", 1);
+        saveState(context, row, { ...state, pending_capacity: pending });
+        appendEvent(context, row.controller_id, "capacity_continuation", intent);
+    });
+    state.pending_capacity = pending;
+}
+export function retainControllerContinuation(state, fingerprint, error) {
+    if (state.last_continuation === fingerprint)
+        throw new AppError(error.code, error.message, 1, error.details);
+    state.last_continuation = fingerprint;
+}
+export function digest(value) { return digestBytes(JSON.stringify(value)); }
+export function digestBytes(value) { return crypto.createHash("sha256").update(value).digest("hex"); }
+export function errorRecord(error) {
+    const record = structuredErrorRecord(error);
+    let typed = false;
+    try {
+        typed = typeof error?.code === "string";
+    }
+    catch { /* Preserve a serializable controller failure for unreadable diagnostics. */ }
+    return { ...record, code: typed ? record.code : "controller_failed" };
+}