@deksden-com/dd-flow-cli 0.9.0-beta.7 → 0.9.0-beta.74

package/dist/services/run-control-worker.js

@@ -0,0 +1,363 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import { spawn } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { setTimeout as delay } from "node:timers/promises";
+import { AppError } from "../shared/errors.js";
+import { resolveProjectRoot } from "../storage/paths.js";
+import { requireProjectByRoot } from "./projects.js";
+import { getResourceDatabase } from "../storage/database.js";
+import { confirmManagedProcess, finishManagedProcess, heartbeatManagedProcess, managedProcessRuntimeHome, processIsAlive, processStartedAt, processTreeIsAlive, registerManagedProcess, resourceHome } from "./managed-processes.js";
+import { interruptControlledSessions, observeControlledSessions, recordRunControlIntent, runControlStatus } from "./run-control.js";
+import { recoveryGuard, recordRecoveryCapture, sealRunRecovery } from "./run-recovery.js";
+import { createEvalRunSnapshot, verifyRecoverySnapshot } from "./eval-snapshots.js";
+import { recoveryObservationBudget, renewRecoveryObservation, withRecoveryObservation } from "./recovery-observation-budget.js";
+export async function reconcileRunControl(context, input) {
+    const status = runControlStatus(context, input);
+    const control = status.control;
+    if (!control?.current || !["requested", "settled"].includes(control.status))
+        throw new AppError("run_control_state_changed", "Reconciliation requires the current draining/sealed control", 1);
+    renewRecoveryObservation({}, input.requestId); // Validate before claiming anything.
+    if (control.status === "requested")
+        await launchControlWorker(context, { ...input, reconcileId: input.requestId });
+    return runControlStatus(context, input);
+}
+/** The caller publishes intent and launches a control owner; it never owns drain. */
+export async function requestRunControl(context, input) {
+    if (input.waitMs !== undefined && (!Number.isSafeInteger(input.waitMs) || input.waitMs < 0 || input.waitMs > 60_000))
+        throw new AppError("usage", "--wait-ms must be between 0 and 60000", 2);
+    const receipt = recordRunControlIntent(context, input);
+    const projectReceipt = () => {
+        const observation = runControlStatus(context, { ...input, controlId: typeof receipt.control_id === "string" ? receipt.control_id : null });
+        return { ...observation, status: observation.control?.status ?? receipt.status, action: input.action, ...(receipt.reused === true ? { reused: true } : {}) };
+    };
+    if (!receipt.control_id || receipt.status !== "requested")
+        return projectReceipt();
+    const scope = { projectRoot: input.projectRoot, runId: input.runId, controlId: String(receipt.control_id), ...(input.reconcileId ? { reconcileId: input.reconcileId } : {}) };
+    await launchControlWorker(context, scope);
+    const deadline = performance.now() + (input.waitMs ?? 0);
+    while (performance.now() < deadline) {
+        const worker = context.db.get("SELECT status FROM run_control_workers WHERE control_id = ?", [scope.controlId]);
+        const current = context.db.get("SELECT status FROM run_controls WHERE control_id = ?", [scope.controlId]);
+        if (current?.status !== "requested" || ["failed", "completed", "superseded"].includes(worker?.status ?? ""))
+            break;
+        await delay(Math.min(100, Math.max(1, deadline - performance.now())));
+    }
+    return projectReceipt();
+}
+async function launchControlWorker(context, input) {
+    const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
+    const run = context.db.get("SELECT run_root, workspace_root FROM runs WHERE project_id = ? AND id = ?", [project.id, input.runId]);
+    if (!run?.run_root)
+        throw new AppError("run_control_unmaterialized", "Control intent is fenced, but the RUN has no directory for its owner", 1);
+    const token = crypto.randomUUID(), directory = path.join(run.run_root, "controls", input.controlId), now = context.now();
+    const home = resourceHome(context);
+    // Reclaim only a proven dead launcher or admitted worker. Lease expiry
+    // never proves death or authorizes a second physical owner.
+    let claimed = false;
+    context.db.beginWriteTransaction();
+    try {
+        claimed = context.db.run("INSERT OR IGNORE INTO run_control_workers (control_id, owner_token, status, resource_home, state_dir, created_at, updated_at) VALUES (?, ?, 'starting', ?, ?, ?, ?)", [input.controlId, token, home, directory, now, now]).changes === 1;
+        if (claimed) {
+            recordControlLauncher(context, input.controlId, token);
+            if (input.reconcileId)
+                context.db.run("UPDATE run_control_workers SET snapshot_json = ? WHERE control_id = ?", [JSON.stringify(renewRecoveryObservation({}, input.reconcileId)), input.controlId]);
+        }
+        context.db.exec("COMMIT");
+    }
+    catch (error) {
+        context.db.exec("ROLLBACK");
+        throw error;
+    }
+    if (!claimed && !reclaimControlWorker(context, input, project.id, token, home))
+        return;
+    try {
+        const entry = fileURLToPath(new URL("../cli.js", import.meta.url));
+        if (!fs.existsSync(entry))
+            throw new AppError("harness_runtime_unbuilt", "Build the CLI before launching its detached control owner", 1);
+        fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
+        const log = path.join(directory, "control.log");
+        const output = fs.openSync(log, "a", 0o600);
+        try {
+            const env = { ...context.env, DD_FLOW_HOME: context.ddFlowHome, DD_FLOW_RESOURCE_HOME: home, DD_FLOW_ENGINE_MODE: "1", DD_FLOW_CONTROL_TOKEN: token };
+            delete env.DD_FLOW_CONTROLLER_TOKEN;
+            delete env.DD_FLOW_SCOPE_CONTROL_TOKEN;
+            const child = spawn(process.execPath, [entry, "run", "control", "serve", "--run", input.runId, "--project-root", project.root, "--control-id", input.controlId, "--json"], { cwd: run.workspace_root, detached: true, stdio: ["ignore", output, output], env });
+            await new Promise((resolve, reject) => { child.once("spawn", resolve); child.once("error", reject); });
+            context.db.run("UPDATE run_control_operations SET receipt_json = json_set(receipt_json, '$.child', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify({ pid: child.pid, pid_started_at: child.pid ? processStartedAt(child.pid) : null }), context.now(), launchOperationId(input.controlId, token)]);
+            child.unref();
+        }
+        finally {
+            fs.closeSync(output);
+        }
+        // The detached worker is the durable owner of the stop.  Do not let the
+        // caller tear down its RUN/home before that owner has either registered
+        // its managed-process lease or recorded a startup failure.
+        await waitForControlWorkerAdmission(context, input.controlId, token);
+    }
+    catch (error) {
+        context.db.run("UPDATE run_control_workers SET status = 'failed', error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'starting' AND owner_pid IS NULL", [JSON.stringify(failure(error)), context.now(), input.controlId, token]);
+        // The durable intent remains accepted even if its worker could not start.
+    }
+}
+async function waitForControlWorkerAdmission(context, controlId, ownerToken, timeoutMs = 5_000) {
+    const deadline = Date.now() + timeoutMs;
+    for (;;) {
+        const worker = context.db.get("SELECT status, process_id, owner_token FROM run_control_workers WHERE control_id = ?", [controlId]);
+        if (!worker || worker.owner_token !== ownerToken)
+            return;
+        // A running worker without a resource lease is still in the vulnerable
+        // registration gap; keep the parent alive until that gap closes.
+        if (worker.process_id || ["failed", "completed", "superseded"].includes(worker.status))
+            return;
+        if (Date.now() >= deadline)
+            throw new AppError("run_control_worker_start_timeout", "Control worker did not publish its managed-process lease", 1, { control_id: controlId, timeout_ms: timeoutMs });
+        await delay(25);
+    }
+}
+const launchOperationId = (controlId, token) => `${controlId}:worker-launch:${crypto.createHash("sha256").update(token).digest("hex")}`;
+function recordControlLauncher(context, controlId, token) {
+    const now = context.now();
+    context.db.run("INSERT INTO run_control_operations (operation_id, control_id, target_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, 'completed', ?, ?, ?)", [launchOperationId(controlId, token), controlId, JSON.stringify({ kind: "control_worker_launch_claim" }), JSON.stringify({ pid: process.pid, pid_started_at: processStartedAt(process.pid), registration: "worker_self@1" }), now, now]);
+}
+function reclaimControlWorker(context, input, projectId, token, home) {
+    const previous = context.db.get("SELECT * FROM run_control_workers WHERE control_id = ?", [input.controlId]);
+    if (!previous || !["starting", "running", "failed"].includes(previous.status) || previous.resource_home !== home)
+        return false;
+    const saved = JSON.parse(previous.snapshot_json ?? "{}");
+    const requested = input.reconcileId ? renewRecoveryObservation(saved, input.reconcileId) : saved;
+    if (recoveryObservationBudget(requested).state().remaining_ms === 0)
+        return false;
+    const registry = fs.existsSync(path.join(home, "runtime.sqlite")) ? getResourceDatabase(home) : null;
+    let record = previous.process_id ? registry?.get("SELECT * FROM managed_processes WHERE id = ?", [previous.process_id]) : null;
+    const operationId = launchOperationId(input.controlId, previous.owner_token);
+    const launch = context.db.get("SELECT receipt_json FROM run_control_operations WHERE operation_id = ?", [operationId]);
+    const launchIdentity = JSON.parse(launch?.receipt_json ?? "null");
+    if (previous.owner_pid && launchIdentity?.registration === "worker_self@1") {
+        const claim = context.db.get("SELECT receipt_json FROM run_control_operations WHERE operation_id = ?", [`${operationId}:worker-claim`]);
+        const identity = JSON.parse(claim?.receipt_json ?? "null");
+        if (!identity || !Number.isSafeInteger(identity.pid) || identity.pid < 1 || identity.pid !== previous.owner_pid || typeof identity.pid_started_at !== "string" || !identity.pid_started_at || processIsAlive(identity))
+            return false;
+        const records = registry?.all("SELECT * FROM managed_processes WHERE operation_id = ?", [`control:${input.controlId}`]) ?? [];
+        const owned = records.filter(item => {
+            const metadata = JSON.parse(item.metadata_json);
+            return metadata.owner_pid === identity.pid && metadata.owner_pid_started_at === identity.pid_started_at;
+        });
+        if (owned.length > 1 || previous.process_id && owned[0]?.id !== previous.process_id)
+            return false;
+        record = owned[0] ?? null;
+        if (record && (record.kind !== "run-control" || record.owner_id !== input.controlId || record.project_id !== projectId || record.run_id !== input.runId || managedProcessRuntimeHome(record, context) !== context.ddFlowHome || previous.process_lease_token && record.lease_token !== previous.process_lease_token || record.pid !== null && record.pid !== identity.pid || processTreeIsAlive(record)))
+            return false;
+        if (records.some(item => item.id !== record?.id && (item.state !== "stopped" || !item.pid_started_at || processTreeIsAlive(item))))
+            return false;
+        if (record && record.pid === null) {
+            // Registration describes this very worker, not an as-yet-unidentified
+            // spawned child. Retain its prior RUN-store birth evidence on retirement.
+            if (registry.run("UPDATE managed_processes SET pid = ?, pid_started_at = ? WHERE id = ? AND lease_token = ? AND pid IS NULL", [identity.pid, identity.pid_started_at, record.id, record.lease_token]).changes !== 1)
+                return false;
+            record.pid = identity.pid;
+            record.pid_started_at = identity.pid_started_at;
+        }
+    }
+    else if (previous.process_id) {
+        if (!record || !previous.owner_pid || !previous.process_lease_token || record.lease_token !== previous.process_lease_token || record.kind !== "run-control" || record.owner_id !== input.controlId || record.project_id !== projectId || record.run_id !== input.runId || record.pid !== previous.owner_pid || !record.pid_started_at || managedProcessRuntimeHome(record, context) !== context.ddFlowHome || processTreeIsAlive(record))
+            return false;
+    }
+    else {
+        // No worker has claimed RUN authority. Prove launcher death and reject
+        // unresolved registrations, including a legacy crash before linkage.
+        if (previous.owner_pid || previous.process_lease_token)
+            return false;
+        const identity = launchIdentity;
+        if (!identity || !Number.isSafeInteger(identity.pid) || identity.pid < 1 || typeof identity.pid_started_at !== "string" || !identity.pid_started_at || processIsAlive(identity))
+            return false;
+        if (identity.child && (!Number.isSafeInteger(identity.child.pid) || identity.child.pid < 1 || identity.child.pid_started_at !== null && typeof identity.child.pid_started_at !== "string" || processIsAlive(identity.child)))
+            return false;
+        if (registry?.all("SELECT * FROM managed_processes WHERE operation_id = ?", [`control:${input.controlId}`]).some(item => item.state !== "stopped" || !item.pid || !item.pid_started_at || processTreeIsAlive(item)))
+            return false;
+    }
+    // The resource store and RUN store are separate: retirement is idempotent,
+    // and a crash before the RUN CAS leaves the same dead owner reconcilable.
+    if (record && !finishManagedProcess(context, { id: record.id, leaseToken: record.lease_token, state: "stopped", reason: "control_owner_crash_reconciled" }))
+        return false;
+    context.db.beginWriteTransaction();
+    try {
+        const changed = context.db.run(`UPDATE run_control_workers SET owner_token = ?, owner_pid = NULL, status = 'starting', process_id = NULL, process_lease_token = NULL, updated_at = ?
+      WHERE control_id = ? AND owner_token = ? AND process_id IS ? AND owner_pid IS ? AND status = ?
+        AND (SELECT receipt_json FROM run_control_operations WHERE operation_id = ?) IS ?
+        AND EXISTS (SELECT 1 FROM run_controls c JOIN run_recovery_guards g ON g.recovery_id = c.recovery_id
+          WHERE c.control_id = run_control_workers.control_id AND c.status = 'requested' AND g.status IN ('draining','sealed')
+            AND g.generation = (SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = g.project_id AND run_id = g.run_id))`, [token, context.now(), input.controlId, previous.owner_token, previous.process_id, previous.owner_pid, previous.status, operationId, launch?.receipt_json ?? null]);
+        if (changed.changes !== 1) {
+            context.db.exec("ROLLBACK");
+            return false;
+        }
+        if (input.reconcileId) {
+            const snapshot = renewRecoveryObservation(JSON.parse(previous.snapshot_json ?? "{}"), input.reconcileId);
+            context.db.run("UPDATE run_control_workers SET snapshot_json = ?, error_json = NULL WHERE control_id = ? AND owner_token = ?", [JSON.stringify(snapshot), input.controlId, token]);
+        }
+        const now = context.now();
+        context.db.run("INSERT INTO run_control_operations (operation_id, control_id, target_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, 'completed', ?, ?, ?)", [`${launchOperationId(input.controlId, previous.owner_token)}:reconciled`, input.controlId, JSON.stringify({ kind: "control_worker_reconciliation", process_id: record?.id ?? null, resource_home: home }), JSON.stringify({ process_id: record?.id ?? null, pid: record?.pid ?? null, pid_started_at: record?.pid_started_at ?? null, prior_status: previous.status, snapshot: JSON.parse(previous.snapshot_json ?? "null"), error: JSON.parse(previous.error_json ?? "null") }), now, now]);
+        recordControlLauncher(context, input.controlId, token);
+        context.db.exec("COMMIT");
+        return true;
+    }
+    catch (error) {
+        context.db.exec("ROLLBACK");
+        throw error;
+    }
+}
+export async function serveRunControl(context, input) {
+    const project = requireProjectByRoot(context, resolveProjectRoot(input.projectRoot));
+    const intent = context.db.get("SELECT * FROM run_controls WHERE control_id = ? AND project_id = ? AND run_id = ?", [input.controlId, project.id, input.runId]);
+    const worker = context.db.get("SELECT * FROM run_control_workers WHERE control_id = ?", [input.controlId]);
+    if (!intent || !worker || worker.owner_token !== context.env.DD_FLOW_CONTROL_TOKEN)
+        throw new AppError("run_control_owner_mismatch", "Control serve does not own this request", 1);
+    const launch = context.db.get("SELECT receipt_json FROM run_control_operations WHERE operation_id = ?", [launchOperationId(input.controlId, worker.owner_token)]);
+    const selfRegister = JSON.parse(launch?.receipt_json ?? "null")?.registration === "worker_self@1";
+    if ((!worker.process_id || !worker.process_lease_token) && !selfRegister)
+        throw new AppError("run_control_owner_mismatch", "Control registration is not authorized", 1);
+    // Physical identity is committed with the sole RUN claim, before the worker
+    // registers itself in the separate resource store or performs any dispatch.
+    context.db.beginWriteTransaction();
+    try {
+        const claimed = context.db.run("UPDATE run_control_workers SET status = 'running', owner_pid = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'starting' AND owner_pid IS NULL", [process.pid, context.now(), input.controlId, worker.owner_token]);
+        if (claimed.changes !== 1)
+            throw new AppError("run_control_already_owned", "A control owner has already claimed this request", 1);
+        const now = context.now();
+        context.db.run("INSERT INTO run_control_operations (operation_id, control_id, target_json, status, receipt_json, created_at, updated_at) VALUES (?, ?, ?, 'completed', ?, ?, ?)", [`${launchOperationId(input.controlId, worker.owner_token)}:worker-claim`, input.controlId, JSON.stringify({ kind: "control_worker_self_claim" }), JSON.stringify({ pid: process.pid, pid_started_at: processStartedAt(process.pid) }), now, now]);
+        context.db.exec("COMMIT");
+    }
+    catch (error) {
+        context.db.exec("ROLLBACK");
+        throw error;
+    }
+    const resourceContext = { ...context, env: { ...context.env, DD_FLOW_RESOURCE_HOME: worker.resource_home } };
+    let leaseLost = false;
+    const heartbeat = setInterval(() => {
+        try {
+            if (!heartbeatManagedProcess(resourceContext, { id: worker.process_id, leaseToken: worker.process_lease_token }))
+                leaseLost = true;
+        }
+        catch {
+            leaseLost = true;
+        }
+    }, 10_000);
+    heartbeat.unref();
+    try {
+        if (!worker.process_id) {
+            const log = path.join(worker.state_dir, "control.log");
+            const record = registerManagedProcess(resourceContext, { kind: "run-control", ownerId: input.controlId, projectId: project.id, runId: input.runId, operationId: `control:${input.controlId}`, uniqueActiveOperation: true, ownerPid: process.pid, stdoutPath: log, stderrPath: log, metadata: { control_id: input.controlId } });
+            worker.process_id = record.id;
+            worker.process_lease_token = record.lease_token;
+            context.db.run("UPDATE run_control_workers SET process_id = ?, process_lease_token = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND owner_pid = ? AND status = 'running'", [record.id, record.lease_token, context.now(), input.controlId, worker.owner_token, process.pid]);
+        }
+        if (!worker.process_lease_token)
+            throw new AppError("run_control_owner_mismatch", "Control owner has no resource lease", 1);
+        confirmManagedProcess(resourceContext, { id: worker.process_id, leaseToken: worker.process_lease_token, pid: process.pid, processGroupId: process.pid, ownerPid: process.pid });
+        let priorObservation = null;
+        try {
+            priorObservation = worker.snapshot_json ? JSON.parse(worker.snapshot_json) : null;
+        }
+        catch { /* A malformed prior snapshot cannot grant a new budget. */
+            priorObservation = { recovery_observation: { remaining_ms: 0 } };
+        }
+        const observation = recoveryObservationBudget(priorObservation);
+        for (;;) {
+            if (leaseLost)
+                throw new AppError("run_control_lease_lost", "Control worker lost its physical ownership lease", 1);
+            const current = context.db.get("SELECT * FROM run_controls WHERE control_id = ?", [input.controlId]);
+            const guard = recoveryGuard(context, project.id, input.runId);
+            if (current.status !== "requested" || guard?.recovery_id !== intent.recovery_id || !["draining", "sealed"].includes(guard.status)) {
+                const status = current.status === "settled" ? "completed" : "superseded";
+                context.db.run("UPDATE run_control_workers SET status = ?, updated_at = ? WHERE control_id = ? AND owner_token = ?", [status, context.now(), input.controlId, worker.owner_token]);
+                return { ok: true, control_id: input.controlId, status };
+            }
+            if (observation.exhausted()) {
+                const latest = context.db.get("SELECT snapshot_json FROM run_control_workers WHERE control_id = ?", [input.controlId]);
+                let retained = {};
+                try {
+                    retained = JSON.parse(latest?.snapshot_json ?? "{}") ?? {};
+                }
+                catch { /* Retain a blocked status even when the old projection is malformed. */ }
+                const snapshot = { ...retained, status: "recovery_blocked", settled: false, pending_reasons: ["recovery_observation_budget_exhausted"],
+                    error: { code: "recovery_observation_budget_exhausted", message: "Control settlement has no confirmed result within the active observation budget" }, recovery_observation: observation.state() };
+                context.db.run("UPDATE run_control_workers SET status = 'failed', snapshot_json = ?, error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify(snapshot), JSON.stringify(snapshot.error), context.now(), input.controlId, worker.owner_token]);
+                return { ok: true, control_id: input.controlId, ...snapshot };
+            }
+            if (guard.status === "sealed") {
+                // Capture retries are local evidence work. Never dispatch another
+                // native interrupt/inspect merely because publication previously failed.
+                const settlement = JSON.parse(guard.settlement_json ?? "{}");
+                try {
+                    if (settlement.settled !== true)
+                        throw new AppError("run_control_settlement_missing", "Sealed control has no settled tree receipt", 1);
+                    const root = path.resolve(context.env.DD_FLOW_RECOVERY_HOME ?? path.join(os.homedir(), ".dd-flow-recovery"));
+                    const capture = guard.capture_path ?? path.join(root, crypto.createHash("sha256").update(context.ddFlowHome).digest("hex"), project.id, input.runId, guard.recovery_id);
+                    const scope = { projectRoot: project.root, runId: input.runId, recoveryId: guard.recovery_id };
+                    if (!fs.existsSync(capture))
+                        createEvalRunSnapshot(context, { ...scope, output: capture });
+                    if (!guard.capture_path)
+                        recordRecoveryCapture(context, { ...scope, capturePath: capture });
+                    else {
+                        const run = context.db.get("SELECT workspace_root FROM runs WHERE project_id = ? AND id = ?", [project.id, input.runId]);
+                        verifyRecoverySnapshot(capture, { ...scope, workspaceRoot: run.workspace_root });
+                    }
+                    const applied = context.db.run(`UPDATE run_controls SET status = 'settled', updated_at = ?
+            WHERE control_id = ? AND status = 'requested' AND recovery_id = ?
+              AND EXISTS (SELECT 1 FROM run_control_workers w WHERE w.control_id = run_controls.control_id AND w.owner_token = ? AND w.status = 'running')
+              AND EXISTS (SELECT 1 FROM run_recovery_guards g WHERE g.recovery_id = run_controls.recovery_id AND g.status = 'sealed' AND g.capture_path = ? AND g.generation = (SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = g.project_id AND run_id = g.run_id))`, [context.now(), input.controlId, guard.recovery_id, worker.owner_token, capture]);
+                    if (applied.changes !== 1)
+                        throw new AppError("run_control_state_changed", "Control changed before its capture could be acknowledged", 1);
+                    const snapshot = { ...settlement, status: "settled", capture_path: capture, recovery_observation: observation.state() };
+                    context.db.run("UPDATE run_control_workers SET status = 'completed', snapshot_json = ?, error_json = NULL, updated_at = ? WHERE control_id = ? AND owner_token = ?", [JSON.stringify(snapshot), context.now(), input.controlId, worker.owner_token]);
+                    return { ok: true, control_id: input.controlId, ...snapshot };
+                }
+                catch (error) {
+                    observation.exhausted();
+                    const snapshot = { ...settlement, status: "requested", capture: { status: "pending", error: failure(error) }, pending_reasons: ["recovery_capture_required"], recovery_observation: observation.state() };
+                    context.db.run("UPDATE run_control_workers SET snapshot_json = ?, error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify(snapshot), JSON.stringify(failure(error)), context.now(), input.controlId, worker.owner_token]);
+                    await delay(observation.nextDelay());
+                    continue;
+                }
+            }
+            const request = JSON.parse(current.request_json);
+            const interrupt = current.action === "stop" || request.escalate === "interrupt" && Date.now() >= Date.parse(current.created_at) + (request.grace_ms ?? 30_000);
+            let snapshot;
+            try {
+                snapshot = await withRecoveryObservation(observation, async () => {
+                    const cancellation = interrupt ? await interruptControlledSessions(context, input) : null;
+                    return { ...await observeControlledSessions(context, input), ...(cancellation ? { cancellation } : {}) };
+                });
+            }
+            catch (error) {
+                snapshot = { ok: true, control_id: input.controlId, status: "requested", settled: false, pending_reasons: ["control_observation_failed"], error: failure(error) };
+            }
+            observation.exhausted();
+            snapshot = { ...snapshot, recovery_observation: observation.state() };
+            context.db.run("UPDATE run_control_workers SET snapshot_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify(snapshot), context.now(), input.controlId, worker.owner_token]);
+            if (snapshot.settled === true) {
+                try {
+                    sealRunRecovery(context, { projectRoot: project.root, runId: input.runId, recoveryId: guard.recovery_id, settlement: JSON.stringify(snapshot) });
+                    continue;
+                }
+                catch (error) {
+                    context.db.run("UPDATE run_control_workers SET snapshot_json = ?, error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ? AND status = 'running'", [JSON.stringify({ ...snapshot, pending_reasons: ["recovery_seal_required"], error: failure(error) }), JSON.stringify(failure(error)), context.now(), input.controlId, worker.owner_token]);
+                }
+            }
+            await delay(observation.nextDelay());
+        }
+    }
+    catch (error) {
+        context.db.run("UPDATE run_control_workers SET status = 'failed', error_json = ?, updated_at = ? WHERE control_id = ? AND owner_token = ?", [JSON.stringify(failure(error)), context.now(), input.controlId, worker.owner_token]);
+        throw error;
+    }
+    finally {
+        clearInterval(heartbeat);
+        if (worker.process_id && worker.process_lease_token)
+            finishManagedProcess(resourceContext, { id: worker.process_id, leaseToken: worker.process_lease_token, state: "stopped", reason: "control_owner_exited" });
+    }
+}
+function failure(error) { return { code: error instanceof AppError ? error.code : "run_control_failed", message: error instanceof Error ? error.message : String(error) }; }