@deksden-com/dd-flow-cli 0.9.0-beta.7 → 0.9.0-beta.75

package/dist/services/runtime-scope-control.js

@@ -0,0 +1,450 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { createContext } from "../runtime/context.js";
+import { getResourceDatabase } from "../storage/database.js";
+import { AppError } from "../shared/errors.js";
+import { finishManagedProcess, managedLeaseReleased, managedProcessBinding, managedProcessStatus, processTreeIsAlive, resourceHome, stopManagedProcess } from "./managed-processes.js";
+import { requestRuntimeScopeControl, runtimeBudgetStatus, scopeLocalProcessRole, validateRuntimeScopeManifest } from "./runtime-budget.js";
+import { nativeInventoryVersion, recordRunControlIntent, retainedDaemonOperations, runControlStatus } from "./run-control.js";
+import { requestRunControl } from "./run-control-worker.js";
+import { adapterSessionId } from "./harness-adapter.js";
+import { assertNativeCancellationReceipt, cancelControlledNativeSession, inspectControlledNativeSession } from "./native-session-control.js";
+import { verifyRecoverySnapshot } from "./eval-snapshots.js";
+import { captureScopeJournal } from "./runtime-scope-capture.js";
+import { historicalDaemonSettlement, assertHistoricalDaemonSettlement } from "./native-daemon-history.js";
+import { managedProviderBelongsToDaemon } from "./managed-daemon-binding.js";
+const scopeInventoryIdentity = (records) => JSON.stringify(records.map(record => managedProcessBinding(record, "")).sort((a, b) => a.process_id.localeCompare(b.process_id)));
+/** Persist the all-role drain boundary, separately from recovery readiness.
+ * Registry -> RUN lock order matches scope dispatch. No await/provider call is
+ * allowed while publishing: RUN resume and late registration cannot interleave.
+ */
+export function retainScopeDrain(context, control, processes, runs, native, probes) {
+    const registry = getResourceDatabase(resourceHome(context));
+    const targets = new Map();
+    const captures = [];
+    registry.beginWriteTransaction();
+    try {
+        for (const run of [...runs].sort((a, b) => String(a.runtime_home).localeCompare(String(b.runtime_home)))) {
+            const home = String(run.runtime_home);
+            let target = targets.get(home);
+            if (!target) {
+                target = createContext({ ...context.env, DD_FLOW_HOME: home, DD_FLOW_RESOURCE_HOME: resourceHome(context), DD_FLOW_ENGINE_MODE: "1" });
+                try {
+                    target.db.beginWriteTransaction();
+                }
+                catch (error) {
+                    target.db.close?.();
+                    throw error;
+                }
+                targets.set(home, target);
+            }
+            const project = target.db.get("SELECT root FROM projects WHERE id = ?", [run.project_id]);
+            const previous = run.receipt;
+            if (!project || !previous.control_id)
+                throw new AppError("runtime_scope_owner_unknown", "Drain requires the retained RUN control", 1);
+            const fresh = runControlStatus(target, { projectRoot: project.root, runId: String(run.run_id), controlId: previous.control_id });
+            if (fresh.settled !== true || fresh.control?.current !== true)
+                throw new AppError("runtime_scope_inventory_changed", "RUN control changed before scope drain publication", 1);
+            for (const record of processes) {
+                if (scopeRunProcessEvidence(record, [run], resourceHome(context)) && !scopeRunProcessEvidence(record, [{ ...run, receipt: fresh }], resourceHome(context)))
+                    throw new AppError("runtime_scope_inventory_changed", "RUN no longer proves its captured process binding", 1);
+            }
+            run.receipt = fresh;
+            // Capture failure does not undo physical drain. Retain the failure beside
+            // the stop evidence, and keep resume fenced until capture is reconciled.
+            try {
+                const selected = fresh.control;
+                const savedRun = target.db.get("SELECT workspace_root FROM runs WHERE project_id = ? AND id = ?", [run.project_id, run.run_id]);
+                if (!selected.capture_path || !path.isAbsolute(selected.capture_path) || !selected.recovery_id || !savedRun)
+                    throw new AppError("runtime_scope_capture_missing", "Scope RUN has no retained recovery capture", 1);
+                const manifest = path.join(selected.capture_path, "snapshot.json");
+                const digest = () => crypto.createHash("sha256").update(fs.readFileSync(manifest)).digest("hex");
+                const before = digest();
+                verifyRecoverySnapshot(selected.capture_path, { runId: String(run.run_id), recoveryId: selected.recovery_id, projectRoot: project.root, workspaceRoot: savedRun.workspace_root });
+                if (digest() !== before)
+                    throw new AppError("runtime_scope_inventory_changed", "RUN capture manifest changed during validation", 1);
+                captures.push({ runtime_home: home, project_id: run.project_id, run_id: run.run_id, control_id: previous.control_id, recovery_id: selected.recovery_id, status: "verified", capture_path: selected.capture_path, manifest_sha256: before });
+            }
+            catch (error) {
+                captures.push({ runtime_home: home, project_id: run.project_id, run_id: run.run_id, control_id: previous.control_id, status: "pending", reason: error instanceof AppError ? error.code : "runtime_scope_capture_failed", message: String(error) });
+            }
+        }
+        const latest = runtimeBudgetStatus(context, control.scope_id);
+        if (latest.fence || latest.control?.generation !== control.generation)
+            throw new AppError("runtime_scope_control_superseded", "Scope drain belongs to an old generation", 1);
+        if (scopeInventoryIdentity(processes) !== scopeInventoryIdentity(latest.processes) || latest.provider_turns.length)
+            throw new AppError("runtime_scope_inventory_changed", "Scope inventory changed before drain publication", 1);
+        for (const owner of native) {
+            if (owner.historical === true) {
+                assertHistoricalDaemonSettlement(context, control.scope_id, owner);
+                continue;
+            }
+            if (typeof owner.state_dir !== "string" || nativeInventoryVersion(owner.state_dir) !== owner.inventory_version)
+                throw new AppError("runtime_scope_inventory_changed", "Native inventory changed before drain publication", 1);
+        }
+        for (const record of processes) {
+            const nativeProof = native.some(owner => owner.native_settled === true && (owner.process_id === record.id || Array.isArray(owner.processes) && owner.processes.some(item => item.process_id === record.id && item.native_settled === true)));
+            const probeProof = probes.some(probe => probe.process_id === record.id && probe.physical_settled === true) && !processTreeIsAlive(record);
+            if (!nativeProof && !probeProof && !scopeRunProcessEvidence(record, runs, resourceHome(context)))
+                throw new AppError("runtime_scope_resource_unsettled", "Scope drain is missing an owned process settlement proof", 1);
+        }
+        let manifest = null, journal = null, manifestReason = "scope_manifest_required";
+        if (latest.manifest) {
+            try {
+                manifest = validateRuntimeScopeManifest(latest.manifest, control.scope_id);
+                journal = captureScopeJournal(context, control, manifest);
+                manifestReason = "scope_operation_reconciliation_required";
+            }
+            catch (error) {
+                manifestReason = error instanceof AppError ? error.code : "runtime_scope_manifest_unavailable";
+            }
+        }
+        const resumableNative = native.filter(owner => owner.historical !== true);
+        const capture = { manifest, journal, run_captures: captures, native_bindings: resumableNative.map(owner => ({ process_id: owner.process_id, role: owner.role, state_dir: owner.state_dir, daemon_id: owner.daemon_id, runtime_owner: owner.runtime_owner, inventory_version: owner.inventory_version, sessions: owner.sessions, operations: owner.operations })), probe_operations: probes, pending_reasons: [...new Set([manifestReason, ...captures.filter(item => item.status !== "verified").map(item => String(item.reason)), ...(resumableNative.length ? ["non_work_recovery_required"] : []), ...(probes.length ? ["probe_reconciliation_required"] : [])])] };
+        const receipt = { scope_id: control.scope_id, generation: control.generation, requested_mode: control.requested_mode, observed_at: context.now(), physical_settled: true, recovery_ready: false, process_bindings: processes.map(record => managedProcessBinding(record, resourceHome(context))), runs, native, probes, capture };
+        const key = crypto.createHash("sha256").update(JSON.stringify([control.scope_id, control.generation])).digest("hex");
+        registry.run("INSERT INTO managed_resources (resource_kind,resource_key,owner_id,lease_token,lease_expires_at,process_id,metadata_json,created_at,updated_at) VALUES ('scope-control-drain',?,?,?,?,NULL,?,?,?) ON CONFLICT(resource_kind,resource_key) DO UPDATE SET metadata_json = excluded.metadata_json, updated_at = excluded.updated_at", [key, control.scope_id, key, context.now(), JSON.stringify(receipt), context.now(), context.now()]);
+        registry.exec("COMMIT");
+        return receipt;
+    }
+    catch (error) {
+        registry.exec("ROLLBACK");
+        throw error;
+    }
+    finally {
+        for (const target of targets.values()) {
+            try {
+                target.db.exec("ROLLBACK");
+            }
+            finally {
+                target.db.close?.();
+            }
+        }
+    }
+}
+export function scopeRunProcessEvidence(record, runs, registryHome) {
+    const metadata = JSON.parse(record.metadata_json);
+    if ((!managedLeaseReleased(record) && (!record.pid || !record.pid_started_at)) || !record.project_id || !record.run_id || !metadata.dd_flow_home)
+        return null;
+    for (const run of runs) {
+        if (run.runtime_home !== metadata.dd_flow_home || run.project_id !== record.project_id || run.run_id !== record.run_id)
+            continue;
+        const receipt = run.receipt;
+        if (receipt?.settled !== true || receipt.control?.current !== true || receipt.scope?.project_id !== record.project_id || receipt.scope.run_id !== record.run_id)
+            continue;
+        const bindingSignature = managedProcessBinding(record, registryHome).registration_sha256;
+        if (receipt.control.settlement?.process_bindings?.some(binding => binding.resource_home === registryHome && binding.process_id === record.id && binding.registration_sha256 === bindingSignature))
+            return { source: "run_control", process_id: record.id };
+    }
+    return null;
+}
+export async function controlRuntimeScopeLocalProbes(context, scopeId, generation, mode) {
+    const registry = getResourceDatabase(resourceHome(context));
+    const inventory = runtimeBudgetStatus(context, scopeId).processes;
+    return await Promise.all(inventory.filter(record => ["eval-baseline", "eval-observer"].includes(record.kind)).map(async (record) => {
+        try {
+            const validate = () => {
+                const head = registry.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control' AND resource_key = ?", [scopeId]);
+                if (!head || JSON.parse(head.metadata_json).generation !== generation || JSON.parse(head.metadata_json).requested_mode !== mode || registry.get("SELECT 1 FROM managed_resources WHERE resource_kind = 'provider-fence' AND resource_key = ?", [scopeId]))
+                    throw new AppError("runtime_scope_control_superseded", "Local probe control belongs to an old generation", 1);
+                const current = registry.get("SELECT * FROM managed_processes WHERE id = ?", [record.id]);
+                if (!scopeLocalProcessRole(record) || !record.pid || !record.pid_started_at || !current || current.kind !== record.kind || current.owner_id !== record.owner_id || current.operation_id !== record.operation_id || current.project_id !== record.project_id || current.run_id !== record.run_id || current.lease_token !== record.lease_token || current.pid !== record.pid || current.pid_started_at !== record.pid_started_at || current.metadata_json !== record.metadata_json)
+                    throw new AppError("runtime_scope_owner_unknown", "Local process requires its exact retained physical registration", 1);
+                return current;
+            };
+            let current = validate();
+            if (mode === "pause" && processTreeIsAlive(current))
+                return { process_id: record.id, physical_settled: false, reason: "local_probe_running" };
+            const operationId = `scope-probe:${crypto.createHash("sha256").update(JSON.stringify([scopeId, generation, record.id, record.lease_token, record.pid_started_at])).digest("hex")}`;
+            registry.beginWriteTransaction();
+            try {
+                current = validate();
+                registry.run("INSERT OR IGNORE INTO managed_resources (resource_kind,resource_key,owner_id,lease_token,lease_expires_at,process_id,metadata_json,created_at,updated_at) VALUES ('scope-control-operation',?,?,?,?,NULL,?,?,?)", [operationId, scopeId, record.lease_token, context.now(), JSON.stringify({ scope_id: scopeId, generation, process_id: record.id, operation: mode === "stop" ? "probe.stop" : "probe.observe", status: "dispatching" }), context.now(), context.now()]);
+                registry.exec("COMMIT");
+            }
+            catch (error) {
+                registry.exec("ROLLBACK");
+                throw error;
+            }
+            // A prior owner may have died after recording intent but before signalling.
+            // Repeating an identity-checked stop never redispatches productive work.
+            if (mode === "stop" && processTreeIsAlive(current))
+                await stopManagedProcess(context, { id: record.id, leaseToken: record.lease_token, force: false, graceMs: 500, reason: operationId });
+            current = validate();
+            if (processTreeIsAlive(current))
+                throw new AppError("runtime_scope_resource_unsettled", "Local probe has not exited", 1);
+            if (!finishManagedProcess(context, { id: record.id, leaseToken: record.lease_token, state: current.state === "failed" ? "failed" : "stopped", reason: operationId }))
+                throw new AppError("runtime_scope_owner_changed", "Probe ownership changed before settlement", 1);
+            const receipt = { process_id: record.id, role: scopeLocalProcessRole(record), physical_settled: true, operation_id: operationId, outcome: "requires_reconciliation" };
+            registry.run("UPDATE managed_resources SET metadata_json = json_set(metadata_json, '$.status', 'completed', '$.receipt', json(?)), updated_at = ? WHERE resource_kind = 'scope-control-operation' AND resource_key = ?", [JSON.stringify(receipt), context.now(), operationId]);
+            return receipt;
+        }
+        catch (error) {
+            return { process_id: record.id, physical_settled: false, reason: error instanceof AppError ? error.code : "runtime_scope_control_failed", message: String(error) };
+        }
+    }));
+}
+export async function controlRuntimeScopeNativeOwners(context, scopeId, generation, action = "observe") {
+    const registry = getResourceDatabase(resourceHome(context));
+    const assertCurrent = () => {
+        const row = registry.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control' AND resource_key = ?", [scopeId]);
+        if (!row || JSON.parse(row.metadata_json).generation !== generation || action === "stop" && JSON.parse(row.metadata_json).requested_mode !== "stop" || registry.get("SELECT 1 FROM managed_resources WHERE resource_kind = 'provider-fence' AND resource_key = ?", [scopeId]))
+            throw new AppError("runtime_scope_control_superseded", "Scope observation belongs to an old generation", 1);
+    };
+    const inventory = runtimeBudgetStatus(context, scopeId).processes;
+    return await Promise.all(inventory.filter(record => record.kind.endsWith("-daemon") && !record.run_id).map(async (record) => {
+        try {
+            assertCurrent();
+            const historical = historicalDaemonSettlement(context, scopeId, record);
+            if (historical) {
+                assertCurrent();
+                return historical;
+            }
+            const metadata = JSON.parse(record.metadata_json);
+            if (!["judge", "probe"].includes(metadata.role ?? "") || !metadata.state_dir || !path.isAbsolute(metadata.state_dir) || !metadata.adapter_executable || !path.isAbsolute(metadata.adapter_executable) || metadata.dispatch_barrier !== "run-generation@1")
+                throw new AppError("runtime_scope_owner_unknown", "Non-Work observation requires a frozen native owner and dispatch barrier", 1);
+            const file = path.join(metadata.state_dir, "daemon.json"), before = fs.readFileSync(file, "utf8");
+            const initialNativeVersion = nativeInventoryVersion(metadata.state_dir, []);
+            if (fs.readFileSync(file, "utf8") !== before)
+                throw new AppError("runtime_scope_inventory_changed", "Native inventory changed while its owner was being resolved", 1);
+            const state = JSON.parse(before);
+            const owner = state.config?.runtime_owner;
+            if (!state.daemon_id || state.pid !== record.pid || !record.pid_started_at || state.resource_process?.id !== record.id || state.resource_process.lease_token !== record.lease_token || !owner || owner.owner_id !== record.owner_id || owner.role !== metadata.role || owner.budget?.scope_id !== scopeId || owner.state_dir !== metadata.state_dir || owner.dd_flow_home !== metadata.dd_flow_home || owner.resource_home !== resourceHome(context) || owner.adapter_executable !== metadata.adapter_executable || (owner.project_id ?? null) !== record.project_id || (owner.run_id ?? null) !== record.run_id || (owner.work_id ?? null) !== record.work_id || !record.operation_id?.startsWith(`${owner.operation_id}:`))
+                throw new AppError("runtime_scope_owner_unknown", "Native observation does not match retained ownership", 1);
+            const assertOwner = () => {
+                assertCurrent();
+                const all = managedProcessStatus(context);
+                if (!all.some(item => item.id === record.id && item.lease_token === record.lease_token && item.pid === record.pid && item.pid_started_at === record.pid_started_at && item.metadata_json === record.metadata_json))
+                    throw new AppError("runtime_scope_owner_changed", "Native registration changed during observation", 1);
+                for (const item of all) {
+                    const saved = JSON.parse(item.metadata_json);
+                    if (saved.state_dir === metadata.state_dir && (saved.budget?.scope_id !== scopeId || item.owner_id !== record.owner_id))
+                        throw new AppError("runtime_scope_owner_unknown", "Native directory has a foreign registration", 1);
+                }
+            };
+            assertOwner();
+            if (!Array.isArray(state.sessions) || !state.sessions.length)
+                throw new AppError("daemon_inventory_invalid", "Native Session inventory is unknown", 1);
+            const ids = state.sessions.map(item => typeof item === "string" && item.trim() ? item : adapterSessionId(item));
+            if (ids.some(id => !id) || new Set(ids).size !== ids.length)
+                throw new AppError("daemon_inventory_invalid", "Native Session inventory is ambiguous", 1);
+            const env = { ...context.env, DD_FLOW_HOME: owner.dd_flow_home, DD_FLOW_BIN: owner.dd_flow_bin, DD_FLOW_RESOURCE_HOME: owner.resource_home };
+            delete env.DD_FLOW_CONTROL_TOKEN;
+            delete env.DD_FLOW_CONTROLLER_TOKEN;
+            const sessions = await Promise.all(ids.map(async (id) => {
+                try {
+                    let cancellation;
+                    let cancellationError;
+                    if (action === "stop") {
+                        const operationId = `scope-cancel:${crypto.createHash("sha256").update(JSON.stringify([scopeId, generation, record.id, state.daemon_id, id])).digest("hex")}`;
+                        const now = context.now();
+                        let inserted = false;
+                        registry.beginWriteTransaction();
+                        try {
+                            assertOwner();
+                            inserted = registry.run("INSERT OR IGNORE INTO managed_resources (resource_kind,resource_key,owner_id,lease_token,lease_expires_at,process_id,metadata_json,created_at,updated_at) VALUES ('scope-control-operation',?,?,?,?,NULL,?,?,?)", [operationId, scopeId, record.lease_token, now, JSON.stringify({ scope_id: scopeId, generation, process_id: record.id, daemon_id: state.daemon_id, session_id: id, status: "dispatching" }), now, now]).changes === 1;
+                            registry.exec("COMMIT");
+                        }
+                        catch (error) {
+                            registry.exec("ROLLBACK");
+                            throw error;
+                        }
+                        try {
+                            const saved = JSON.parse(registry.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control-operation' AND resource_key = ?", [operationId]).metadata_json);
+                            if (saved.status === "completed") {
+                                assertNativeCancellationReceipt(saved.receipt, id);
+                                cancellation = saved.receipt;
+                            }
+                            else if (saved.status === "failed") {
+                                cancellation = saved.receipt;
+                                try {
+                                    cancellationError = JSON.parse(saved.error ?? "");
+                                }
+                                catch {
+                                    cancellationError = { code: "cancellation_outcome_unknown", message: saved.error ?? "Retained cancellation has no conclusive outcome" };
+                                }
+                            }
+                            else {
+                                const result = await cancelControlledNativeSession({ executable: metadata.adapter_executable, sessionId: id, stateDir: metadata.state_dir, env, operationId, observe: !inserted, assertCurrent: assertOwner });
+                                registry.run("UPDATE managed_resources SET metadata_json = json_set(metadata_json, '$.status', ?, '$.receipt', json(?)), updated_at = ? WHERE resource_kind = 'scope-control-operation' AND resource_key = ?", [result.status, JSON.stringify(result.receipt), context.now(), operationId]);
+                                cancellation = result.receipt;
+                            }
+                        }
+                        catch (error) {
+                            cancellationError = { code: error instanceof AppError ? error.code : "cancellation_outcome_unknown", message: String(error) };
+                            registry.run("UPDATE managed_resources SET metadata_json = json_set(metadata_json, '$.status', 'outcome_unknown', '$.error', ?), updated_at = ? WHERE resource_kind = 'scope-control-operation' AND resource_key = ?", [JSON.stringify(cancellationError), context.now(), operationId]);
+                        }
+                        if (cancellationError?.code === "controller_operation_mismatch")
+                            throw new AppError("controller_operation_mismatch", "Cancellation evidence belongs to another operation or Session", 1);
+                    }
+                    const receipt = await inspectControlledNativeSession({ executable: metadata.adapter_executable, sessionId: id, stateDir: metadata.state_dir, env, operationId: `scope:${scopeId}:${generation}:inspect:${crypto.randomUUID()}`, assertCurrent: assertOwner });
+                    // Physical settlement is determined by this fresh inspection. A past
+                    // cancel ACK (or an inconclusive cancel outcome) is retained as
+                    // evidence, never substituted for the state of the native tree.
+                    return { session_id: id, settled: receipt.settled === true, receipt, ...(cancellation ? { cancellation } : {}), ...(cancellationError ? { cancellation_error: cancellationError } : {}) };
+                }
+                catch (error) {
+                    return { session_id: id, settled: false, reason: error instanceof AppError ? error.code : "runtime_scope_session_control_failed", message: String(error) };
+                }
+            }));
+            assertOwner();
+            if (nativeInventoryVersion(metadata.state_dir, []) !== initialNativeVersion)
+                throw new AppError("runtime_scope_inventory_changed", "Native inventory changed during observation", 1);
+            const operations = retainedDaemonOperations(metadata.state_dir);
+            const unsettled = operations.filter(operation => !["completed", "failed"].includes(String(operation.state)));
+            const nativeSettled = sessions.every(session => session.settled) && !unsettled.length;
+            const processes = inventory.filter(item => JSON.parse(item.metadata_json).state_dir === metadata.state_dir).map(item => {
+                const owned = item.id === record.id || managedProviderBelongsToDaemon(item, record, state) && item.operation_id?.startsWith(`${owner.operation_id}:`) === true;
+                return { process_id: item.id, native_settled: owned && nativeSettled, reason: !owned ? "runtime_scope_owner_unknown" : nativeSettled ? null : "native_session_settlement_required" };
+            });
+            return { process_id: record.id, role: metadata.role, state_dir: metadata.state_dir, daemon_id: state.daemon_id, runtime_owner: owner, inventory_version: nativeInventoryVersion(metadata.state_dir, operations), native_settled: nativeSettled, processes, sessions, operations, unsettled_operations: unsettled.map(operation => operation.operation_id) };
+        }
+        catch (error) {
+            return { process_id: record.id, native_settled: false, reason: error instanceof AppError ? error.code : "runtime_scope_observation_failed", message: String(error) };
+        }
+    }));
+}
+/** Scope routing only: existing RUN workers own physical drain and capture. */
+export async function controlRuntimeScopeRuns(context, scopeId, requestId, mode, reconcileId) {
+    const control = requestRuntimeScopeControl(context, scopeId, requestId, mode);
+    const registry = getResourceDatabase(resourceHome(context));
+    const inventory = runtimeBudgetStatus(context, scopeId).processes;
+    const runs = [];
+    const seen = new Set();
+    for (const record of inventory.filter(item => item.kind === "run-controller")) {
+        let target;
+        try {
+            const metadata = JSON.parse(record.metadata_json);
+            if (metadata.role !== "controller" || metadata.controller_id !== record.owner_id || !record.project_id || !record.run_id || !metadata.dd_flow_home || !path.isAbsolute(metadata.dd_flow_home) || !fs.existsSync(path.join(metadata.dd_flow_home, "db.sqlite")))
+                throw new AppError("runtime_scope_owner_unknown", "Scope RUN requires its retained runtime home and controller registration", 1);
+            const key = JSON.stringify([metadata.dd_flow_home, record.project_id, record.run_id]);
+            if (seen.has(key))
+                continue;
+            target = createContext({ ...context.env, DD_FLOW_HOME: metadata.dd_flow_home, DD_FLOW_RESOURCE_HOME: resourceHome(context), DD_FLOW_ENGINE_MODE: "1" });
+            const owner = target.db.get("SELECT manifest_json, process_id, process_lease_token FROM run_controllers WHERE controller_id = ? AND project_id = ? AND run_id = ?", [record.owner_id, record.project_id, record.run_id]);
+            const manifest = owner && JSON.parse(owner.manifest_json);
+            const project = target.db.get("SELECT root FROM projects WHERE id = ?", [record.project_id]);
+            if (!owner || owner.process_id !== record.id || owner.process_lease_token !== record.lease_token || manifest?.resource_home !== resourceHome(context) || manifest.budget?.scope_id !== scopeId || !project?.root)
+                throw new AppError("runtime_scope_owner_unknown", "Scope RUN registration does not match the retained controller", 1);
+            const runRequestId = `scope:${crypto.createHash("sha256").update(JSON.stringify([scopeId, control.generation, key])).digest("hex")}`;
+            let input = { projectRoot: project.root, runId: record.run_id, requestId: runRequestId, action: mode };
+            // Lock order is resource registry then RUN database. No provider or await
+            // occurs here; a newer scope request cannot interleave RUN intent creation.
+            registry.beginWriteTransaction();
+            try {
+                const current = registry.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control' AND resource_key = ?", [scopeId]);
+                if (!current || JSON.parse(current.metadata_json).generation !== control.generation || registry.get("SELECT 1 FROM managed_resources WHERE resource_kind = 'provider-fence' AND resource_key = ?", [scopeId]))
+                    throw new AppError("runtime_scope_control_superseded", "Scope request no longer owns admission", 1);
+                if (!registry.get("SELECT 1 FROM managed_processes WHERE id = ? AND lease_token = ? AND owner_id = ? AND project_id = ? AND run_id = ? AND metadata_json = ?", [record.id, record.lease_token, record.owner_id, record.project_id, record.run_id, record.metadata_json]))
+                    throw new AppError("runtime_scope_owner_changed", "RUN owner changed before scope intent dispatch", 1);
+                // A narrower caller may already have fenced this RUN. Reuse its exact
+                // intent; a scope stop still escalates a pause through the RUN arbiter.
+                const active = runControlStatus(target, input).control;
+                if (active?.current && ["requested", "settled"].includes(active.status) && (active.requested_mode === mode || active.requested_mode === "stop")) {
+                    input = { ...input, requestId: active.request_id, action: active.requested_mode,
+                        ...(active.requested_mode === "stop" ? { force: active.request.force ?? false } : { graceMs: active.request.grace_ms ?? 30_000, ...(active.request.escalate ? { escalate: active.request.escalate } : {}) }) };
+                }
+                recordRunControlIntent(target, input);
+                registry.exec("COMMIT");
+            }
+            catch (error) {
+                registry.exec("ROLLBACK");
+                throw error;
+            }
+            seen.add(key);
+            const receipt = await requestRunControl(target, { ...input, waitMs: 0, ...(reconcileId ? { reconcileId: crypto.createHash("sha256").update(JSON.stringify([scopeId, control.generation, reconcileId])).digest("hex") } : {}) });
+            runs.push({ runtime_home: metadata.dd_flow_home, project_id: record.project_id, run_id: record.run_id, request_id: input.requestId, receipt });
+        }
+        catch (error) {
+            runs.push({ process_id: record.id, run_id: record.run_id, pending: true, reason: error instanceof AppError ? error.code : "runtime_scope_control_failed", message: String(error) });
+        }
+        finally {
+            target?.db.close?.();
+        }
+    }
+    // Individual receipts are not an all-role settlement proof. Judges, probes,
+    // late registrations and unmaterialized executions still require scope drain.
+    const [native, probes] = await Promise.all([
+        controlRuntimeScopeNativeOwners(context, scopeId, control.generation, mode === "stop" ? "stop" : "observe"),
+        controlRuntimeScopeLocalProbes(context, scopeId, control.generation, mode)
+    ]);
+    for (const run of runs) {
+        if (run.pending || typeof run.runtime_home !== "string" || typeof run.run_id !== "string")
+            continue;
+        let target;
+        try {
+            target = createContext({ ...context.env, DD_FLOW_HOME: run.runtime_home, DD_FLOW_RESOURCE_HOME: resourceHome(context), DD_FLOW_ENGINE_MODE: "1" }, "read_existing");
+            const project = target.db.get("SELECT root FROM projects WHERE id = ?", [run.project_id]);
+            const selected = run.receipt;
+            if (!project || !selected.control_id)
+                throw new AppError("runtime_scope_owner_unknown", "Scope RUN has no retained control identity", 1);
+            run.receipt = runControlStatus(target, { projectRoot: project.root, runId: run.run_id, controlId: selected.control_id });
+        }
+        catch (error) {
+            run.pending = true;
+            run.receipt = null;
+            run.reason = error instanceof AppError ? error.code : "runtime_scope_observation_failed";
+            run.message = String(error);
+        }
+        finally {
+            target?.db.close?.();
+        }
+    }
+    for (const owner of native) {
+        if (typeof owner.state_dir !== "string" || typeof owner.inventory_version !== "string")
+            continue;
+        try {
+            if (owner.historical === true) {
+                assertHistoricalDaemonSettlement(context, scopeId, owner);
+                continue;
+            }
+            if (nativeInventoryVersion(owner.state_dir) !== owner.inventory_version)
+                throw new AppError("runtime_scope_inventory_changed", "Native inventory changed while other owners were draining", 1);
+        }
+        catch (error) {
+            owner.native_settled = false;
+            owner.reason = error instanceof AppError ? error.code : "runtime_scope_observation_failed";
+            if (Array.isArray(owner.processes))
+                for (const item of owner.processes) {
+                    item.native_settled = false;
+                    item.reason = owner.reason;
+                }
+        }
+    }
+    const latest = runtimeBudgetStatus(context, scopeId);
+    const current = !latest.fence && latest.control?.generation === control.generation;
+    const identity = (records) => JSON.stringify(records.map(record => [record.id, record.kind, record.owner_id, record.lease_token, record.pid, record.pid_started_at, record.project_id, record.run_id, record.work_id, record.check_id, record.operation_id, record.metadata_json]));
+    const inventoryChanged = identity(inventory) !== identity(latest.processes);
+    const nodes = latest.processes.map(record => {
+        const metadata = JSON.parse(record.metadata_json);
+        const probe = probes.find(item => item.process_id === record.id);
+        const daemon = native.find(item => item.process_id === record.id);
+        const processProofs = native.flatMap(item => Array.isArray(item.processes) ? item.processes : []).filter(item => item.process_id === record.id);
+        const nativeProcess = processProofs.find(item => item.native_settled === true) ?? processProofs[0];
+        const run = scopeRunProcessEvidence(record, runs, resourceHome(context));
+        // These observations cover their exact registered resource, not arbitrary
+        // siblings sharing a RUN or native directory. Unmapped owners stay pending.
+        const evidence = probe ?? nativeProcess ?? daemon;
+        const observed = probe?.physical_settled === true || nativeProcess?.native_settled === true || daemon?.native_settled === true || run !== null;
+        const valid = current && !inventoryChanged;
+        return { process_id: record.id, kind: record.kind, role: metadata.role ?? "unknown", run_id: record.run_id,
+            status: valid && observed ? "completed" : "pending", settled: valid && observed,
+            source: probe ? "local_probe" : daemon ? "native_owner" : nativeProcess ? "native_provider" : run ? "run_control" : null,
+            reason: !current ? "runtime_scope_control_superseded" : inventoryChanged ? "runtime_scope_inventory_changed" : observed ? null : evidence?.reason ?? "runtime_scope_resource_settlement_required" };
+    });
+    const providerTurns = latest.provider_turns;
+    let drain = null;
+    let drainError = null;
+    if (current && !inventoryChanged && !providerTurns.length && nodes.every(node => node.settled) && runs.every(run => !run.pending && run.receipt?.settled === true)) {
+        try {
+            drain = retainScopeDrain(context, control, latest.processes, runs, native, probes);
+        }
+        catch (error) {
+            drainError = error instanceof AppError ? error.code : "runtime_scope_drain_publication_failed";
+        }
+    }
+    // A drained process inventory alone misses admitted remote work whose local
+    // owner exited. Keep these obligations in the durable worker observation.
+    return { scope_id: scopeId, control, current, inventory_changed: inventoryChanged, nodes, runs, native, probes, provider_turns: providerTurns, drain, settled: false, pending_reason: !current ? "runtime_scope_control_superseded" : inventoryChanged ? "runtime_scope_inventory_changed" : providerTurns.length ? "scope_provider_turn_reconciliation_required" : drainError ?? (drain ? "scope_capture_required" : "scope_drain_required") };
+}