@deksden-com/dd-flow-cli 0.9.0-beta.16 → 0.9.0-beta.161

package/dist/services/runtime-scope-stop.js

@@ -0,0 +1,211 @@
+import crypto from "node:crypto";
+import { nativeShutdownConfirmed } from "../harness-runtime/lib/native-children.mjs";
+import fs from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import { createContext } from "../runtime/context.js";
+import { AppError } from "../shared/errors.js";
+import { runHarnessAdapter } from "./harness-adapter.js";
+import { managedLeaseReleased, managedProcessBinding, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processTreeIsAlive, resourceHome, retainedResourceHomeMatches, stopManagedProcess } from "./managed-processes.js";
+import { terminalCheckSettled } from "./code-checks.js";
+import { fenceRuntimeBudget, runtimeBudgetStatus, scopeLocalProcessRole } from "./runtime-budget.js";
+import { retainedDaemonOperations } from "./run-control.js";
+import { historicalDaemonSettlement } from "./native-daemon-history.js";
+import { managedProviderBelongsToDaemon } from "./managed-daemon-binding.js";
+/** Native daemons own cancellation after the observing CLI disconnects. */
+export async function stopRuntimeScope(context, scopeId, requestId) {
+    const fence = fenceRuntimeBudget(context, scopeId, requestId);
+    const inventory = runtimeBudgetStatus(context, scopeId).processes;
+    const nodes = await Promise.all(inventory.filter(record => record.kind.endsWith("-daemon")).map(async (record) => {
+        try {
+            const historical = historicalDaemonSettlement(context, scopeId, record);
+            if (historical)
+                return historical;
+            const metadata = JSON.parse(record.metadata_json);
+            if (!metadata.state_dir || !path.isAbsolute(metadata.state_dir) || !metadata.adapter_executable || !path.isAbsolute(metadata.adapter_executable))
+                throw new AppError("runtime_scope_owner_unknown", "Native stop requires its retained directory and executable", 1);
+            const file = path.join(metadata.state_dir, "daemon.json");
+            const read = () => JSON.parse(fs.readFileSync(file, "utf8"));
+            const state = read();
+            const owner = state.config?.runtime_owner;
+            if (!state.daemon_id || !owner || owner.budget?.scope_id !== scopeId || owner.owner_id !== record.owner_id || owner.role !== metadata.role || owner.state_dir !== metadata.state_dir || owner.adapter_executable !== metadata.adapter_executable || !retainedResourceHomeMatches(context, owner.resource_home) || (owner.project_id ?? null) !== record.project_id || (owner.run_id ?? null) !== record.run_id || (owner.work_id ?? null) !== record.work_id || !record.operation_id?.startsWith(`${owner.operation_id}:`) || state.resource_process?.id !== record.id || state.resource_process.lease_token !== record.lease_token || state.pid !== record.pid)
+                throw new AppError("runtime_scope_owner_unknown", "Native owner does not match its registration", 1);
+            const closed = state.shutdown_state === "clean" && state.active_tree === false && !processTreeIsAlive(record)
+                ? retainedDaemonOperations(metadata.state_dir).find(operation => operation.daemon_id === state.daemon_id && operation.operation === "daemon.stop" && operation.state === "completed" && operation.result?.stopped === true && nativeShutdownConfirmed(operation.result, { daemonId: state.daemon_id ?? "" })) : undefined;
+            const operationId = closed ? String(closed.operation_id) : `scope-stop:${crypto.createHash("sha256").update(JSON.stringify([scopeId, record.id, state.daemon_id])).digest("hex")}`;
+            const requested = path.join(metadata.state_dir, "operations", crypto.createHash("sha256").update(operationId).digest("hex"), "requested.json");
+            const prior = fs.existsSync(requested);
+            const current = managedProcessStatus(context).find(item => item.id === record.id);
+            if (!current || current.lease_token !== record.lease_token || current.pid !== record.pid || current.pid_started_at !== record.pid_started_at)
+                throw new AppError("runtime_scope_owner_changed", "Native registration changed before control", 1);
+            if (!prior && !processIsAlive(current))
+                throw new AppError("runtime_scope_outcome_unknown", "Dead native owner requires retained shutdown evidence", 1);
+            const env = { ...context.env, DD_FLOW_HOME: owner.dd_flow_home, DD_FLOW_BIN: owner.dd_flow_bin, DD_FLOW_RESOURCE_HOME: resourceHome(context) };
+            delete env.DD_FLOW_CONTROL_TOKEN;
+            delete env.DD_FLOW_CONTROLLER_TOKEN;
+            const receipt = await runHarnessAdapter({ executable: metadata.adapter_executable,
+                args: prior ? ["daemon", "operation", "--operation-id", operationId, "--state-dir", metadata.state_dir, "--json"] : ["daemon", "stop", "--cancel-tree", "--state-dir", metadata.state_dir, "--json"],
+                env, operationId: prior ? `${operationId}:observe` : operationId, timeoutMs: 30_000 });
+            if (prior && receipt.state === "failed" && receipt.operation_id === operationId && receipt.daemon_id === state.daemon_id) {
+                const failure = receipt.error;
+                throw new AppError(failure?.code ?? "runtime_scope_control_failed", failure?.message ?? "Native close operation failed", 1, failure?.details);
+            }
+            if (prior && (receipt.operation_id !== operationId || receipt.daemon_id !== state.daemon_id || receipt.operation !== "daemon.stop" || receipt.state !== "completed"))
+                throw new AppError("runtime_scope_outcome_unknown", "Native close operation has not completed", 1);
+            const result = (prior ? receipt.result : receipt);
+            const after = read();
+            const finalOwner = managedProcessStatus(context).find(item => item.id === record.id);
+            if (!finalOwner || finalOwner.lease_token !== record.lease_token || finalOwner.pid !== record.pid || finalOwner.pid_started_at !== record.pid_started_at || after.resource_process?.id !== record.id || after.resource_process.lease_token !== record.lease_token || JSON.stringify(after.config?.runtime_owner) !== JSON.stringify(owner))
+                throw new AppError("runtime_scope_owner_changed", "Native ownership changed during control", 1);
+            if (result?.stopped !== true || !nativeShutdownConfirmed(result, { daemonId: state.daemon_id }) || after.daemon_id !== state.daemon_id || after.shutdown_state !== "clean" || after.active_tree !== false || processTreeIsAlive(finalOwner))
+                throw new AppError("runtime_scope_not_settled", "Native close has not proved whole-tree settlement", 1);
+            const processes = managedProcessStatus(context).filter(item => managedProviderBelongsToDaemon(item, finalOwner, after) && item.operation_id?.startsWith(`${owner.operation_id}:`) && !processTreeIsAlive(item)).map(item => ({ process_id: item.id, native_settled: true, binding: managedProcessBinding(item, resourceHome(context)) }));
+            return { process_id: record.id, state_dir: metadata.state_dir, settled: true, native_settled: true, physical_stopped: true, operation_id: operationId, owner_operation_id: owner.operation_id, processes };
+        }
+        catch (error) {
+            const failure = error;
+            const native = failure.details?.cause && typeof failure.details.cause === "object" ? failure.details.cause : null;
+            const physical = await stopVerifiedNativeProcesses(context, scopeId, record);
+            const stateDir = JSON.parse(record.metadata_json).state_dir;
+            return { process_id: record.id, ...(stateDir ? { state_dir: stateDir } : {}), settled: false, native_settled: false, physical_stopped: physical.stopped,
+                reason: native?.code ?? failure.code ?? "runtime_scope_control_failed", message: native?.message ?? failure.message ?? String(error),
+                ...(native ? { cause: { code: native.code ?? null, message: native.message ?? null, details: native.details ?? null } } : {}), ...(physical.error ? { cleanup_error: physical.error } : {}),
+                ...(physical.provider ? { processes: [{ process_id: physical.provider.id, native_settled: false, binding: managedProcessBinding(physical.provider, resourceHome(context)) }] } : {}) };
+        }
+    }));
+    const covered = new Set(nodes.map(node => node.process_id));
+    for (const record of runtimeBudgetStatus(context, scopeId).processes) {
+        if (covered.has(record.id))
+            continue;
+        if (managedLeaseReleased(record)) {
+            nodes.push({ process_id: record.id, settled: true });
+            continue;
+        }
+        const metadata = JSON.parse(record.metadata_json);
+        // RUN control owns active controllers/checks. Adopt only its already-finished
+        // physical registrations here; death alone is not a completed RUN operation.
+        const terminalCheck = record.kind === "check" && metadata.role === "check" && record.check_id && record.project_id && record.run_id
+            ? (() => {
+                const home = managedProcessRuntimeHome(record, context);
+                if (!home)
+                    return null;
+                let reader;
+                try {
+                    reader = createContext({ ...context.env, DD_FLOW_HOME: home }, "read_existing");
+                }
+                catch {
+                    return null;
+                }
+                try {
+                    return terminalCheckSettled(reader, record);
+                }
+                finally {
+                    reader.db.close?.();
+                }
+            })() : null;
+        if ((record.kind === "run-controller" && metadata.role === "controller" && record.operation_id
+            || record.kind === "check" && metadata.role === "check" && record.check_id)
+            && record.project_id && record.run_id && ["stopped", "failed"].includes(record.state)
+            && record.pid && record.pid_started_at && (terminalCheck || !processTreeIsAlive(record))) {
+            const operationId = record.operation_id ?? record.check_id;
+            nodes.push({ process_id: record.id, state_dir: metadata.state_dir ?? "", settled: true, operation_id: operationId, owner_operation_id: operationId });
+            continue;
+        }
+        if (scopeLocalProcessRole(record) && previousBoot(record)) {
+            nodes.push({ process_id: record.id, settled: false, native_settled: false, physical_stopped: true, reason: "runtime_scope_outcome_unknown", message: "Physical owner predates host boot; logical outcome is retained" });
+            continue;
+        }
+        if (scopeLocalProcessRole(record) && record.operation_id && record.pid && record.pid_started_at) {
+            try {
+                await stopManagedProcess(context, { id: record.id, leaseToken: record.lease_token, force: false, graceMs: 500, reason: `scope-stop:${scopeId}` });
+                nodes.push({ process_id: record.id, state_dir: "", settled: true, operation_id: record.operation_id, owner_operation_id: record.operation_id });
+            }
+            catch (error) {
+                nodes.push({ process_id: record.id, settled: false, reason: "runtime_scope_resource_unsettled", message: String(error) });
+            }
+            continue;
+        }
+        const binding = JSON.stringify(managedProcessBinding(record, resourceHome(context)));
+        const parent = nodes.find(node => node.settled && node.state_dir === metadata.state_dir && "processes" in node && node.processes?.some(item => item.process_id === record.id && item.native_settled && JSON.stringify(item.binding) === binding));
+        if (parent && !processTreeIsAlive(record)) {
+            nodes.push({ process_id: record.id, state_dir: metadata.state_dir, settled: true, physical_stopped: true, operation_id: parent.operation_id, owner_operation_id: parent.owner_operation_id });
+        }
+        else {
+            const retired = nodes.find(node => node.physical_stopped === true && node.state_dir === metadata.state_dir && node.processes?.some(item => item.process_id === record.id && JSON.stringify(item.binding) === binding));
+            if (retired && !processTreeIsAlive(record))
+                nodes.push({ process_id: record.id, ...(metadata.state_dir ? { state_dir: metadata.state_dir } : {}), settled: false, native_settled: false, physical_stopped: true, ...(retired.reason ? { reason: retired.reason } : {}) });
+            else if (previousBoot(record))
+                nodes.push({ process_id: record.id, settled: false, native_settled: false, physical_stopped: true, reason: "runtime_scope_outcome_unknown", message: "Physical owner predates host boot; logical outcome is retained" });
+            else
+                nodes.push({ process_id: record.id, settled: false, reason: "runtime_scope_resource_unsettled", message: "Resource needs its own settlement proof" });
+        }
+    }
+    return { scope_id: scopeId, fence, settled: nodes.every(node => node.settled),
+        physical_stopped: nodes.every(node => node.physical_stopped === true || node.settled && node.physical_stopped !== null)
+            ? true : nodes.some(node => node.physical_stopped === null) ? null : false, nodes };
+}
+async function stopVerifiedNativeProcesses(context, scopeId, daemon) {
+    const metadata = JSON.parse(daemon.metadata_json);
+    if (!metadata.state_dir || !path.isAbsolute(metadata.state_dir))
+        return { stopped: null };
+    let state;
+    try {
+        state = JSON.parse(fs.readFileSync(path.join(metadata.state_dir, "daemon.json"), "utf8"));
+    }
+    catch {
+        return { stopped: null };
+    }
+    const owner = state.config?.runtime_owner;
+    const current = managedProcessStatus(context).find(item => item.id === daemon.id);
+    if (!owner || !current || current.lease_token !== daemon.lease_token || current.pid !== daemon.pid || current.pid_started_at !== daemon.pid_started_at
+        || !daemon.pid || !daemon.pid_started_at || owner.budget?.scope_id !== scopeId || owner.owner_id !== daemon.owner_id
+        || owner.state_dir !== metadata.state_dir || owner.role !== metadata.role || owner.adapter_executable !== metadata.adapter_executable
+        || owner.dd_flow_home !== metadata.dd_flow_home || !retainedResourceHomeMatches(context, owner.resource_home)
+        || state.resource_process?.id !== daemon.id || state.resource_process.lease_token !== daemon.lease_token
+        || (state.pid !== daemon.pid && !(state.pid == null && state.shutdown_state === "starting")))
+        return { stopped: null };
+    const registered = managedProcessStatus(context);
+    const providers = registered.filter(item => managedProviderBelongsToDaemon(item, daemon, state));
+    const unpublished = !state.provider_process && !state.provider_pid ? registered.filter(item => {
+        const child = JSON.parse(item.metadata_json);
+        return item.kind === daemon.kind.replace(/-daemon$/, "-provider") && item.owner_id === daemon.owner_id && item.project_id === daemon.project_id && item.run_id === daemon.run_id && item.work_id === daemon.work_id
+            && item.operation_id?.startsWith(`${owner.operation_id}:`) && item.pid && item.pid_started_at && child.owner_pid === daemon.pid && child.owner_pid_started_at === daemon.pid_started_at
+            && child.state_dir === metadata.state_dir && child.role === metadata.role && child.adapter_executable === metadata.adapter_executable && child.dd_flow_home === metadata.dd_flow_home
+            && child.dispatch_barrier === "run-generation@1" && child.budget?.scope_id === scopeId;
+    }) : [];
+    const provider = providers.length === 1 ? providers[0] : unpublished.length === 1 ? unpublished[0] : undefined;
+    if (provider && previousBoot(daemon) && previousBoot(provider))
+        return { stopped: true, provider };
+    // Legacy daemons did not register their provider. A host reboot proves local
+    // process exit, not native Session completion. Never signal a reused PID.
+    if (providers.length === 0 && unpublished.length === 0 && !state.provider_process && !state.provider_pid && previousBoot(daemon))
+        return { stopped: true };
+    if (providers.length > 1 || unpublished.length > 1 || state.provider_pid && !provider || !provider && state.shutdown_state !== "starting")
+        return { stopped: null };
+    try {
+        if (provider)
+            await stopManagedProcess(context, { id: provider.id, leaseToken: provider.lease_token, force: false, graceMs: 500, reason: `scope-stop:${scopeId}` });
+        await stopManagedProcess(context, { id: daemon.id, leaseToken: daemon.lease_token, force: false, graceMs: 500, reason: `scope-stop:${scopeId}` });
+        return { stopped: !provider ? null : !processTreeIsAlive(daemon) && !processTreeIsAlive(provider), ...(provider ? { provider } : {}) };
+    }
+    catch (cleanup) {
+        return { stopped: false, ...(provider ? { provider } : {}), error: { code: cleanup.code ?? "physical_stop_failed", message: String(cleanup) } };
+    }
+}
+function previousBoot(record) {
+    const metadata = JSON.parse(record.metadata_json);
+    // A waiting, unconfirmed launch cannot outlive the pre-boot owner that alone
+    // could release it. This is physical retirement, not a successful launch.
+    const identity = record.pid ? record : record.state === "starting" && Number.isSafeInteger(metadata.owner_pid) && metadata.owner_pid > 0 && metadata.owner_pid_started_at
+        ? { pid: metadata.owner_pid, pid_started_at: metadata.owner_pid_started_at, metadata_json: JSON.stringify(metadata.owner_birth ? { process_birth: metadata.owner_birth } : {}) } : null;
+    if (!identity || processIsAlive(identity))
+        return false;
+    // Legacy lstart has no timezone. Keep a conservative 26-hour ambiguity
+    // margin; recent/malformed identities must use ordinary ownership proof.
+    const started = Date.parse(record.started_at);
+    const birth = identity.pid_started_at && /^\w{3} \w{3}\s+\d{1,2} \d{2}:\d{2}:\d{2} \d{4}$/.test(identity.pid_started_at)
+        ? Date.parse(`${identity.pid_started_at} UTC`) : NaN;
+    const uptime = os.uptime();
+    const boundary = Date.now() - uptime * 1000 - 26 * 60 * 60 * 1000;
+    return Number.isFinite(uptime) && uptime > 0 && Number.isFinite(started) && Number.isFinite(birth) && started < boundary && birth < boundary;
+}

package/dist/services/runtime-scope-worker.js

@@ -0,0 +1,314 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { spawn } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { setTimeout as delay } from "node:timers/promises";
+import { AppError, errorRecord } from "../shared/errors.js";
+import { getResourceDatabase } from "../storage/database.js";
+import { assertManagedProcessLease, confirmManagedProcess, finishManagedProcess, renewManagedProcess, processIsAlive, processStartedAt, registerManagedProcess, resourceHome } from "./managed-processes.js";
+import { managedLeaseMonitor } from "./managed-lease-monitor.js";
+import { physicalStartup } from "./physical-startup.js";
+import { publicRuntimeScopeWorker, requestRuntimeScopeControl, runtimeBudgetStatus, prepareRuntimeScopeFence, parseRuntimeBudget } from "./runtime-budget.js";
+import { controlRuntimeScopeRuns } from "./runtime-scope-control.js";
+import { prepareRuntimeScopeResume, recordRuntimeScopeResume } from "./runtime-scope-resume.js";
+import { recoveryObservationBudget, renewRecoveryObservation, withRecoveryObservation } from "./recovery-observation-budget.js";
+export async function launchRuntimeScopeResume(context, input) {
+    const resume = recordRuntimeScopeResume(context, input);
+    if (!resume.current)
+        return { scope_id: input.scopeId, resume, current: false, settled: false };
+    if (resume.status === "released")
+        return { scope_id: input.scopeId, resume, release: resume.release, current: true, dispatch_blocked: resume.dispatch_blocked, settled: false };
+    const control = runtimeBudgetStatus(context, input.scopeId).control;
+    if (control?.generation !== input.generation)
+        throw new AppError("runtime_scope_control_superseded", "Scope resume source changed before owner launch", 1);
+    const owner = await launchRuntimeScopeControl(context, input.scopeId, control.request_id, control.requested_mode, undefined, undefined, input.requestId);
+    return { ...owner, resume, dispatch_blocked: true, settled: false };
+}
+const keyFor = (scope, generation) => crypto.createHash("sha256").update(JSON.stringify([scope, generation])).digest("hex");
+function birth() {
+    const started = processStartedAt(process.pid);
+    if (!started)
+        throw new AppError("runtime_scope_owner_unknown", "Scope owner birth identity is unavailable", 1);
+    return { pid: process.pid, pid_started_at: started };
+}
+export function prepareRuntimeScopeReconciliation(context, scopeId, generation, requestId) {
+    prepareRuntimeScopeFence(scopeId, requestId);
+    if (!Number.isSafeInteger(generation) || generation < 1)
+        throw new AppError("validation", "Scope reconciliation requires a positive integer generation", 2);
+    renewRecoveryObservation({}, requestId);
+    const db = getResourceDatabase(resourceHome(context), "read_existing");
+    try {
+        const row = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control' AND resource_key = ?", [scopeId]);
+        const control = row ? JSON.parse(row.metadata_json) : null;
+        if (!control || control.generation !== generation)
+            throw new AppError("runtime_scope_control_superseded", "Reconciliation requires the current controlled scope generation", 1);
+        return control;
+    }
+    finally {
+        db.close?.();
+    }
+}
+export async function reconcileRuntimeScopeControl(context, scopeId, generation, requestId) {
+    const control = prepareRuntimeScopeReconciliation(context, scopeId, generation, requestId);
+    return launchRuntimeScopeControl(context, scopeId, control.request_id, control.requested_mode, undefined, requestId);
+}
+export async function launchRuntimeScopeControl(context, scopeId, requestId, mode, manifest, reconcileId, resumeId, prepared) {
+    const control = requestRuntimeScopeControl(context, scopeId, requestId, mode, manifest, prepared), db = getResourceDatabase(resourceHome(context)), key = keyFor(scopeId, control.generation);
+    const launcher = birth();
+    const publication = (() => {
+        for (let attempt = 0;; attempt++) {
+            const prior = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control-worker' AND resource_key = ?", [key]);
+            const existing = prior ? JSON.parse(prior.metadata_json) : null;
+            if (existing) {
+                const identity = existing.owner ?? existing.launcher;
+                if (!identity || !Number.isSafeInteger(identity.pid) || identity.pid < 1 || typeof identity.pid_started_at !== "string" || !identity.pid_started_at || !["starting", "running", "completed", "failed", "superseded"].includes(existing.status))
+                    throw new AppError("runtime_scope_owner_unknown", "Retained scope owner has no verifiable birth identity", 1);
+            }
+            const existingAlive = !!existing && (existing.owner ? processIsAlive(existing.owner) : existing.status !== "failed" && processIsAlive(existing.launcher));
+            try {
+                return db.writeTransaction(() => {
+                    const current = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control' AND resource_key = ?", [scopeId]);
+                    if (db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control-worker' AND resource_key = ?", [key])?.metadata_json !== prior?.metadata_json)
+                        throw new AppError("runtime_scope_owner_changed", "Scope owner changed while its physical identity was checked", 1);
+                    if (!current || JSON.parse(current.metadata_json).generation !== control.generation || db.get("SELECT 1 FROM managed_resources WHERE resource_kind = 'provider-fence' AND resource_key = ?", [scopeId]))
+                        return { current: false, existing };
+                    if (existing && existingAlive)
+                        return { current: true, existing };
+                    if (existing?.status === "completed" && !reconcileId && !resumeId)
+                        return { current: true, existing };
+                    const renewal = resumeId ? `resume:${resumeId}` : reconcileId;
+                    const snapshot = renewal ? renewRecoveryObservation(existing?.snapshot ?? {}, renewal) : existing?.snapshot;
+                    if (existing && recoveryObservationBudget(snapshot).state().remaining_ms === 0)
+                        return { current: true, existing };
+                    const cleanupRequest = resumeId ? undefined : reconcileId ?? existing?.reconcile_id;
+                    const worker = { scope_id: scopeId, generation: control.generation, request_id: requestId, mode, token: crypto.randomUUID(), status: "starting", launcher, ...(snapshot ? { snapshot } : {}), ...(cleanupRequest ? { reconcile_id: cleanupRequest } : {}) };
+                    const now = context.now();
+                    db.run("INSERT INTO managed_resources (resource_kind,resource_key,owner_id,lease_token,lease_expires_at,process_id,metadata_json,created_at,updated_at) VALUES ('scope-control-worker',?,?,?,?,NULL,?,?,?) ON CONFLICT(resource_kind,resource_key) DO UPDATE SET lease_token = excluded.lease_token, metadata_json = excluded.metadata_json, updated_at = excluded.updated_at", [key, scopeId, worker.token, now, JSON.stringify(worker), now, now]);
+                    return { worker };
+                }, "runtimeScope.workerReserve");
+            }
+            catch (error) {
+                // No worker has spawned yet. Recheck physical identity after a racing update.
+                if (!(error instanceof AppError) || error.code !== "runtime_scope_owner_changed" || attempt >= 3)
+                    throw error;
+            }
+        }
+    })();
+    if (!publication.worker)
+        return { scope_id: scopeId, control, current: publication.current, settled: false, worker: publication.existing ? publicRuntimeScopeWorker(publication.existing) : null };
+    const worker = publication.worker;
+    try {
+        const entry = fileURLToPath(new URL("../cli.js", import.meta.url));
+        if (!fs.existsSync(entry))
+            throw new AppError("harness_runtime_unbuilt", "Build the CLI before launching scope control", 1);
+        const directory = path.join(resourceHome(context), "scope-controls", key);
+        fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
+        const log = fs.openSync(path.join(directory, "control.log"), "a", 0o600);
+        try {
+            const env = { ...context.env, DD_FLOW_HOME: context.ddFlowHome, DD_FLOW_RESOURCE_HOME: resourceHome(context), DD_FLOW_ENGINE_MODE: "1", DD_FLOW_SCOPE_CONTROL_TOKEN: worker.token };
+            delete env.DD_FLOW_CONTROL_TOKEN;
+            delete env.DD_FLOW_CONTROLLER_TOKEN;
+            const child = spawn(process.execPath, [entry, "runtime", "scope", "serve", "--scope-id", scopeId, "--generation", String(control.generation), "--json"], { env, detached: true, stdio: ["ignore", log, log] });
+            await new Promise((resolve, reject) => { child.once("spawn", resolve); child.once("error", reject); });
+            child.unref();
+        }
+        finally {
+            fs.closeSync(log);
+        }
+    }
+    catch (error) {
+        db.run("UPDATE managed_resources SET metadata_json = json_set(metadata_json, '$.status', 'failed', '$.error', ?), updated_at = ? WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ? AND json_extract(metadata_json, '$.owner') IS NULL", [String(error), context.now(), key, worker.token]);
+    }
+    const retained = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control-worker' AND resource_key = ?", [key]);
+    return { scope_id: scopeId, control, current: true, settled: false, worker: publicRuntimeScopeWorker(JSON.parse(retained.metadata_json)) };
+}
+export function prepareRuntimeScopeServing(context, scopeId, generation) {
+    parseRuntimeBudget(JSON.stringify({ schema_id: "dd-flow/runtime-budget@1", scope_id: scopeId, per_harness: {} }));
+    if (!Number.isSafeInteger(generation) || generation < 1)
+        throw new AppError("validation", "Scope worker requires a positive integer generation", 2);
+    const token = context.env.DD_FLOW_SCOPE_CONTROL_TOKEN;
+    if (!token)
+        throw new AppError("runtime_scope_owner_unknown", "Scope worker requires its launch token", 1);
+    const key = keyFor(scopeId, generation), db = getResourceDatabase(resourceHome(context), "read_existing");
+    try {
+        requireStartingScopeWorker(db, key, token);
+    }
+    finally {
+        db.close?.();
+    }
+    return { key, token };
+}
+function requireStartingScopeWorker(db, key, token, row = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ?", [key, token])) {
+    if (!row)
+        throw new AppError("runtime_scope_owner_unknown", "Scope worker claim changed", 1);
+    const worker = JSON.parse(row.metadata_json);
+    if (worker.owner || worker.status !== "starting")
+        throw new AppError("runtime_scope_already_owned", "Scope control already has a physical owner", 1);
+    return worker;
+}
+export async function serveRuntimeScopeControl(context, scopeId, generation) {
+    const { key, token } = prepareRuntimeScopeServing(context, scopeId, generation);
+    const db = getResourceDatabase(resourceHome(context));
+    const startup = physicalStartup(context.databaseDeadlineAt);
+    let claimedOwner = undefined;
+    const assertStartupCurrent = () => {
+        const control = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control' AND resource_key = ?", [scopeId]);
+        const fence = db.get("SELECT 1 FROM managed_resources WHERE resource_kind = 'provider-fence' AND resource_key = ?", [scopeId]);
+        const row = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ?", [key, token]);
+        const current = row ? JSON.parse(row.metadata_json) : null;
+        if (!control || JSON.parse(control.metadata_json).generation !== generation || fence || !current || current.token !== token || current.status !== (claimedOwner ? "running" : "starting") || current.owner?.pid !== claimedOwner?.pid || current.owner?.pid_started_at !== claimedOwner?.pid_started_at)
+            throw new AppError("runtime_scope_control_superseded", "Scope startup no longer owns this control", 1);
+    };
+    const starting = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ?", [key, token]);
+    let worker = requireStartingScopeWorker(db, key, token, starting);
+    let retainedMetadata = starting.metadata_json;
+    let claimedByThisServe = false;
+    const persistWorker = () => {
+        const next = JSON.stringify(worker);
+        if (db.run("UPDATE managed_resources SET metadata_json = ?, updated_at = ? WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ? AND metadata_json = ?", [next, context.now(), key, token, retainedMetadata]).changes !== 1)
+            throw new AppError("runtime_scope_owner_unknown", "Scope owner changed before its retained result could be saved", 1);
+        retainedMetadata = next;
+    };
+    let heartbeat;
+    let lease;
+    // A fence must never be held indefinitely by an observer that cannot obtain
+    // fresh provider evidence. A later operator request continues the retained
+    // budget; this owner records why it stopped without claiming settlement.
+    let observationCleanup;
+    let cleanupFailure;
+    const result = await (async () => {
+        try {
+            const owner = await startup.birth(assertStartupCurrent);
+            const claim = db.writeTransaction(() => {
+                assertStartupCurrent();
+                const current = requireStartingScopeWorker(db, key, token);
+                current.owner = owner;
+                current.status = "running";
+                db.run("UPDATE managed_resources SET metadata_json = ?, updated_at = ? WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ?", [JSON.stringify(current), context.now(), key, token]);
+                return { worker: current, metadata: JSON.stringify(current) };
+            });
+            worker = claim.worker;
+            retainedMetadata = claim.metadata;
+            claimedOwner = owner;
+            claimedByThisServe = true;
+            const observation = observationCleanup = recoveryObservationBudget(worker.snapshot);
+            worker.snapshot = { ...(worker.snapshot ?? {}), recovery_observation: observation.state() };
+            const record = registerManagedProcess(context, { kind: "scope-control", ownerId: key, operationId: `scope-control:${key}:${token}`, ownerPid: process.pid, metadata: { role: "control", scope_id: scopeId, generation } });
+            worker.process_id = record.id;
+            worker.process_lease_token = record.lease_token;
+            const confirmed = await startup.observe(() => confirmManagedProcess(context, { id: record.id, leaseToken: record.lease_token, pid: process.pid, processGroupId: process.pid, ownerPid: process.pid }), assertStartupCurrent, record.lease_expires_at);
+            persistWorker();
+            lease = managedLeaseMonitor(attemptMs => renewManagedProcess({ ...context, resourceDatabaseMode: "prepared_existing", databaseDeadlineAt: Date.now() + attemptMs }, { id: record.id, leaseToken: record.lease_token }), confirmed.lease_expires_at);
+            let timerFailure;
+            const assertScopeCurrent = () => {
+                const latest = runtimeBudgetStatus(context, scopeId);
+                if (latest.control?.generation !== generation)
+                    throw new AppError("runtime_scope_control_superseded", "Scope control was superseded during renewal", 1);
+            };
+            const ownedContext = { ...context, assertControlOwner: () => {
+                    lease.assertNoRenewal();
+                    assertManagedProcessLease(context, { id: record.id, leaseToken: record.lease_token, pid: process.pid });
+                    assertScopeCurrent();
+                } };
+            heartbeat = setInterval(() => { lease.background(assertScopeCurrent, error => { timerFailure = error; }); }, 10_000);
+            heartbeat.unref();
+            for (;;) {
+                if (timerFailure)
+                    throw timerFailure;
+                await lease.ensureConfirmed(assertScopeCurrent);
+                const status = runtimeBudgetStatus(context, scopeId);
+                const release = status.release;
+                if (release?.current === true && release.generation === generation) {
+                    worker.status = "completed";
+                    worker.snapshot = release;
+                    break;
+                }
+                if (status.control?.generation !== generation || status.fence) {
+                    worker.status = "superseded";
+                    break;
+                }
+                observation.observe(status);
+                if (observation.exhausted()) {
+                    const final = runtimeBudgetStatus(context, scopeId);
+                    const terminal = final.release;
+                    if (terminal?.current === true && terminal.generation === generation) {
+                        worker.status = "completed";
+                        worker.snapshot = terminal;
+                        break;
+                    }
+                    if (final.control?.generation !== generation || final.fence) {
+                        worker.status = "superseded";
+                        break;
+                    }
+                    worker.status = "failed";
+                    worker.error = "recovery_observation_budget_exhausted";
+                    worker.snapshot = {
+                        ...(worker.snapshot ?? {}),
+                        status: "recovery_blocked",
+                        settled: false,
+                        pending_reasons: ["recovery_observation_budget_exhausted"],
+                        error: { code: "recovery_observation_budget_exhausted", message: "Scope recovery observation exceeded its 120 second budget" },
+                        recovery_observation: observation.state(),
+                    };
+                    return { scope_id: scopeId, generation, status: "recovery_blocked", settled: false, error: worker.error };
+                }
+                const resume = status.resume;
+                const outcome = await withRecoveryObservation(observation, async () => (worker.reconcile_id ? null : await prepareRuntimeScopeResume(ownedContext, scopeId, generation, resume?.generation === generation ? resume.request_id : undefined))
+                    ?? await controlRuntimeScopeRuns(ownedContext, scopeId, worker.request_id, worker.mode, worker.reconcile_id));
+                await lease.ensureConfirmed(assertScopeCurrent);
+                observation.observe(outcome);
+                worker.snapshot = { ...outcome, recovery_observation: observation.state() };
+                persistWorker();
+                if (worker.snapshot.drain?.physical_settled === true) {
+                    worker.status = "completed";
+                    break;
+                }
+                await delay(observation.nextDelay());
+            }
+            return { scope_id: scopeId, generation, status: worker.status };
+        }
+        catch (error) {
+            if (error instanceof AppError && error.code === "runtime_scope_control_superseded") {
+                const release = runtimeBudgetStatus(context, scopeId).release;
+                worker.status = release?.current === true && release.generation === generation ? "completed" : "superseded";
+                if (worker.status === "completed")
+                    worker.snapshot = release;
+                return { scope_id: scopeId, generation, status: worker.status };
+            }
+            worker.status = "failed";
+            worker.error = String(error);
+            worker.snapshot = { ...(worker.snapshot ?? {}), error: errorRecord(error) };
+            throw error;
+        }
+        finally {
+            lease?.close();
+            if (heartbeat)
+                clearInterval(heartbeat);
+            observationCleanup?.exhausted();
+            worker.snapshot = { ...(worker.snapshot ?? {}), ...(observationCleanup ? { recovery_observation: observationCleanup.state() } : {}) };
+            let persistenceFailure;
+            try {
+                persistWorker();
+            }
+            catch (error) {
+                persistenceFailure = error;
+                process.stderr.write(`${JSON.stringify({ primary: worker.error, primary_error: worker.snapshot.error, phase: "persist_scope_owner", secondary: errorRecord(error) })}\n`);
+            }
+            try {
+                const current = claimedByThisServe && db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ?", [key, token]);
+                if (current && current.metadata_json === retainedMetadata && !persistenceFailure && worker.process_id && worker.process_lease_token)
+                    finishManagedProcess(context, { id: worker.process_id, leaseToken: worker.process_lease_token, state: worker.status === "failed" ? "failed" : "stopped", reason: "scope_owner_exited" });
+            }
+            catch (error) {
+                cleanupFailure = error;
+                process.stderr.write(`${JSON.stringify({ primary: worker.error ?? String(persistenceFailure), primary_error: worker.snapshot.error ?? (persistenceFailure === undefined ? undefined : errorRecord(persistenceFailure)), phase: "finish_scope_owner", secondary: errorRecord(error) })}\n`);
+            }
+            if (persistenceFailure)
+                cleanupFailure = persistenceFailure;
+        }
+    })();
+    if (cleanupFailure !== undefined && !worker.error)
+        throw cleanupFailure;
+    return result;
+}

package/dist/services/runtime-service.js

@@ -2,25 +2,29 @@ import fs from "node:fs";
 import crypto from "node:crypto";
 import path from "node:path";
 import { AppError } from "../shared/errors.js";
+import { validateObservationDuration } from "../harness-runtime/lib/observation-clock.mjs";
+import { assertRunMutationAllowed } from "./run-recovery.js";
 import { requireProjectByRoot } from "./projects.js";
 import { resolveRun } from "./runs.js";
 import { codeExecutionEnvironment, runCheck } from "./code-checks.js";
-import { confirmManagedProcess, finishManagedProcess, heartbeatManagedProcess, managedProcessStatus, processTreeIsAlive, registerManagedProcess, reservePorts, resourceHome, stopManagedProcess } from "./managed-processes.js";
-/** Foreground supervisor: the tool may yield while this command owns the service.
- * Its ready receipt is emitted immediately; a separate stop ends this command.
- * Readiness is infrastructure evidence, never a successful scenario verdict. */
-export async function startRuntimeService(context, input) {
-    if (!Number.isFinite(input.timeoutMs) || input.timeoutMs <= 0)
-        throw new AppError("validation", "Readiness timeout must be positive and finite", 2);
-    if (!input.command.trim() || !input.ports.length || input.ports.some(name => !/^[a-z][a-z0-9_]*$/.test(name)) || new Set(input.ports).size !== input.ports.length || !input.ports.includes(input.readyPort) || !input.readyPath.startsWith("/") || input.readyPath.startsWith("//"))
+import { retainedBrowserEnvironment } from "./check-environment.js";
+import { confirmManagedProcess, finishManagedProcess, renewManagedProcess, managedProcessStatus, processTreeIsAlive, registerManagedProcess, reservePorts, resourceHome, stopManagedProcess } from "./managed-processes.js";
+export function prepareRuntimeService(context, input) {
+    validateObservationDuration(input.timeoutMs);
+    if (typeof input.command !== "string" || !input.command.trim() || !Array.isArray(input.ports) || !input.ports.length || input.ports.some(name => typeof name !== "string" || !/^[a-z][a-z0-9_]*$/.test(name)) || new Set(input.ports).size !== input.ports.length || !input.ports.includes(input.readyPort) || typeof input.readyPath !== "string" || !input.readyPath.startsWith("/") || input.readyPath.startsWith("//"))
         throw new AppError("validation", "Declare unique lowercase port names, a ready-port from that list and a local /ready-path", 2);
     const project = requireProjectByRoot(context, path.resolve(input.projectRoot));
     const run = resolveRun(context, project.id, input.runId);
     if (!run?.run_root || !run.workspace_root)
         throw new AppError("not_found", "Registered RUN with a materialized workspace is required", 2);
+    const generation = assertRunMutationAllowed(context, project.id, run.id);
+    return { project, run, runHome: run.run_root, generation };
+}
+export async function startRuntimeService(context, input) {
+    const { project, run, runHome, generation } = prepareRuntimeService(context, input);
     input = { ...input, runId: run.id };
     const id = `PROC-${crypto.randomUUID()}`;
-    const directory = path.join(run.run_root, "runtime", id);
+    const directory = path.join(runHome, "runtime", id);
     fs.mkdirSync(directory, { recursive: true });
     const record = registerManagedProcess(context, { id, kind: "runtime-service", ownerId: `run:${input.runId}`, ownerPid: process.pid, runId: input.runId, projectId: project.id, stdoutPath: path.join(directory, "stdout.log"), stderrPath: path.join(directory, "stderr.log"), metadata: { command: input.command } });
     const receiptPath = path.join(directory, "service.json");
@@ -30,22 +34,23 @@ export async function startRuntimeService(context, input) {
     const save = () => { const temporary = `${receiptPath}.${process.pid}.tmp`; fs.writeFileSync(temporary, JSON.stringify(receipt, null, 2)); fs.renameSync(temporary, receiptPath); };
     try {
         const allocation = await reservePorts(context, { ownerId: record.owner_id, processId: record.id, names: input.ports });
-        const environment = { ...codeExecutionEnvironment(run.workspace_root), ...Object.fromEntries(Object.entries(allocation.ports).map(([name, port]) => [`DD_FLOW_PORT_${name.toUpperCase()}`, String(port)])), DD_FLOW_EVIDENCE_DIR: directory, DD_FLOW_CHECK_COMPLETION_FILE: path.join(directory, "completion.json") };
+        const environment = { ...codeExecutionEnvironment(run.workspace_root, retainedBrowserEnvironment(context.env)), ...Object.fromEntries(Object.entries(allocation.ports).map(([name, port]) => [`DD_FLOW_PORT_${name.toUpperCase()}`, String(port)])), DD_FLOW_EVIDENCE_DIR: directory, DD_FLOW_CHECK_COMPLETION_FILE: path.join(directory, "completion.json") };
         const quoted = (value) => `'${value.replaceAll("'", "'\\''")}'`;
         Object.assign(receipt, { ports: allocation.ports, environment: Object.fromEntries(Object.entries(environment).filter(([key]) => key.startsWith("DD_FLOW_PORT_"))), stop_command: `DD_FLOW_HOME=${quoted(context.ddFlowHome)} DD_FLOW_RESOURCE_HOME=${quoted(resourceHome(context))} dd-flow runtime process stop --id ${record.id} --lease-token ${record.lease_token} --json` });
         save();
+        input.progress?.(`service ${record.id} preparing; receipt: ${receiptPath}`);
+        assertRunMutationAllowed(context, project.id, run.id, generation);
         stdout = fs.openSync(String(receipt.stdout_path), "a");
         stderr = fs.openSync(String(receipt.stderr_path), "a");
-        const running = runCheck(input.command, run.workspace_root, environment, stdout, stderr, elapsed => input.progress?.(`service ${record.id} alive (${elapsed}s); receipt: ${receiptPath}`), pid => confirmManagedProcess(context, { id: record.id, leaseToken: record.lease_token, pid, processGroupId: process.platform === "win32" ? null : pid }), () => heartbeatManagedProcess(context, { id: record.id, leaseToken: record.lease_token }));
+        const running = runCheck(input.command, run.workspace_root, environment, stdout, stderr, elapsed => input.progress?.(`service ${record.id} alive (${elapsed}s); receipt: ${receiptPath}`), pid => confirmManagedProcess(context, { id: record.id, leaseToken: record.lease_token, pid, processGroupId: process.platform === "win32" ? null : pid }), attemptMs => renewManagedProcess({ ...context, resourceDatabaseMode: "prepared_existing", databaseDeadlineAt: Date.now() + attemptMs }, { id: record.id, leaseToken: record.lease_token }), { inactivityTimeoutMs: input.timeoutMs, phase: () => ready ? "serving" : "startup" });
         void running.then(() => { settled = true; });
-        const deadline = performance.now() + input.timeoutMs;
-        while (!settled && performance.now() < deadline) {
+        while (!settled) {
             try {
                 const response = await fetch(`http://127.0.0.1:${allocation.ports[input.readyPort]}${input.readyPath}`, { signal: AbortSignal.timeout(1_000), redirect: "error" });
                 await response.body?.cancel();
                 ready = response.ok;
             }
-            catch { /* Startup is observed until the declared deadline. */ }
+            catch { /* Only actual startup output advances the owned command clock. */ }
             if (ready)
                 break;
             await new Promise(resolve => setTimeout(resolve, 100));