@deksden-com/dd-flow-cli 0.9.0-beta.7 → 0.9.0-beta.74

package/dist/services/runtime-scope-stop.js

@@ -0,0 +1,99 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { AppError } from "../shared/errors.js";
+import { runHarnessAdapter } from "./harness-adapter.js";
+import { managedLeaseReleased, managedProcessBinding, managedProcessStatus, processIsAlive, processTreeIsAlive, resourceHome, stopManagedProcess } from "./managed-processes.js";
+import { fenceRuntimeBudget, runtimeBudgetStatus, scopeLocalProcessRole } from "./runtime-budget.js";
+import { retainedDaemonOperations } from "./run-control.js";
+import { historicalDaemonSettlement } from "./native-daemon-history.js";
+import { managedProviderBelongsToDaemon } from "./managed-daemon-binding.js";
+/** Native daemons own cancellation after the observing CLI disconnects. */
+export async function stopRuntimeScope(context, scopeId, requestId) {
+    const fence = fenceRuntimeBudget(context, scopeId, requestId);
+    const inventory = runtimeBudgetStatus(context, scopeId).processes;
+    const nodes = await Promise.all(inventory.filter(record => record.kind.endsWith("-daemon")).map(async (record) => {
+        try {
+            const historical = historicalDaemonSettlement(context, scopeId, record);
+            if (historical)
+                return historical;
+            const metadata = JSON.parse(record.metadata_json);
+            if (!metadata.state_dir || !path.isAbsolute(metadata.state_dir) || !metadata.adapter_executable || !path.isAbsolute(metadata.adapter_executable))
+                throw new AppError("runtime_scope_owner_unknown", "Native stop requires its retained directory and executable", 1);
+            const file = path.join(metadata.state_dir, "daemon.json");
+            const read = () => JSON.parse(fs.readFileSync(file, "utf8"));
+            const state = read();
+            const owner = state.config?.runtime_owner;
+            if (!state.daemon_id || !owner || owner.budget?.scope_id !== scopeId || owner.owner_id !== record.owner_id || owner.role !== metadata.role || owner.state_dir !== metadata.state_dir || owner.adapter_executable !== metadata.adapter_executable || owner.resource_home !== resourceHome(context) || (owner.project_id ?? null) !== record.project_id || (owner.run_id ?? null) !== record.run_id || (owner.work_id ?? null) !== record.work_id || !record.operation_id?.startsWith(`${owner.operation_id}:`) || state.resource_process?.id !== record.id || state.resource_process.lease_token !== record.lease_token || state.pid !== record.pid)
+                throw new AppError("runtime_scope_owner_unknown", "Native owner does not match its registration", 1);
+            const closed = state.shutdown_state === "clean" && state.active_tree === false && !processTreeIsAlive(record)
+                ? retainedDaemonOperations(metadata.state_dir).find(operation => operation.daemon_id === state.daemon_id && operation.operation === "daemon.stop" && operation.state === "completed" && operation.result?.stopped === true && operation.result?.clean === true) : undefined;
+            const operationId = closed ? String(closed.operation_id) : `scope-stop:${crypto.createHash("sha256").update(JSON.stringify([scopeId, record.id, state.daemon_id])).digest("hex")}`;
+            const requested = path.join(metadata.state_dir, "operations", crypto.createHash("sha256").update(operationId).digest("hex"), "requested.json");
+            const prior = fs.existsSync(requested);
+            const current = managedProcessStatus(context).find(item => item.id === record.id);
+            if (!current || current.lease_token !== record.lease_token || current.pid !== record.pid || current.pid_started_at !== record.pid_started_at)
+                throw new AppError("runtime_scope_owner_changed", "Native registration changed before control", 1);
+            if (!prior && !processIsAlive(current))
+                throw new AppError("runtime_scope_outcome_unknown", "Dead native owner requires retained shutdown evidence", 1);
+            const env = { ...context.env, DD_FLOW_HOME: owner.dd_flow_home, DD_FLOW_BIN: owner.dd_flow_bin, DD_FLOW_RESOURCE_HOME: owner.resource_home };
+            delete env.DD_FLOW_CONTROL_TOKEN;
+            delete env.DD_FLOW_CONTROLLER_TOKEN;
+            const receipt = await runHarnessAdapter({ executable: metadata.adapter_executable,
+                args: prior ? ["daemon", "operation", "--operation-id", operationId, "--state-dir", metadata.state_dir, "--json"] : ["daemon", "stop", "--cancel-tree", "--state-dir", metadata.state_dir, "--json"],
+                env, operationId: prior ? `${operationId}:observe` : operationId, timeoutMs: 30_000 });
+            if (prior && (receipt.operation_id !== operationId || receipt.daemon_id !== state.daemon_id || receipt.operation !== "daemon.stop" || receipt.state !== "completed"))
+                throw new AppError("runtime_scope_outcome_unknown", "Native close operation has not completed", 1);
+            const result = (prior ? receipt.result : receipt);
+            const after = read();
+            const finalOwner = managedProcessStatus(context).find(item => item.id === record.id);
+            if (!finalOwner || finalOwner.lease_token !== record.lease_token || finalOwner.pid !== record.pid || finalOwner.pid_started_at !== record.pid_started_at || after.resource_process?.id !== record.id || after.resource_process.lease_token !== record.lease_token || JSON.stringify(after.config?.runtime_owner) !== JSON.stringify(owner))
+                throw new AppError("runtime_scope_owner_changed", "Native ownership changed during control", 1);
+            if (result?.stopped !== true || result.clean !== true || after.daemon_id !== state.daemon_id || after.shutdown_state !== "clean" || after.active_tree !== false || processTreeIsAlive(finalOwner))
+                throw new AppError("runtime_scope_not_settled", "Native close has not proved whole-tree settlement", 1);
+            const processes = managedProcessStatus(context).filter(item => managedProviderBelongsToDaemon(item, finalOwner, after) && item.operation_id?.startsWith(`${owner.operation_id}:`) && !processTreeIsAlive(item)).map(item => ({ process_id: item.id, native_settled: true, binding: managedProcessBinding(item, resourceHome(context)) }));
+            return { process_id: record.id, state_dir: metadata.state_dir, settled: true, operation_id: operationId, owner_operation_id: owner.operation_id, processes };
+        }
+        catch (error) {
+            return { process_id: record.id, settled: false, reason: error instanceof AppError ? error.code : "runtime_scope_control_failed", message: String(error) };
+        }
+    }));
+    const covered = new Set(nodes.map(node => node.process_id));
+    for (const record of runtimeBudgetStatus(context, scopeId).processes) {
+        if (covered.has(record.id))
+            continue;
+        if (managedLeaseReleased(record)) {
+            nodes.push({ process_id: record.id, settled: true });
+            continue;
+        }
+        const metadata = JSON.parse(record.metadata_json);
+        // RUN control owns active controllers/checks. Adopt only its already-finished
+        // physical registrations here; death alone is not a completed RUN operation.
+        if ((record.kind === "run-controller" && metadata.role === "controller" && record.operation_id
+            || record.kind === "check" && metadata.role === "check" && record.check_id)
+            && record.project_id && record.run_id && ["stopped", "failed"].includes(record.state)
+            && record.pid && record.pid_started_at && !processTreeIsAlive(record)) {
+            const operationId = record.operation_id ?? record.check_id;
+            nodes.push({ process_id: record.id, state_dir: metadata.state_dir ?? "", settled: true, operation_id: operationId, owner_operation_id: operationId });
+            continue;
+        }
+        if (scopeLocalProcessRole(record) && record.operation_id && record.pid && record.pid_started_at) {
+            try {
+                await stopManagedProcess(context, { id: record.id, leaseToken: record.lease_token, force: false, graceMs: 500, reason: `scope-stop:${scopeId}` });
+                nodes.push({ process_id: record.id, state_dir: "", settled: true, operation_id: record.operation_id, owner_operation_id: record.operation_id });
+            }
+            catch (error) {
+                nodes.push({ process_id: record.id, settled: false, reason: "runtime_scope_resource_unsettled", message: String(error) });
+            }
+            continue;
+        }
+        const binding = JSON.stringify(managedProcessBinding(record, resourceHome(context)));
+        const parent = nodes.find(node => node.settled && node.state_dir === metadata.state_dir && "processes" in node && node.processes?.some(item => item.process_id === record.id && item.native_settled && JSON.stringify(item.binding) === binding));
+        if (parent && !processTreeIsAlive(record)) {
+            nodes.push({ process_id: record.id, state_dir: metadata.state_dir, settled: true, operation_id: parent.operation_id, owner_operation_id: parent.owner_operation_id });
+        }
+        else
+            nodes.push({ process_id: record.id, settled: false, reason: "runtime_scope_resource_unsettled", message: "Resource needs its own settlement proof" });
+    }
+    return { scope_id: scopeId, fence, settled: nodes.every(node => node.settled), nodes };
+}

package/dist/services/runtime-scope-worker.js

@@ -0,0 +1,210 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { spawn } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { setTimeout as delay } from "node:timers/promises";
+import { AppError } from "../shared/errors.js";
+import { getResourceDatabase } from "../storage/database.js";
+import { confirmManagedProcess, finishManagedProcess, heartbeatManagedProcess, processIsAlive, processStartedAt, registerManagedProcess, resourceHome } from "./managed-processes.js";
+import { publicRuntimeScopeWorker, requestRuntimeScopeControl, runtimeBudgetStatus } from "./runtime-budget.js";
+import { controlRuntimeScopeRuns } from "./runtime-scope-control.js";
+import { prepareRuntimeScopeResume, recordRuntimeScopeResume } from "./runtime-scope-resume.js";
+import { recoveryObservationBudget, renewRecoveryObservation, withRecoveryObservation } from "./recovery-observation-budget.js";
+export async function launchRuntimeScopeResume(context, input) {
+    const resume = recordRuntimeScopeResume(context, input);
+    if (!resume.current)
+        return { scope_id: input.scopeId, resume, current: false, settled: false };
+    if (resume.status === "released")
+        return { scope_id: input.scopeId, resume, release: resume.release, current: true, dispatch_blocked: resume.dispatch_blocked, settled: false };
+    const control = runtimeBudgetStatus(context, input.scopeId).control;
+    if (control?.generation !== input.generation)
+        throw new AppError("runtime_scope_control_superseded", "Scope resume source changed before owner launch", 1);
+    const owner = await launchRuntimeScopeControl(context, input.scopeId, control.request_id, control.requested_mode, undefined, undefined, input.requestId);
+    return { ...owner, resume, dispatch_blocked: true, settled: false };
+}
+const keyFor = (scope, generation) => crypto.createHash("sha256").update(JSON.stringify([scope, generation])).digest("hex");
+function birth() {
+    const started = processStartedAt(process.pid);
+    if (!started)
+        throw new AppError("runtime_scope_owner_unknown", "Scope owner birth identity is unavailable", 1);
+    return { pid: process.pid, pid_started_at: started };
+}
+export async function reconcileRuntimeScopeControl(context, scopeId, generation, requestId) {
+    const control = runtimeBudgetStatus(context, scopeId).control;
+    if (!control || control.generation !== generation)
+        throw new AppError("runtime_scope_control_superseded", "Reconciliation requires the current controlled scope generation", 1);
+    renewRecoveryObservation({}, requestId);
+    return launchRuntimeScopeControl(context, scopeId, control.request_id, control.requested_mode, undefined, requestId);
+}
+export async function launchRuntimeScopeControl(context, scopeId, requestId, mode, manifest, reconcileId, resumeId) {
+    const control = requestRuntimeScopeControl(context, scopeId, requestId, mode, manifest), db = getResourceDatabase(resourceHome(context)), key = keyFor(scopeId, control.generation);
+    let worker;
+    db.beginWriteTransaction();
+    try {
+        const current = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control' AND resource_key = ?", [scopeId]);
+        const prior = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control-worker' AND resource_key = ?", [key]);
+        const existing = prior ? JSON.parse(prior.metadata_json) : null;
+        if (existing) {
+            const identity = existing.owner ?? existing.launcher;
+            if (!identity || !Number.isSafeInteger(identity.pid) || identity.pid < 1 || typeof identity.pid_started_at !== "string" || !identity.pid_started_at || !["starting", "running", "completed", "failed", "superseded"].includes(existing.status))
+                throw new AppError("runtime_scope_owner_unknown", "Retained scope owner has no verifiable birth identity", 1);
+        }
+        if (!current || JSON.parse(current.metadata_json).generation !== control.generation || db.get("SELECT 1 FROM managed_resources WHERE resource_kind = 'provider-fence' AND resource_key = ?", [scopeId])) {
+            db.exec("COMMIT");
+            return { scope_id: scopeId, control, current: false, settled: false, worker: existing ? publicRuntimeScopeWorker(existing) : null };
+        }
+        if (existing && (existing.owner ? processIsAlive(existing.owner) : existing.status !== "failed" && processIsAlive(existing.launcher))) {
+            db.exec("COMMIT");
+            return { scope_id: scopeId, control, current: true, settled: false, worker: publicRuntimeScopeWorker(existing) };
+        }
+        if (existing?.status === "completed" && !reconcileId && !resumeId) {
+            db.exec("COMMIT");
+            return { scope_id: scopeId, control, current: true, settled: false, worker: publicRuntimeScopeWorker(existing) };
+        }
+        const renewal = resumeId ? `resume:${resumeId}` : reconcileId;
+        const snapshot = renewal ? renewRecoveryObservation(existing?.snapshot ?? {}, renewal) : existing?.snapshot;
+        if (existing && recoveryObservationBudget(snapshot).state().remaining_ms === 0) {
+            db.exec("COMMIT");
+            return { scope_id: scopeId, control, current: true, settled: false, worker: publicRuntimeScopeWorker(existing) };
+        }
+        const cleanupRequest = resumeId ? undefined : reconcileId ?? existing?.reconcile_id;
+        worker = { scope_id: scopeId, generation: control.generation, request_id: requestId, mode, token: crypto.randomUUID(), status: "starting", launcher: birth(), ...(snapshot ? { snapshot } : {}), ...(cleanupRequest ? { reconcile_id: cleanupRequest } : {}) };
+        const now = context.now();
+        db.run("INSERT INTO managed_resources (resource_kind,resource_key,owner_id,lease_token,lease_expires_at,process_id,metadata_json,created_at,updated_at) VALUES ('scope-control-worker',?,?,?,?,NULL,?,?,?) ON CONFLICT(resource_kind,resource_key) DO UPDATE SET lease_token = excluded.lease_token, metadata_json = excluded.metadata_json, updated_at = excluded.updated_at", [key, scopeId, worker.token, now, JSON.stringify(worker), now, now]);
+        db.exec("COMMIT");
+    }
+    catch (error) {
+        db.exec("ROLLBACK");
+        throw error;
+    }
+    try {
+        const entry = fileURLToPath(new URL("../cli.js", import.meta.url));
+        if (!fs.existsSync(entry))
+            throw new AppError("harness_runtime_unbuilt", "Build the CLI before launching scope control", 1);
+        const directory = path.join(resourceHome(context), "scope-controls", key);
+        fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
+        const log = fs.openSync(path.join(directory, "control.log"), "a", 0o600);
+        try {
+            const env = { ...context.env, DD_FLOW_HOME: context.ddFlowHome, DD_FLOW_RESOURCE_HOME: resourceHome(context), DD_FLOW_ENGINE_MODE: "1", DD_FLOW_SCOPE_CONTROL_TOKEN: worker.token };
+            delete env.DD_FLOW_CONTROL_TOKEN;
+            delete env.DD_FLOW_CONTROLLER_TOKEN;
+            const child = spawn(process.execPath, [entry, "runtime", "scope", "serve", "--scope-id", scopeId, "--generation", String(control.generation), "--json"], { env, detached: true, stdio: ["ignore", log, log] });
+            await new Promise((resolve, reject) => { child.once("spawn", resolve); child.once("error", reject); });
+            child.unref();
+        }
+        finally {
+            fs.closeSync(log);
+        }
+    }
+    catch (error) {
+        db.run("UPDATE managed_resources SET metadata_json = json_set(metadata_json, '$.status', 'failed', '$.error', ?), updated_at = ? WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ? AND json_extract(metadata_json, '$.owner') IS NULL", [String(error), context.now(), key, worker.token]);
+    }
+    const retained = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control-worker' AND resource_key = ?", [key]);
+    return { scope_id: scopeId, control, current: true, settled: false, worker: publicRuntimeScopeWorker(JSON.parse(retained.metadata_json)) };
+}
+export async function serveRuntimeScopeControl(context, scopeId, generation) {
+    if (!Number.isSafeInteger(generation) || generation < 1 || !context.env.DD_FLOW_SCOPE_CONTROL_TOKEN)
+        throw new AppError("runtime_scope_owner_unknown", "Scope worker requires its launch token and generation", 1);
+    const db = getResourceDatabase(resourceHome(context)), key = keyFor(scopeId, generation), token = context.env.DD_FLOW_SCOPE_CONTROL_TOKEN;
+    let worker;
+    db.beginWriteTransaction();
+    try {
+        const row = db.get("SELECT metadata_json FROM managed_resources WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ?", [key, token]);
+        if (!row)
+            throw new AppError("runtime_scope_owner_unknown", "Scope worker claim changed", 1);
+        worker = JSON.parse(row.metadata_json);
+        if (worker.owner || worker.status !== "starting")
+            throw new AppError("runtime_scope_already_owned", "Scope control already has a physical owner", 1);
+        worker.owner = birth();
+        worker.status = "running";
+        db.run("UPDATE managed_resources SET metadata_json = ?, updated_at = ? WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ?", [JSON.stringify(worker), context.now(), key, token]);
+        db.exec("COMMIT");
+    }
+    catch (error) {
+        db.exec("ROLLBACK");
+        throw error;
+    }
+    let heartbeat;
+    // A fence must never be held indefinitely by an observer that cannot obtain
+    // fresh provider evidence. A later operator request continues the retained
+    // budget; this owner records why it stopped without claiming settlement.
+    const observation = recoveryObservationBudget(worker.snapshot);
+    try {
+        const record = registerManagedProcess(context, { kind: "scope-control", ownerId: key, operationId: `scope-control:${key}:${token}`, ownerPid: process.pid, metadata: { role: "control", scope_id: scopeId, generation } });
+        worker.process_id = record.id;
+        worker.process_lease_token = record.lease_token;
+        confirmManagedProcess(context, { id: record.id, leaseToken: record.lease_token, pid: process.pid, processGroupId: process.pid, ownerPid: process.pid });
+        db.run("UPDATE managed_resources SET metadata_json = ?, updated_at = ? WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ?", [JSON.stringify(worker), context.now(), key, token]);
+        let leaseLost = false;
+        heartbeat = setInterval(() => { try {
+            if (!heartbeatManagedProcess(context, { id: record.id, leaseToken: record.lease_token }))
+                leaseLost = true;
+        }
+        catch {
+            leaseLost = true;
+        } }, 10_000);
+        heartbeat.unref();
+        for (;;) {
+            if (leaseLost)
+                throw new AppError("runtime_scope_owner_changed", "Scope worker lost its managed lease", 1);
+            if (observation.exhausted()) {
+                worker.status = "failed";
+                worker.error = "recovery_observation_budget_exhausted";
+                worker.snapshot = {
+                    ...(worker.snapshot ?? {}),
+                    status: "recovery_blocked",
+                    settled: false,
+                    pending_reasons: ["recovery_observation_budget_exhausted"],
+                    error: { code: "recovery_observation_budget_exhausted", message: "Scope recovery observation exceeded its 120 second budget" },
+                    recovery_observation: observation.state(),
+                };
+                return { scope_id: scopeId, generation, status: "recovery_blocked", settled: false, error: worker.error };
+            }
+            const status = runtimeBudgetStatus(context, scopeId);
+            const release = status.release;
+            if (release?.current === true && release.generation === generation) {
+                worker.status = "completed";
+                worker.snapshot = release;
+                break;
+            }
+            if (status.control?.generation !== generation || status.fence) {
+                worker.status = "superseded";
+                break;
+            }
+            const resume = status.resume;
+            const outcome = await withRecoveryObservation(observation, async () => (worker.reconcile_id ? null : await prepareRuntimeScopeResume(context, scopeId, generation, resume?.generation === generation ? resume.request_id : undefined))
+                ?? await controlRuntimeScopeRuns(context, scopeId, worker.request_id, worker.mode, worker.reconcile_id));
+            observation.exhausted();
+            worker.snapshot = { ...outcome, recovery_observation: observation.state() };
+            db.run("UPDATE managed_resources SET metadata_json = ?, updated_at = ? WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ?", [JSON.stringify(worker), context.now(), key, token]);
+            if (worker.snapshot.drain?.physical_settled === true) {
+                worker.status = "completed";
+                break;
+            }
+            await delay(observation.nextDelay());
+        }
+        return { scope_id: scopeId, generation, status: worker.status };
+    }
+    catch (error) {
+        if (error instanceof AppError && error.code === "runtime_scope_control_superseded") {
+            const release = runtimeBudgetStatus(context, scopeId).release;
+            worker.status = release?.current === true && release.generation === generation ? "completed" : "superseded";
+            if (worker.status === "completed")
+                worker.snapshot = release;
+            return { scope_id: scopeId, generation, status: worker.status };
+        }
+        worker.status = "failed";
+        worker.error = String(error);
+        throw error;
+    }
+    finally {
+        if (heartbeat)
+            clearInterval(heartbeat);
+        observation.exhausted();
+        worker.snapshot = { ...(worker.snapshot ?? {}), recovery_observation: observation.state() };
+        db.run("UPDATE managed_resources SET metadata_json = ?, updated_at = ? WHERE resource_kind = 'scope-control-worker' AND resource_key = ? AND lease_token = ?", [JSON.stringify(worker), context.now(), key, token]);
+        if (worker.process_id && worker.process_lease_token)
+            finishManagedProcess(context, { id: worker.process_id, leaseToken: worker.process_lease_token, state: worker.status === "failed" ? "failed" : "stopped", reason: "scope_owner_exited" });
+    }
+}

package/dist/services/runtime-service.js

@@ -0,0 +1,96 @@
+import fs from "node:fs";
+import crypto from "node:crypto";
+import path from "node:path";
+import { AppError } from "../shared/errors.js";
+import { assertRunMutationAllowed } from "./run-recovery.js";
+import { requireProjectByRoot } from "./projects.js";
+import { resolveRun } from "./runs.js";
+import { codeExecutionEnvironment, runCheck } from "./code-checks.js";
+import { confirmManagedProcess, finishManagedProcess, heartbeatManagedProcess, managedProcessStatus, processTreeIsAlive, registerManagedProcess, reservePorts, resourceHome, stopManagedProcess } from "./managed-processes.js";
+/** Foreground supervisor: the tool may yield while this command owns the service.
+ * Its ready receipt is emitted immediately; a separate stop ends this command.
+ * Readiness is infrastructure evidence, never a successful scenario verdict. */
+export async function startRuntimeService(context, input) {
+    if (!Number.isFinite(input.timeoutMs) || input.timeoutMs <= 0)
+        throw new AppError("validation", "Readiness timeout must be positive and finite", 2);
+    if (!input.command.trim() || !input.ports.length || input.ports.some(name => !/^[a-z][a-z0-9_]*$/.test(name)) || new Set(input.ports).size !== input.ports.length || !input.ports.includes(input.readyPort) || !input.readyPath.startsWith("/") || input.readyPath.startsWith("//"))
+        throw new AppError("validation", "Declare unique lowercase port names, a ready-port from that list and a local /ready-path", 2);
+    const project = requireProjectByRoot(context, path.resolve(input.projectRoot));
+    const run = resolveRun(context, project.id, input.runId);
+    if (!run?.run_root || !run.workspace_root)
+        throw new AppError("not_found", "Registered RUN with a materialized workspace is required", 2);
+    input = { ...input, runId: run.id };
+    const generation = assertRunMutationAllowed(context, project.id, run.id);
+    const id = `PROC-${crypto.randomUUID()}`;
+    const directory = path.join(run.run_root, "runtime", id);
+    fs.mkdirSync(directory, { recursive: true });
+    const record = registerManagedProcess(context, { id, kind: "runtime-service", ownerId: `run:${input.runId}`, ownerPid: process.pid, runId: input.runId, projectId: project.id, stdoutPath: path.join(directory, "stdout.log"), stderrPath: path.join(directory, "stderr.log"), metadata: { command: input.command } });
+    const receiptPath = path.join(directory, "service.json");
+    let stdout, stderr;
+    let settled = false, ready = false;
+    const receipt = { schema_id: "dd-flow/runtime-service@1", process_id: record.id, run_id: input.runId, status: "starting", started_at: context.now(), receipt_path: receiptPath, stdout_path: path.join(directory, "stdout.log"), stderr_path: path.join(directory, "stderr.log") };
+    const save = () => { const temporary = `${receiptPath}.${process.pid}.tmp`; fs.writeFileSync(temporary, JSON.stringify(receipt, null, 2)); fs.renameSync(temporary, receiptPath); };
+    try {
+        const allocation = await reservePorts(context, { ownerId: record.owner_id, processId: record.id, names: input.ports });
+        const environment = { ...codeExecutionEnvironment(run.workspace_root), ...Object.fromEntries(Object.entries(allocation.ports).map(([name, port]) => [`DD_FLOW_PORT_${name.toUpperCase()}`, String(port)])), DD_FLOW_EVIDENCE_DIR: directory, DD_FLOW_CHECK_COMPLETION_FILE: path.join(directory, "completion.json") };
+        const quoted = (value) => `'${value.replaceAll("'", "'\\''")}'`;
+        Object.assign(receipt, { ports: allocation.ports, environment: Object.fromEntries(Object.entries(environment).filter(([key]) => key.startsWith("DD_FLOW_PORT_"))), stop_command: `DD_FLOW_HOME=${quoted(context.ddFlowHome)} DD_FLOW_RESOURCE_HOME=${quoted(resourceHome(context))} dd-flow runtime process stop --id ${record.id} --lease-token ${record.lease_token} --json` });
+        save();
+        input.progress?.(`service ${record.id} preparing; receipt: ${receiptPath}`);
+        assertRunMutationAllowed(context, project.id, run.id, generation);
+        stdout = fs.openSync(String(receipt.stdout_path), "a");
+        stderr = fs.openSync(String(receipt.stderr_path), "a");
+        const running = runCheck(input.command, run.workspace_root, environment, stdout, stderr, elapsed => input.progress?.(`service ${record.id} alive (${elapsed}s); receipt: ${receiptPath}`), pid => confirmManagedProcess(context, { id: record.id, leaseToken: record.lease_token, pid, processGroupId: process.platform === "win32" ? null : pid }), () => heartbeatManagedProcess(context, { id: record.id, leaseToken: record.lease_token }));
+        void running.then(() => { settled = true; });
+        const deadline = performance.now() + input.timeoutMs;
+        while (!settled && performance.now() < deadline) {
+            try {
+                const response = await fetch(`http://127.0.0.1:${allocation.ports[input.readyPort]}${input.readyPath}`, { signal: AbortSignal.timeout(1_000), redirect: "error" });
+                await response.body?.cancel();
+                ready = response.ok;
+            }
+            catch { /* Startup is observed until the declared deadline. */ }
+            if (ready)
+                break;
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+        if (!ready) {
+            await stopManagedProcess(context, { id: record.id, leaseToken: record.lease_token });
+            await running;
+            throw new AppError("service_not_ready", "Service did not pass its declared HTTP readiness check; inspect retained logs", 2, { receipt_path: receiptPath });
+        }
+        Object.assign(receipt, { status: "ready", ready_at: context.now() });
+        save();
+        input.progress?.(`service ready: ${JSON.stringify(receipt)}; keep this supervisor running, use its exact stop command when finished`);
+        const outcome = await running;
+        const current = managedProcessStatus(context).find(item => item.id === record.id);
+        if (current.state === "stopping") {
+            const settleBy = performance.now() + 5_000;
+            while (processTreeIsAlive(current) && performance.now() < settleBy)
+                await new Promise(resolve => setTimeout(resolve, 25));
+        }
+        if (processTreeIsAlive(current))
+            throw new AppError("process_tree_not_settled", "Service still owns live children; retain resources", 2, { process_id: record.id });
+        if (!["stopped", "failed"].includes(current.state))
+            finishManagedProcess(context, { id: record.id, leaseToken: record.lease_token, state: current.state === "stopping" || outcome.exitCode === 0 ? "stopped" : "failed", reason: current.state === "stopping" ? "requested_stop" : outcome.error });
+        Object.assign(receipt, { status: managedProcessStatus(context).find(item => item.id === record.id).state, finished_at: context.now(), exit_code: outcome.exitCode });
+        save();
+        if (receipt.status === "failed")
+            throw new AppError("service_failed", "Service exited unsuccessfully; inspect retained logs", 2, receipt);
+        return receipt;
+    }
+    catch (error) {
+        const current = managedProcessStatus(context).find(item => item.id === record.id);
+        if (!processTreeIsAlive(current))
+            finishManagedProcess(context, { id: record.id, leaseToken: record.lease_token, state: "failed", reason: error instanceof Error ? error.message : String(error) });
+        Object.assign(receipt, { status: "failed", error: error instanceof Error ? error.message : String(error), cleanup_pending: processTreeIsAlive(current) });
+        save();
+        throw error;
+    }
+    finally {
+        if (stdout !== undefined)
+            fs.closeSync(stdout);
+        if (stderr !== undefined)
+            fs.closeSync(stderr);
+    }
+}

package/dist/services/schema-validation.js

@@ -6,7 +6,7 @@ import { Ajv } from "ajv/dist/ajv.js";
 import { Ajv2020 } from "ajv/dist/2020.js";
 import { normalizeFlowContract } from "../domain/flow-contract.js";
 import { AppError } from "../shared/errors.js";
-import { findRunHome, readRunEngineBinding } from "./run-engine-bindings.js";
+import { readRunEngineBinding } from "./run-engine-bindings.js";
 export function captureMemoryBankBaseline(input) {
     const projectRoot = path.resolve(input.projectRoot);
     const paths = normalizeMemoryBankPaths(projectRoot, input.paths);
@@ -94,7 +94,7 @@ export function validateSchema(options) {
         throw new AppError("usage", "--schema must be a schema name such as mb-upgrade-review-data", 2);
     }
     const filePath = path.resolve(options.file);
-    const data = readJson(filePath, "input file");
+    const data = Object.hasOwn(options, "data") ? options.data : readJson(filePath, "input file");
     const schemaResolution = resolveSchema(options);
     const schema = readJson(schemaResolution.path, "schema");
     // Ajv's default export only carries the draft-07 meta-schema.  New stage
@@ -117,8 +117,8 @@ export function validateSchema(options) {
     }
     const valid = validate(data);
     const errors = valid ? [] : (validate.errors ?? []).map(formatAjvError);
-    const semanticErrors = valid ? validateSemanticSchema(options.schemaName, data) : [];
-    const allErrors = [...errors, ...semanticErrors];
+    const contractErrors = valid ? validateContractInvariants(options.schemaName, data) : [];
+    const allErrors = [...errors, ...contractErrors];
     if (allErrors.length > 0) {
         throw new AppError("schema_validation", `${path.basename(filePath)} does not match schema`, 2, {
             schema: schemaResolution,
@@ -181,7 +181,7 @@ const historicalSchemaRegistry = {
         }]
 };
 function resolveRunBoundSchema(options) {
-    if (!options.projectRoot || options.schemaDir)
+    if (!options.runRoot || options.schemaDir)
         return null;
     const data = readJson(path.resolve(options.file), "input file");
     const root = asRecord(data);
@@ -190,13 +190,13 @@ function resolveRunBoundSchema(options) {
     const runId = options.runId ?? inferredRunId;
     if (typeof runId !== "string")
         return null;
-    const projectRoot = path.resolve(options.projectRoot);
-    const located = findRunHome(options.ddFlowHome ?? process.env.DD_FLOW_HOME ?? path.join(path.dirname(projectRoot), ".dd-flow"), projectRoot, runId);
-    if (!located || !isInside(located.run_home, path.resolve(options.file)))
-        return null;
-    const binding = readRunEngineBinding(located.binding_path, { allowMissing: true });
+    const runRoot = path.resolve(options.runRoot);
+    const bindingPath = path.join(runRoot, "engine-binding.json");
+    const binding = readRunEngineBinding(bindingPath);
     if (!binding)
-        return null;
+        throw new AppError("run_engine_binding_missing", "RUN engine binding is missing", 1, { path: bindingPath });
+    if (binding.run_id !== runId)
+        throw new AppError("run_engine_binding_invalid", "RUN engine binding belongs to another RUN", 1, { expected_run_id: runId, actual_run_id: binding.run_id, path: bindingPath });
     const manifestPath = path.join(binding.engine.snapshot_root, "engine.json");
     const manifest = readJson(manifestPath, "bound engine manifest");
     const integrity = asRecord(objectValue(manifest, "integrity"));
@@ -284,7 +284,7 @@ function formatAjvError(error) {
         keyword: error.keyword
     };
 }
-function validateSemanticSchema(schemaName, data) {
+function validateContractInvariants(schemaName, data) {
     const root = asRecord(data);
     if (!root) {
         return [];

package/dist/services/stage-context.js

@@ -76,8 +76,8 @@ export function renderExternalStageContext(input) {
     ].join("\n");
 }
 function validateContext(value, expectedStage) {
-    if (value.schema_id !== "dd-eval/stage-context@1")
-        throw new AppError("validation", "Stage context must use dd-eval/stage-context@1", 2);
+    if (value.schema_id !== "dd-flow/stage-context@1" && value.schema_id !== "dd-eval/stage-context@1")
+        throw new AppError("validation", "Stage context must use dd-flow/stage-context@1", 2);
     if (value.stage !== expectedStage)
         throw new AppError("validation", "Stage context does not match --stage", 2, { expected_stage: expectedStage, context_stage: value.stage });
     if (!value.objective?.trim())

package/dist/services/stage-lifecycle.js

@@ -8,7 +8,7 @@ import { persistProtocolState, requireProtocol, readProtocolRuntimeState } from
 import { attachFlowRunStage, completeFlowRunStage, getFlowRunStatus, startFlowRun } from "./runs.js";
 import { validateSchema } from "./schema-validation.js";
 import { reconcileSessionCoverage, registerFlowSession } from "./sessions.js";
-import { usageForRun } from "./usage.js";
+import { recalculateRunUsage, usageForRun } from "./usage.js";
 import { boundCanonicalPlan } from "./plan-runtime.js";
 import { AppError } from "../shared/errors.js";
 import { planJsonPath, resolveProjectRoot } from "../storage/paths.js";
@@ -18,6 +18,17 @@ import { registerProject } from "./projects.js";
 import { registerProtocol } from "./protocols.js";
 import { previewNextEntityId } from "./ids.js";
 import { hookSessionIdentity, sessionIdForHookEvent } from "./hooks.js";
+import { assertRunMutationAllowed } from "./run-recovery.js";
+function stageUsageProjection(context, view, stage, startedAt) {
+    if (!startedAt)
+        return { status: "unavailable", input_tokens: null, cached_input_tokens: null, output_tokens: null, total_tokens: null, session_count: 0, observed_at: context.now() };
+    const value = recalculateRunUsage(context, { projectId: view.run.project_id, runId: view.run.id, stage: { name: stage, startedAt, completedAt: context.now() } });
+    const totals = value.totals, coverage = value.coverage;
+    const count = (name) => typeof coverage?.[name] === "number" ? coverage[name] : 0;
+    const measured = count("measured"), other = Object.values(coverage ?? {}).reduce((total, item) => total + (typeof item === "number" ? item : 0), 0) - measured;
+    const tokens = (name) => typeof totals?.[name] === "number" ? totals[name] : null;
+    return { status: measured === 0 ? "unavailable" : other > 0 || ["input_tokens", "cache_read_input_tokens", "output_tokens", "total_tokens"].some(name => tokens(name) === null) ? "partial" : "measured", input_tokens: tokens("input_tokens"), cached_input_tokens: tokens("cache_read_input_tokens"), output_tokens: tokens("output_tokens"), total_tokens: tokens("total_tokens"), session_count: Array.isArray(value.sessions) ? value.sessions.length : 0, observed_at: typeof value.observed_at === "string" ? value.observed_at : context.now() };
+}
 const stagePromptSections = [
     "stage_identity",
     "authoritative_runtime_facts",
@@ -53,7 +64,9 @@ function bootstrapStageRun(context, projectRoot, input) {
         if (!fs.existsSync(source) || !fs.statSync(source).isFile()) {
             throw new AppError("not_found", "--intake-file must point to an existing file", 1, { intake_file: intakeFile });
         }
-        const intakeDir = path.join(started.run.run_home_path ?? path.dirname(started.run.id), "intake");
+        if (!started.run.run_root)
+            throw new AppError("runtime_missing", "New RUN has no artifact root", 1, { run_id: started.run.id });
+        const intakeDir = path.join(started.run.run_root, "intake");
         fs.mkdirSync(intakeDir, { recursive: true });
         fs.copyFileSync(source, path.join(intakeDir, "user-request.md"));
     }
@@ -117,6 +130,7 @@ export function startStage(context, input) {
     if (!runId)
         throw new AppError("usage", "stage start requires a RUN id or --bootstrap", 2);
     const before = runView(getFlowRunStatus(context, { projectRoot, runId }));
+    assertRunMutationAllowed(context, before.run.project_id, before.run.id);
     const dir = input.dir ?? defaultStageDir(input.stage, before.run.flow_kind);
     const attached = runView(attachFlowRunStage(context, {
         projectRoot,
@@ -216,7 +230,7 @@ export function startStage(context, input) {
         })),
         worker_prompt_markdown: prompt
     });
-    validateSchema({ schemaName: "stage-prompt", file: promptDataPath, projectRoot, runId: attached.run.id, ddFlowHome: context.ddFlowHome });
+    validateSchema({ schemaName: "stage-prompt", file: promptDataPath, projectRoot, runId: attached.run.id, runRoot: runHome, ddFlowHome: context.ddFlowHome });
     atomicWrite(path.join(stageRoot, "stage-start.json"), {
         schema_id: "dd-flow/stage-start@2",
         run_id: attached.run.id,
@@ -261,6 +275,7 @@ export function finishStage(context, input) {
         throw new AppError("usage", "stage finish requires a RUN id", 2);
     const runId = input.runId;
     const view = runView(getFlowRunStatus(context, { projectRoot, runId }));
+    assertRunMutationAllowed(context, view.run.project_id, view.run.id);
     const existing = view.index.stage_runs?.find((stage) => stage.stage === input.stage);
     if (!existing) {
         throw new AppError("not_found", `Run stage is not attached: ${input.stage}`, 1, { run_id: view.run.id, stage: input.stage });
@@ -273,7 +288,7 @@ export function finishStage(context, input) {
     const semanticFile = resolveStageFile(input.semanticFile ?? "@stage/stage-input.json", stageRoot, runHome);
     const receipt = beginFinishReceipt(context, stageRoot, semanticFile, view.run.id, input.stage);
     try {
-        validateSchema({ schemaName: "stage-finish-input", file: semanticFile, projectRoot, runId, ddFlowHome: context.ddFlowHome });
+        validateSchema({ schemaName: "stage-finish-input", file: semanticFile, projectRoot, runId, runRoot: runHome, ddFlowHome: context.ddFlowHome });
         const semantic = readSemanticFile(semanticFile, runHome);
         const status = stringValue(semantic.status) ?? "done";
         if (!["done", "waiting_for_user", "blocked", "failed"].includes(status)) {
@@ -288,12 +303,13 @@ export function finishStage(context, input) {
         const planFinish = input.stage === "plan"
             ? preparePlanFinish(context, projectRoot, view, stageRoot)
             : undefined;
+        const stageUsage = stageUsageProjection(context, view, input.stage, existing.started_at);
         const report = buildStageReport(context, view, input.stage, status, semantic, stageRoot, lint, planFinish);
         const dataPath = path.join(stageRoot, "stage-report.json");
         const reportPath = path.join(stageRoot, "stage-report.md");
         const htmlPath = path.join(stageRoot, "stage-report.html");
         atomicWrite(dataPath, report);
-        validateSchema({ schemaName: stageReportSchemaName(input.stage, planFinish), file: dataPath, projectRoot, runId, ddFlowHome: context.ddFlowHome });
+        validateSchema({ schemaName: stageReportSchemaName(input.stage, planFinish), file: dataPath, projectRoot, runId, runRoot: runHome, ddFlowHome: context.ddFlowHome });
         atomicWrite(reportPath, renderMarkdown(report));
         atomicWrite(htmlPath, renderHtml(projectRoot, report));
         if (view.run.subject.type === "protocol") {
@@ -308,7 +324,8 @@ export function finishStage(context, input) {
             stageReport: htmlPath,
             data: dataPath,
             dataSchemaId: String(report.schema_id),
-            report: reportPath
+            report: reportPath,
+            usage: stageUsage
         });
         finishReceipt(receipt, { outcome: "accepted", result: { status, report: dataPath } });
         appendAudit(context, {
@@ -410,7 +427,12 @@ function nextActionForProtocolStage(stage) {
     return `continue_${stage}`;
 }
 function runHomePath(run) {
-    return run.run_home_path ?? path.dirname(run.run_index_path);
+    if (!run.run_root)
+        throw new AppError("runtime_missing", "RUN has no artifact root", 1, { run_id: run.id });
+    return run.run_root;
+}
+function runStatePath(run) {
+    return path.join(runHomePath(run), "run.json");
 }
 export function defaultStageDir(stage, flowKind) {
     const normalized = stage === "implementation" ? "code" : stage;
@@ -702,7 +724,7 @@ function buildStageReport(context, view, stage, status, semantic, stageRoot, lin
     if (stage === "code" || stage === "implementation") {
         return {
             schema_id: "dd-flow/code-stage-report@2",
-            run: { run_id: view.run.id, run_state: view.run.run_index_path },
+            run: { run_id: view.run.id, run_state: runStatePath(view.run) },
             stage: { name: stage, dir: path.basename(stageRoot), status },
             project: { id: view.run.project_id, title: path.basename(view.run.project_root) },
             subject: { id: view.run.subject.id, title: view.run.subject.id },
@@ -712,7 +734,7 @@ function buildStageReport(context, view, stage, status, semantic, stageRoot, lin
                 summary: result,
                 next_action: stringValue(semantic.next_action) ?? "Proceed to readiness."
             },
-            breadcrumbs: [{ label: "RUN", href: view.run.run_index_path, status: "available" }],
+            breadcrumbs: [{ label: "RUN", href: runStatePath(view.run), status: "available" }],
             implemented_goals: [{ title: `Stage ${stage}`, summary: result }],
             acceptance_scenarios: acceptance.map((summary, index) => ({
                 id: `SCN-${stage}-${index + 1}`,
@@ -763,7 +785,7 @@ function buildStageReport(context, view, stage, status, semantic, stageRoot, lin
             memory_bank_scope: "changed_files_and_links_only",
             status: "passed"
         },
-        breadcrumbs: [{ label: "RUN", href: view.run.run_index_path, status: "available" }]
+        breadcrumbs: [{ label: "RUN", href: runStatePath(view.run), status: "available" }]
     };
 }
 function preparePlanFinish(context, projectRoot, view, stageRoot) {