@deksden-com/dd-flow-cli 0.9.0-beta.15 → 0.9.0-beta.151

package/dist/harness-runtime/lib/managed-daemon.mjs

@@ -0,0 +1,698 @@
+import { lstat, readFile, readdir, mkdir, appendFile } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import path from "node:path";
+import { commandJson } from "./process-json.mjs";
+import { errorRecord, reportedError, diagnosticJson } from "./operation-errors.mjs";
+import { writeJsonAtomic } from "./runner-events.mjs";
+import { runBoundedCommand } from "./native-hook-command.mjs";
+import { processSnapshot } from "./process-snapshot.mjs";
+import { createRenewalState, maintenanceRetryable, renewalConfirmed, renewalDelay, renewalFailure, renewalRemaining, RENEWAL_POLICY } from "./lease-renewal.mjs";
+
+const leases = new Map();
+// A rejected Promise is not physical exit. Do not start another writer for
+// this exact binding after an unconfirmed maintenance subtree cleanup.
+const maintenancePoison = new Map();
+const maintenanceFlights = new Map();
+
+/** Freeze resource ownership independently of whether Subject hooks are enabled. */
+export function bindRuntimeOwner(config, env = process.env) {
+  const raw = Object.hasOwn(config, "runtime_owner") ? config.runtime_owner === null ? undefined : JSON.stringify(config.runtime_owner) : env.DD_FLOW_RUNTIME_OWNER;
+  if (raw === undefined) return config.runtime_owner = null;
+  let owner;
+  try { if (raw.length > 16_384) throw new Error(); owner = JSON.parse(raw); }
+  catch { throw Object.assign(new Error("runtime ownership descriptor is invalid"), { code: "runtime_owner_invalid" }); }
+  const fields = ["schema_id", "owner_id", "role", "project_id", "run_id", "work_id", "operation_id", "state_dir", "dd_flow_home", "dd_flow_bin", "resource_home", "adapter_executable", "budget"];
+  const text = value => typeof value === "string" && value.trim().length > 0;
+  if (!owner || typeof owner !== "object" || Array.isArray(owner) || Object.keys(owner).some(key => !fields.includes(key))
+    || owner.schema_id !== "dd-flow/runtime-owner@1" || !text(owner.owner_id) || !text(owner.operation_id)
+    || !["subject", "worker", "merge", "judge", "probe"].includes(owner.role)
+    || ["state_dir", "dd_flow_home", "dd_flow_bin", "resource_home"].some(key => !text(owner[key]) || !path.isAbsolute(owner[key]))
+    || (owner.adapter_executable !== undefined && (!text(owner.adapter_executable) || !path.isAbsolute(owner.adapter_executable)))
+    || ["project_id", "run_id", "work_id"].some(key => owner[key] !== undefined && !text(owner[key]))
+    || (["subject", "worker", "merge"].includes(owner.role) && (!text(owner.project_id) || !text(owner.run_id)))
+    || (["worker", "merge"].includes(owner.role) && !text(owner.work_id))
+    || (["judge", "probe"].includes(owner.role) && owner.work_id !== undefined)) {
+    throw Object.assign(new Error("runtime ownership descriptor does not match its role and scope"), { code: "runtime_owner_invalid" });
+  }
+  if (owner.budget !== undefined) {
+    const budget = owner.budget, names = ["codex-desktop", "zcode-acp", "grok-acp", "antigravity-cli", "opencode-server", "droid-cli"];
+    if (!budget || typeof budget !== "object" || Array.isArray(budget) || Object.keys(budget).some(key => !["schema_id", "scope_id", "per_harness"].includes(key)) || budget.schema_id !== "dd-flow/runtime-budget@1" || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(budget.scope_id ?? "") || !budget.per_harness || typeof budget.per_harness !== "object" || Array.isArray(budget.per_harness) || Object.entries(budget.per_harness).some(([name, limit]) => !names.includes(name) || !Number.isSafeInteger(limit) || limit < 1 || limit > 1024)) throw Object.assign(new Error("runtime owner budget is invalid"), { code: "runtime_budget_invalid" });
+  }
+  config.runtime_owner = Object.fromEntries(fields.filter(key => owner[key] !== undefined).map(key => [key, owner[key]]));
+  return config.runtime_owner;
+}
+
+/** Reject rebinding before provider setup, including the already-running path. */
+export async function prepareRuntimeOwner(config, stateDir, env = process.env) {
+  config.stateDir = path.resolve(stateDir);
+  const owner = bindRuntimeOwner(config, env);
+  if (owner && path.resolve(owner.state_dir) !== path.resolve(stateDir)) {
+    throw Object.assign(new Error("runtime owner belongs to another state directory"), { code: "runtime_owner_mismatch" });
+  }
+  let previous;
+  try { previous = JSON.parse(await readFile(path.join(stateDir, "daemon.json"), "utf8")); }
+  catch (error) { if (error.code === "ENOENT") return owner; throw error; }
+  const retained = bindRuntimeOwner({ runtime_owner: previous?.config?.runtime_owner ?? null }, {});
+  if (JSON.stringify(retained) !== JSON.stringify(owner)) {
+    throw Object.assign(new Error("retained daemon ownership differs; use explicit recovery instead of rebinding its sessions"), { code: "runtime_owner_mismatch" });
+  }
+  return owner;
+}
+
+export async function assertDaemonOwnership(stateDir, dispatch = {}, assertCurrent = () => {}, progress = async () => {}, waiting = async () => {}) {
+  const gate = async () => {
+    assertCurrent();
+    let confirmed = false;
+    for (const lease of leases.values()) {
+      if (lease.stateDir !== path.resolve(stateDir)) continue;
+      await reconfirmLease(lease, assertCurrent);
+      assertCurrent();
+      confirmed = true;
+    }
+    if (confirmed) await progress('ownership_confirmed');
+  };
+  await gate();
+  const state = JSON.parse(await readFile(path.join(stateDir, "daemon.json"), "utf8"));
+  const owner = bindRuntimeOwner({ runtime_owner: state.config?.runtime_owner ?? null }, {});
+  if (!owner) return;
+  const record = state.resource_process;
+  if (owner.state_dir !== stateDir || !record?.id || !record.lease_token) throw Object.assign(new Error("owned daemon has no confirmed resource binding"), { code: "process_ownership_unknown" });
+  scheduleDaemonProcessHeartbeat(state.config, record);
+  const admissionRenewal = createRenewalState({ leaseExpiresAt: record.lease_expires_at });
+  for (;;) {
+    await gate();
+    // daemon.json is a snapshot, not the current authority. gate() verified
+    // the exact retained owner and its latest committed renewal above.
+    const retained = leases.get(leaseKey(state.config, record));
+    if (!retained || record.registration_sha256 && record.registration_sha256 !== retained.record.registration_sha256) throw Object.assign(new Error("Admission registration differs from confirmed owner"), { code: "process_maintenance_receipt_invalid" });
+    retainReceipt(record, retained.record);
+    if (!renewalConfirmed(admissionRenewal, performance.now(), record.lease_expires_at)) requireTimelyMaintenance(admissionRenewal, "admission", retained.record, record);
+    admissionRenewal.uncertainSince ??= performance.now();
+    let result;
+    try { result = await runtimeProcess(state.config, "admission", { id: record.id, "lease-token": record.lease_token, owner: owner.owner_id, "state-dir": stateDir, "daemon-id": state.daemon_id ?? state.config?.daemonId, ...dispatch }, { timeoutMs: renewalRemaining(admissionRenewal) }); }
+    catch (error) {
+      if (!maintenanceRetryable(error) || !renewalFailure(admissionRenewal)) throw error;
+      await new Promise(resolve => setTimeout(resolve, Math.min(renewalDelay(admissionRenewal), renewalRemaining(admissionRenewal))));
+      continue;
+    }
+    assertCurrent();
+    if (result?.ok !== true || result.process_id !== record.id || result.admitted !== true && result.reason !== "provider_capacity") throw Object.assign(new Error("productive runtime admission was not confirmed"), { code: "process_ownership_unknown" });
+    requireTimelyMaintenance(admissionRenewal, "admission", result, record);
+    // This bound ACK ends admission uncertainty. The post-ACK lease checks
+    // have their own episodes; successful renewal must not exhaust an already
+    // confirmed admission while the provider/daemon still retain ownership.
+    if (!renewalConfirmed(admissionRenewal)) requireTimelyMaintenance(admissionRenewal, "admission", result, record);
+    await gate();
+    await waiting({ ...result, lease_expires_at: result.lease_expires_at ?? record.lease_expires_at });
+    if (result.admitted === true) { await progress('admission_confirmed'); return; }
+    const predecessor = result.blocking_turns?.find(turn => turn?.process_id === record.id
+      && turn?.session_id === dispatch["native-session-id"]
+      && turn?.operation_id !== dispatch["operation-id"]);
+    if (predecessor) throw Object.assign(new Error("the same managed Session has an unsettled productive predecessor; reconcile it before dispatching another Turn"), {
+      code: "settlement_reconciliation_required",
+      details: { session_id: dispatch["native-session-id"], predecessor_operation_id: predecessor.operation_id, predecessor_operation: predecessor.operation }
+    });
+    // The daemon, not its disposable socket client, owns this queue. Recheck
+    // the RUN fence and process lease on every admission attempt.
+    await new Promise(resolve => setTimeout(resolve, 250));
+  }
+}
+
+/** Snapshot terminal operations BEFORE observing native idle. A later Turn
+ * cannot be released by an older observation of the same Session. */
+export async function budgetObservationScope(stateDir, sessionId) {
+  const state = await budgetState(stateDir);
+  if (!state?.config?.runtime_owner?.budget || !sessionId) return null;
+  const daemonId = state.daemon_id ?? state.config?.daemonId, operationIds = [];
+  for (const name of await readdir(path.join(stateDir, "operations")).catch(error => { if (error.code === "ENOENT") return []; throw error; })) {
+    const directory = path.join(stateDir, "operations", name);
+    try {
+      const requested = JSON.parse(await readFile(path.join(directory, "requested.json"), "utf8"));
+      const result = JSON.parse(await readFile(path.join(directory, "result.json"), "utf8"));
+      if (requested.daemon_id === daemonId && requested.session_id === sessionId && ["completed", "failed", "interrupted"].includes(result.state) && ["session.prompt", "session.start", "session.resume"].includes(requested.operation)) operationIds.push(requested.operation_id);
+    } catch (error) { if (error.code !== "ENOENT") throw error; }
+  }
+  return { stateDir, state, daemonId, sessionId, operationIds };
+}
+
+export async function releaseBudgetObservation(scope, observation) {
+  if (!scope || !scope.operationIds.length || observation?.settled !== true) return;
+  const id = observation.provider_session_id ?? observation.session_id ?? observation.adapter_session_id;
+  if (id !== scope.sessionId) throw Object.assign(new Error("budget observation belongs to another Session"), { code: "session_identity_mismatch" });
+  const directory = path.join(scope.stateDir, "budget-observations");
+  await mkdir(directory, { recursive: true, mode: 0o700 });
+  const file = path.join(directory, `${randomUUID()}.json`);
+  await writeJsonAtomic(file, { daemon_id: scope.daemonId, session_id: scope.sessionId, settled: true, operation_ids: scope.operationIds, observed_at: new Date().toISOString() });
+  await releaseBudget(scope.state, { "observation-file": file });
+}
+
+export async function completeBudgetOperation(stateDir, request, result, observe) {
+  const state = await budgetState(stateDir);
+  if (!state?.config?.runtime_owner?.budget) return { state: "not_required" };
+  if (["session.create", "session.fork"].includes(request.operation)) {
+    await releaseBudget(state, { "operation-id": request.id });
+    return { state: "settled", operation_ids: [request.id], source: "creation_receipt" };
+  }
+  const sessionId = request.params?.sessionId;
+  // A productive daemon receipt is bound to this exact dispatch. Reuse its
+  // complete tree proof instead of immediately asking the provider again.
+  if (["session.prompt", "session.start", "session.resume"].includes(request.operation)
+    && result?.settled === true
+    && typeof sessionId === "string"
+    && (result.provider_session_id ?? result.session_id ?? result.adapter_session_id) === sessionId) {
+    const scope = { stateDir, state, daemonId: state.daemon_id ?? state.config?.daemonId, sessionId, operationIds: [request.id] };
+    await releaseBudgetObservation(scope, result);
+    return { state: "settled", operation_ids: [request.id], source: "operation_receipt" };
+  }
+  // A known live tree is not a reason to poll it again. Preserve capacity and
+  // let the next explicit reconciliation obtain fresh proof.
+  if (result?.settled === false) return { state: "pending", reason: "tree_unsettled" };
+  const scope = await budgetObservationScope(stateDir, request.params?.sessionId);
+  if (!scope?.operationIds.length) return { state: "not_required" };
+  // Reconciliation requires a fresh, explicit proof.  Treating an arbitrary
+  // productive reply as proof would silently release a live turn.
+  if (!observe) return { state: "pending", reason: "settlement_evidence_required" };
+  const observation = await observe();
+  if (observation?.settled !== true) return { state: "pending", reason: "tree_unsettled" };
+  await releaseBudgetObservation(scope, observation);
+  return { state: "settled", operation_ids: scope.operationIds, source: "reconciliation_observation" };
+}
+
+async function budgetState(stateDir) {
+  try { return JSON.parse(await readFile(path.join(stateDir, "daemon.json"), "utf8")); }
+  catch (error) { if (error.code === "ENOENT") return null; throw error; }
+}
+
+async function releaseBudget(state, options) {
+  const record = state.resource_process, owner = state.config.runtime_owner;
+  const receipt = await retryBookkeeping(timeoutMs => runtimeProcess(state.config, "release-turn", { id: record.id, "lease-token": record.lease_token, owner: owner.owner_id, "daemon-id": state.daemon_id ?? state.config?.daemonId, ...options }, { timeoutMs }));
+  if (receipt?.ok !== true) throw Object.assign(new Error("runtime budget release was not confirmed"), { code: "runtime_budget_release_unconfirmed" });
+}
+
+export async function runtimeProcess(config, action, options = {}, { timeoutMs = RENEWAL_POLICY.budgetMs, signal } = {}) {
+  const resourceHome = config.runtime_owner?.resource_home ?? config.resourceHome;
+  if (action === "status" || !options.id || !resourceHome) return executeRuntimeProcess(config, action, options, { timeoutMs, signal });
+  const key = JSON.stringify([path.resolve(resourceHome), options.id, options["lease-token"] ?? null]);
+  const validBudget = Number.isFinite(timeoutMs) && timeoutMs > 0;
+  const deadline = performance.now() + (validBudget ? timeoutMs : 0);
+  const prior = maintenanceFlights.get(key);
+  let started = false;
+  let settle;
+  const flight = new Promise(resolve => { settle = resolve; });
+  maintenanceFlights.set(key, flight);
+  try {
+    if (!validBudget) throw Object.assign(new Error("Managed maintenance budget is invalid or exhausted before dispatch"), { code: "process_maintenance_timeout", details: { effect: "no_effect", native_dispatch_started: false } });
+    if (prior) {
+      let timer;
+      try {
+        await Promise.race([prior, new Promise((_, reject) => {
+          const expire = () => {
+            const remaining = deadline - performance.now();
+            // A timer wakeup is not proof that its monotonic deadline passed.
+            if (remaining > 0) { timer = setTimeout(expire, Math.min(2_147_483_647, Math.ceil(remaining))); return; }
+            reject(Object.assign(new Error("managed maintenance queue deadline expired"), { code: "process_maintenance_timeout", details: { effect: "no_effect" } }));
+          };
+          expire();
+        })]);
+      } finally { clearTimeout(timer); }
+    }
+    const remaining = deadline - performance.now();
+    if (remaining <= 0) throw Object.assign(new Error("managed maintenance queue deadline expired"), { code: "process_maintenance_timeout", details: { effect: "no_effect" } });
+    started = true;
+    return await executeRuntimeProcess(config, action, options, { timeoutMs: remaining, signal });
+  } catch (error) {
+    const lease = leases.get(key);
+    // A settled transport may have lost its ACK after COMMIT. Observe the exact
+    // owner, never replay the writer. Productive admission still checks RUN.
+    const registration = lease?.record.registration_sha256 ?? options["registration-sha256"];
+    if (action === "heartbeat" && error.details?.effect === "unknown" && error.details?.cleanup_unconfirmed !== true && /^[a-f0-9]{64}$/.test(registration ?? "")) {
+      const readback = await executeRuntimeProcess(config, "status", { id: options.id, "lease-token": options["lease-token"], "registration-sha256": registration }, { timeoutMs: RENEWAL_POLICY.budgetMs, signal });
+      if (readback?.ok === true && readback.process_id === options.id && readback.registration_sha256 === registration) return readback;
+      throw Object.assign(new Error("Managed heartbeat readback did not establish the exact effect"), { code: "process_maintenance_receipt_invalid" });
+    }
+    const text = value => typeof value === "string" ? value.slice(0, 128) : undefined;
+    const request = { phase: started ? "request" : "queue", action, process_id: text(options.id), operation_id: text(options["operation-id"] ?? options.operation ?? config.runtime_owner?.operation_id), role: text(lease?.record.kind ?? options.role ?? config.runtime_owner?.role), attempt_kind: prior ? "join" : "fresh", attempt: (lease?.renewal.attempt ?? 0) + 1, remaining_ms: Math.max(0, Math.round(deadline - performance.now())), native_dispatch_started: false, maintenance_effect: started ? "unknown" : "no_effect" };
+    // Parent queue/import failures precede any child trace. Keep only bounded,
+    // credential-free context and the existing child phase allowlist.
+    const childPhases = Array.isArray(error.details?.hook_diagnostics) ? error.details.hook_diagnostics : [];
+    error.details = { ...error.details, hook_diagnostics: [request, ...childPhases.slice(-15)] };
+    throw error;
+  } finally {
+    // Even a waiter whose deadline expires must retain its predecessor's
+    // physical fence until that predecessor settles.
+    const finish = () => {
+      settle();
+      if (maintenanceFlights.get(key) === flight) maintenanceFlights.delete(key);
+    };
+    if (prior) void prior.then(finish); else finish();
+  }
+}
+
+/** Only after the original maintenance transport physically settled. This
+ * endpoint reads exact current authority and cannot renew or reserve a turn. */
+export async function readbackDaemonProcess(config, record, { signal } = {}) {
+  if (!record?.id || !record.lease_token || !/^[a-f0-9]{64}$/.test(record.registration_sha256 ?? "")) throw Object.assign(new Error("Readback has no exact registration"), { code: "process_maintenance_receipt_invalid" });
+  const result = await runtimeProcess(config, "status", { id: record.id, "lease-token": record.lease_token, "registration-sha256": record.registration_sha256 }, { timeoutMs: RENEWAL_POLICY.budgetMs, signal });
+  if (result?.ok !== true || result.process_id !== record.id || result.registration_sha256 !== record.registration_sha256) throw Object.assign(new Error("Readback returned foreign authority"), { code: "process_maintenance_receipt_invalid" });
+  retainReceipt(record, result);
+  return result;
+}
+
+async function executeRuntimeProcess(config, action, options = {}, { timeoutMs, signal }) {
+  const owner = config.runtime_owner;
+  const resourceHome = owner?.resource_home ?? config.resourceHome;
+  const ddFlowHome = owner?.dd_flow_home ?? config.ddFlowHome;
+  const ddFlowBin = owner?.dd_flow_bin ?? config.ddFlowBin;
+  if (!resourceHome || !ddFlowHome || !ddFlowBin) return null;
+  const poisonKey = token => JSON.stringify([path.resolve(resourceHome), options.id, token ?? null]);
+  if (action !== "status" && options.id) {
+    const poison = maintenancePoison.get(poisonKey(options["lease-token"])) ?? maintenancePoison.get(poisonKey(null));
+    if (poison && !(action === "finish" && poison.finish === JSON.stringify(options))) throw poison.error;
+  }
+  const args = ["runtime", "process", action];
+  for (const [key, value] of Object.entries(options)) if (value !== null && value !== undefined) args.push(`--${key}`, String(value));
+  if (["register", "confirm", "heartbeat", "admission", "check-admission", "finish", "release-turn", "status"].includes(action)) {
+    const maintenanceStarted = performance.now();
+    process.stderr.write(`${JSON.stringify({ kind: "process_maintenance_phase", phase: "request", observed_at: new Date().toISOString(), operation_id: options['operation-id'] ?? config.runtime_owner?.operation_id ?? null, action, process_id: options.id ?? null, remaining_ms: Math.round(timeoutMs), native_dispatch_started: false })}\n`);
+    let output;
+    try { output = await runBoundedCommand({ bin: ddFlowBin, args: [...args, "--json"], cwd: config.cwd,
+      env: { ...(config.env ?? {}), DD_FLOW_HOME: ddFlowHome, DD_FLOW_RESOURCE_HOME: resourceHome }, timeoutMs,
+      timeoutCode: action === "heartbeat" ? "process_heartbeat_timeout" : "process_maintenance_timeout", inheritDeadline: false, cleanupWithinBudget: true,
+      // Storage silence is not loss of authority: the original command/lease
+      // deadline remains finite, including slow I/O and multiple bounded reads.
+      maintenanceProgress: { action, processId: options.id ?? null, startupMs: Math.min(timeoutMs, RENEWAL_POLICY.startupMs), inactivityMs: timeoutMs }, signal }); }
+    catch (error) {
+      if (action !== "status" && options.id && error.details?.cleanup_unconfirmed) maintenancePoison.set(poisonKey(options["lease-token"]), {
+        error, ...(action === "finish" && error.code === "process_maintenance_timeout" ? { finish: JSON.stringify(options) } : {})
+      });
+      throw error;
+    }
+    let result;
+    try { result = JSON.parse(output); } catch { throw Object.assign(new Error("Invalid managed process maintenance response"), { code: "process_maintenance_receipt_invalid", details: { action } }); }
+    if (result?.ok === false && result.error?.code) throw Object.assign(new Error(result.error.message ?? "Managed process maintenance failed"), { code: result.error.code, details: result.error.details });
+    process.stderr.write(`${JSON.stringify({ kind: "process_maintenance_phase", phase: "ack", observed_at: new Date().toISOString(), duration_ms: Math.round(performance.now() - maintenanceStarted), operation_id: options['operation-id'] ?? config.runtime_owner?.operation_id ?? null, action, process_id: options.id ?? null, lease_expires_at: result.lease_expires_at ?? result.process?.lease_expires_at ?? null, native_dispatch_started: false })}\n`);
+    return result;
+  }
+  return await commandJson(ddFlowBin, args, {
+    cwd: config.cwd,
+    env: { ...(config.env ?? {}), DD_FLOW_HOME: ddFlowHome, DD_FLOW_RESOURCE_HOME: resourceHome }
+  });
+}
+
+export async function registerDaemonProcess(config, input) {
+  const owner = bindRuntimeOwner(config);
+  const harness = ({ "codex-daemon": "codex-desktop", "codex-provider": "codex-desktop", "zcode-daemon": "zcode-acp", "zcode-provider": "zcode-acp", "grok-daemon": "grok-acp", "grok-provider": "grok-acp", "agy-daemon": "antigravity-cli", "opencode-daemon": "opencode-server", "opencode-provider": "opencode-server", "droid-daemon": "droid-cli" })[input.kind];
+  const scope = owner ? { owner: owner.owner_id, "project-id": owner.project_id, run: owner.run_id, work: owner.work_id, role: owner.role, "state-dir": owner.state_dir, "adapter-executable": owner.adapter_executable, ...(owner.budget ? { "budget-json": JSON.stringify(owner.budget), harness } : {}), ...(owner.project_id && owner.run_id || owner.budget ? { "dispatch-barrier": "run-generation@1" } : {}), operation: `${owner.operation_id}:${input.operation ?? input.kind}` } : {};
+  const options = { ...input, ...scope, id: `PROC-${input.kind}-${randomUUID()}`, "owner-pid": process.pid, "lease-ms": 300_000 };
+  const renewal = createRenewalState();
+  renewal.uncertainSince = performance.now();
+  let result;
+  for (;;) {
+    try { result = await runtimeProcess(config, "register", options, { timeoutMs: renewalRemaining(renewal) }); break; }
+    catch (error) {
+      const unknownSettled = error.details?.effect === "unknown" && error.details?.cleanup_unconfirmed !== true;
+      if ((!maintenanceRetryable(error) && !unknownSettled) || !renewalFailure(renewal)) throw error;
+      // The child has exited before a timeout is retryable. Reconcile the exact
+      // startup identity, not an arbitrary PID or another operation's owner.
+      let status;
+      try { status = await retryBookkeeping(timeoutMs => runtimeProcess(config, "status", {}, { timeoutMs }), renewal); }
+      catch (reconciliationError) {
+        // Retain the failed write's phases without changing the reconciliation
+        // failure's classification or authorizing another startup attempt.
+        reconciliationError.details = { ...reconciliationError.details, registration_error: errorRecord(error) };
+        throw reconciliationError;
+      }
+      const retained = status?.processes?.find(record => record.id === options.id);
+      if (retained) {
+        let metadata; try { metadata = JSON.parse(retained.metadata_json); } catch { /* Invalid binding below. */ }
+        if (retained.kind !== options.kind || retained.owner_id !== options.owner || retained.operation_id !== options.operation || retained.state !== "starting" || metadata?.owner_pid !== process.pid) throw Object.assign(new Error("Startup reconciliation changed managed process binding"), { code: "process_maintenance_receipt_invalid" });
+        result = { process: retained }; break;
+      }
+      if (!Array.isArray(status?.processes)) throw Object.assign(new Error("Startup reconciliation has no registry proof"), { code: "process_maintenance_receipt_invalid" });
+      await new Promise(resolve => setTimeout(resolve, Math.min(renewalDelay(renewal), renewalRemaining(renewal))));
+    }
+  }
+  if (!result) return null;
+  if (result.process?.id !== options.id || !result.process.lease_token) throw Object.assign(new Error("Managed startup receipt belongs to another registration"), { code: "process_maintenance_receipt_invalid" });
+  requireTimelyMaintenance(renewal, "register", result, result.process);
+  return { id: result.process.id, lease_token: result.process.lease_token, kind: input.kind, lease_expires_at: result.process.lease_expires_at };
+}
+
+export async function confirmDaemonProcess(config, record, child) {
+  if (!record) return;
+  if (!child?.pid) {
+    await finishDaemonProcess(config, record, "failed", "daemon_spawn_missing_pid");
+    return;
+  }
+  const renewal = createRenewalState();
+  const result = await retryBookkeeping(timeoutMs => runtimeProcess(config, "confirm", { id: record.id, "lease-token": record.lease_token, pid: child.pid, "process-group-id": child.pid, "owner-pid": record.kind?.endsWith("-daemon") ? child.pid : process.pid, "lease-ms": 300_000 }, { timeoutMs }), renewal);
+  if (!result?.process || result.process.id !== record.id || result.process.lease_token !== record.lease_token) throw Object.assign(new Error("Managed process confirmation binding is invalid"), { code: "process_maintenance_receipt_invalid" });
+  retainReceipt(record, result.process);
+  requireTimelyMaintenance(renewal, "confirm", result, record);
+}
+
+/** Provider bridges use a different process group from their daemon. Publish
+ * the exact binding before initialize can start productive provider traffic. */
+export async function startManagedBridge({ config, state, bridge, kind, publish, log }) {
+  let provider;
+  try {
+    state.pid = process.pid;
+    await publish(state);
+    provider = config.runtime_owner ? await registerDaemonProcess(config, { kind, owner: config.runtime_owner.owner_id, operation: kind }) : null;
+    state.provider_process = provider;
+    state.startup_phase = "owner_registered";
+    await publish(state);
+    bridge.options.onSpawn = async child => {
+      await confirmDaemonProcess(config, provider, child);
+      state.provider_pid = child.pid;
+      state.startup_phase = "provider_spawned";
+      await publish(state);
+      scheduleDaemonProcessHeartbeat(config, provider);
+    };
+    const initialized = await bridge.start();
+    state.startup_phase = "native_initialized";
+    await publish(state);
+    return initialized;
+  }
+  catch (error) {
+    await cleanupFailedStart(config, provider, bridge.child, error);
+    state.shutdown_state = error.details?.cleanup_failed ? "cleanup_failed" : "failed";
+    await retainStartupFailure({ state, publish, log, error });
+    throw error;
+  }
+}
+
+/** A retained cause belongs to this startup generation, never a previous PID. */
+export function assertStartupChild(child, expected, observed, log) {
+  const sameGeneration = observed?.daemon_id === expected.daemon_id && (observed?.generation ?? null) === (expected.generation ?? null)
+    && (observed?.config?.runtime_owner?.owner_id ?? null) === (expected.config?.runtime_owner?.owner_id ?? null)
+    && (observed?.config?.runtime_owner?.operation_id ?? null) === (expected.config?.runtime_owner?.operation_id ?? null);
+  if (sameGeneration && observed?.startup_error) throw reportedError(observed.startup_error, "Native startup failed");
+  if (child.exitCode !== null || child.signalCode !== null) throw Object.assign(new Error("Native daemon exited before readiness"), {
+    code: "daemon_start_failed", retryable: false, details: { daemon_id: expected.daemon_id,
+      pid: child.pid ?? null, exit_code: child.exitCode, signal: child.signalCode, daemon_log: log }
+  });
+}
+
+export async function retainStartupFailure({ state, publish, log, error }) {
+  state.startup_error = JSON.parse(diagnosticJson(errorRecord(error)));
+  try { await publish(state); }
+  catch (publication) {
+    error.details = { ...error.details, startup_publication_error: errorRecord(publication), daemon_log: log };
+    // No credentials/config/argv: the log is only a bounded error record fallback.
+    try { if (log) await appendFile(log, `${diagnosticJson({ daemon_id: state.daemon_id, startup_error: errorRecord(error) })}\n`, { mode: 0o600 }); }
+    catch (logging) { error.details.startup_log_error = errorRecord(logging); }
+  }
+}
+
+export async function closeManagedBridge(config, state, bridge) {
+  await bridge.close();
+  await finishDaemonProcess(config, state.provider_process, "stopped", "daemon_stopped");
+}
+
+export function scheduleDaemonProcessHeartbeat(config, record, { signal } = {}) {
+  if (!record?.id || !record.lease_token) return;
+  const key = leaseKey(config, record);
+  if (leases.has(key)) {
+    if (signal && leases.get(key).signal !== signal) throw Object.assign(new Error("Heartbeat cancellation owner differs from the retained lease"), { code: "process_maintenance_signal_conflict" });
+    return;
+  }
+  const lease = { pending: null, error: null, timer: null, lost: false, closing: false, signal, expiresAt: Date.parse(record.lease_expires_at ?? ""), renewal: createRenewalState({ leaseExpiresAt: record.lease_expires_at }), stateDir: path.resolve(config.runtime_owner?.state_dir ?? config.stateDir ?? config.cwd), config, record };
+  leases.set(key, lease);
+  // Retry no-effect contention inside the same authority episode while native
+  // work is active; scheduling ticks are not ownership evidence.
+  lease.timer = setInterval(() => reconfirmLease(lease, () => {}).catch(() => {}), 60_000);
+  lease.timer.unref();
+  if (signal) {
+    signal.addEventListener("abort", () => clearInterval(lease.timer), { once: true });
+    if (signal.aborted) clearInterval(lease.timer);
+  }
+}
+
+export async function heartbeatDaemonProcess(config, record, { signal } = {}) {
+  if (!record?.id || !record.lease_token) return;
+  scheduleDaemonProcessHeartbeat(config, record, { signal });
+  const lease = leases.get(leaseKey(config, record));
+  if (lease.closing) throw Object.assign(new Error("Managed process is closing"), { code: "process_closing" });
+  if (lease.lost) throw lease.error;
+  if (lease.error && !maintenanceRetryable(lease.error)) throw lease.error;
+  if (!renewalRemaining(lease.renewal)) throw lease.error ?? Object.assign(new Error("Managed lease authority has expired before heartbeat dispatch"), {
+    code: "process_ownership_unknown", details: { process_id: record.id, reason: "lease_authority_expired" }
+  });
+  if (lease.pending) return await lease.pending;
+  lease.pending = (async () => {
+    try {
+      lease.renewal.uncertainSince ??= performance.now();
+      const result = await runtimeProcess(config, "heartbeat", { id: record.id, "lease-token": record.lease_token, "lease-ms": 300_000 }, { timeoutMs: renewalRemaining(lease.renewal), signal: lease.signal });
+      if (result?.ok === false) {
+        lease.lost = true; clearInterval(lease.timer);
+        throw Object.assign(new Error("managed process lease is no longer owned; new productive operations are blocked"), { code: "process_lease_lost", details: { process_id: record.id } });
+      }
+      if (result?.ok !== true || result.process_id !== record.id) throw Object.assign(new Error("Managed renewal response belongs to another process"), { code: "process_maintenance_receipt_invalid", details: { process_id: record.id } });
+      if (!lease.closing) {
+        retainReceipt(record, result);
+        if (!renewalConfirmed(lease.renewal, performance.now(), record.lease_expires_at)) requireTimelyMaintenance(lease.renewal, "heartbeat", result, record);
+        lease.error = null; lease.expiresAt = Date.parse(record.lease_expires_at);
+      }
+    } catch (cause) {
+      if (!lease.closing) renewalFailure(lease.renewal);
+      lease.error = cause.code === "process_lease_lost" ? cause : Object.assign(new Error("cannot confirm managed process ownership"), { code: "process_ownership_unknown", cause, details: { process_id: record.id, cause: errorRecord(cause) } });
+      throw lease.error;
+    }
+  })();
+  try { await lease.pending; } finally { lease.pending = null; }
+}
+
+export async function finishDaemonProcess(config, process, state = "stopped", reason = "daemon_stopped") {
+  if (process?.id && process.lease_token) {
+    const key = leaseKey(config, process), lease = leases.get(key);
+    if (lease) { lease.closing = true; clearInterval(lease.timer); await lease.pending?.catch(() => {}); if (lease.error?.cause?.details?.cleanup_unconfirmed) throw lease.error; }
+    // A predecessor owns its transport fence; finish gets an independent
+    // settlement window only after that flight physically settled.
+    const settlement = createRenewalState(); settlement.uncertainSince = performance.now();
+    const result = await retryBookkeeping(timeoutMs => runtimeProcess(config, "finish", { id: process.id, "lease-token": process.lease_token, state, reason }, { timeoutMs }), settlement, true);
+    if (result?.ok !== true) throw Object.assign(new Error("managed process finish was not accepted"), { code: "process_lease_lost", details: { process_id: process.id } });
+    leases.delete(key);
+  }
+}
+
+function leaseKey(config, record) { return JSON.stringify([path.resolve(config.runtime_owner?.resource_home ?? config.resourceHome), record.id, record.lease_token]); }
+function retainReceipt(record, receipt) {
+  const expiresAt = Date.parse(receipt.lease_expires_at ?? "");
+  if (!Number.isFinite(expiresAt) || expiresAt <= Date.now() + RENEWAL_POLICY.safetyMs || !/^[a-f0-9]{64}$/.test(receipt.registration_sha256 ?? "") || record.registration_sha256 && record.registration_sha256 !== receipt.registration_sha256) throw Object.assign(new Error("Managed lease receipt has no authoritative expiry/binding"), { code: "process_maintenance_receipt_invalid", details: { process_id: record.id } });
+  record.lease_expires_at = receipt.lease_expires_at;
+  record.registration_sha256 = receipt.registration_sha256;
+}
+async function reconfirmLease(lease, assertCurrent) {
+  for (;;) {
+    assertCurrent();
+    if (lease.closing || lease.lost) throw lease.error ?? Object.assign(new Error("Managed process is closing"), { code: "process_closing" });
+    try { await heartbeatDaemonProcess(lease.config, lease.record); }
+    catch (error) {
+      if (!maintenanceRetryable(error) || !renewalRemaining(lease.renewal)) throw error;
+      await new Promise(resolve => setTimeout(resolve, Math.min(renewalDelay(lease.renewal), renewalRemaining(lease.renewal))));
+      continue;
+    }
+    assertCurrent();
+    if (lease.closing || lease.error || !Number.isFinite(lease.expiresAt) || Date.now() >= lease.expiresAt) throw lease.error ?? Object.assign(new Error("Committed lease receipt expired before admission"), { code: "process_ownership_unknown" });
+    return;
+  }
+}
+async function retryBookkeeping(action, renewal = createRenewalState(), repeatFinish = false) {
+  renewal.uncertainSince ??= performance.now();
+  for (;;) {
+    try { return await action(renewalRemaining(renewal)); }
+    catch (error) {
+      // Exact finish is an idempotent terminal-state CAS, not productive IO.
+      // Its old helper may still commit the same finish; all other writes stay fenced.
+      if (!(maintenanceRetryable(error) || repeatFinish && error.code === "process_maintenance_timeout" && error.details?.cleanup_unconfirmed === true) || !renewalFailure(renewal)) throw error;
+      await new Promise(resolve => setTimeout(resolve, Math.min(renewalDelay(renewal), renewalRemaining(renewal))));
+    }
+  }
+}
+
+// A current exact committed ACK establishes authority, not an arbitrary
+// elapsed-time cutoff. Cleanup finish/release remains a non-productive fact.
+function requireTimelyMaintenance(renewal, action, receipt, record) {
+  const expiry = receipt?.process?.lease_expires_at ?? receipt?.lease_expires_at;
+  if (expiry && renewalConfirmed(renewal, performance.now(), expiry)) return;
+  if (!expiry && renewalRemaining(renewal)) return;
+  const safeText = value => typeof value === "string" ? value.slice(0, 128) : undefined;
+  const hash = receipt?.process?.registration_sha256 ?? receipt?.registration_sha256 ?? record?.registration_sha256;
+  throw Object.assign(new Error("Managed ownership confirmation arrived after its uncertainty budget"), {
+    code: "process_ownership_unknown", details: { action, process_id: safeText(record?.id ?? receipt?.process_id),
+      registration_sha256: typeof hash === "string" && /^[a-f0-9]{64}$/.test(hash) ? hash : undefined,
+      lease_expires_at: safeText(receipt?.process?.lease_expires_at ?? receipt?.lease_expires_at ?? record?.lease_expires_at),
+      effect: action === "admission" && receipt?.admitted !== true ? "no_effect" : "committed", native_dispatch_started: false }
+  });
+}
+
+/** Preserve the startup error and retain the lease/resources if cleanup cannot
+ * prove termination. A failed cleanup is not a terminal managed process. */
+export async function cleanupFailedStart(config, record, child, original) {
+  try {
+    await stopProcessGroup(child);
+    const stopped = child?.pid ? { pid: child.pid, pgid: process.platform === "win32" ? null : child.pid,
+      settled: true, physical_exit: "ESRCH", observed_at: new Date().toISOString() } : undefined;
+    await finishDaemonProcess(config, record, "failed", original.code ?? "startup_failed");
+    return stopped;
+  } catch (cleanup) {
+    original.details = { ...original.details, cleanup_failed: errorRecord(cleanup), process_id: record?.id ?? null, pid: child?.pid ?? null };
+  }
+}
+
+/** The provider has its own process group. A failed daemon start must settle
+ * that exact retained process as well as the daemon; PID-only cleanup is unsafe. */
+export async function cleanupFailedDaemonStart(config, daemonRecord, daemonChild, original, readState, daemonId) {
+  const stopped = await cleanupFailedStart(config, daemonRecord, daemonChild, original);
+  if (!daemonRecord) return;
+  try {
+    const state = await readState();
+    const owner = config.runtime_owner;
+    if (!state || state.daemon_id !== daemonId || state.resource_process?.id !== daemonRecord.id || state.resource_process?.lease_token !== daemonRecord.lease_token
+      || (owner && (state.config?.runtime_owner?.owner_id !== owner.owner_id || path.resolve(state.config?.runtime_owner?.state_dir ?? "") !== path.resolve(owner.state_dir)))) {
+      throw Object.assign(new Error("Failed daemon state no longer matches the startup owner"), { code: "daemon_cleanup_identity_mismatch" });
+    }
+    const provider = state.provider_process;
+    if (provider) await runtimeProcess(config, "stop", { id: provider.id, "lease-token": provider.lease_token });
+    // Confirmation can fail before the registry learns the child's PID. The
+    // owned child handle still proves its physical exit, not a productive ACK.
+    if (stopped && !original.details?.cleanup_failed && !original.details?.provider_cleanup_failed && state.pid === stopped.pid && !provider && state.provider_pid == null && owner) {
+      original.details = { ...original.details, startup_child_cleanup: {
+        ...stopped, daemon_id: daemonId, process_id: daemonRecord.id,
+        owner_id: owner.owner_id, project_id: owner.project_id, run_id: owner.run_id,
+        operation_id: owner.operation_id, state_dir: owner.state_dir
+      } };
+    }
+  } catch (cleanup) {
+    original.details = { ...original.details, provider_cleanup_failed: errorRecord(cleanup) };
+  }
+}
+
+const processStops = new WeakMap();
+export async function closeBridgeProcess(bridge, drain, stopOptions = {}) {
+  if (bridge.closePending) return await bridge.closePending;
+  const pending = (async () => {
+    let primary;
+    try {
+      if (!bridge.physicallyClosed && bridge.child) {
+        const child = bridge.child;
+        if (!bridge.stdinEnded && !child.stdin?.destroyed) { bridge.stdinEnded = true; child.stdin?.end(); }
+        if (child.exitCode == null && child.signalCode == null) {
+          await new Promise(resolve => {
+            const done = () => { clearTimeout(timer); child.removeListener("exit", done); resolve(); };
+            const timer = setTimeout(done, 2_000); child.once("exit", done);
+          });
+        }
+        await stopProcessGroup(child, 2_000, stopOptions);
+        bridge.physicallyClosed = true;
+      }
+    } catch (error) { primary = error; }
+    let drainTimer;
+    try { await Promise.race([drain(), new Promise((_, reject) => { drainTimer = setTimeout(() => reject(Object.assign(new Error("bridge drain did not finish"), { code: "bridge_drain_timeout" })), 10_000); })]); }
+    catch (error) { if (primary) { primary.cleanup_error = errorRecord(error); primary.details = { ...primary.details, cleanup_error: primary.cleanup_error }; } else primary = error; }
+    finally { clearTimeout(drainTimer); }
+    if (primary) throw primary;
+  })();
+  bridge.closePending = pending;
+  try { return await pending; }
+  finally { bridge.closePending = null; }
+}
+
+export async function stopProcessGroup(child, graceMs = 1_000, { detached = true, settlementMs = 10_000, started = child?.started ?? null } = {}) {
+  if (!child?.pid) return;
+  if (processStops.has(child)) return await processStops.get(child);
+  const pending = stopOwnedChild(child, graceMs, detached, settlementMs, started);
+  processStops.set(child, pending);
+  try { return await pending; } finally { processStops.delete(child); }
+}
+async function stopOwnedChild(child, graceMs, detached, settlementMs, started) {
+  const target = process.platform === "win32" || !detached ? child.pid : -child.pid;
+  const details = signal => ({ pid: child.pid, pgid: target < 0 ? child.pid : null, signal, platform: process.platform, exit_code: child.exitCode ?? null, signal_code: child.signalCode ?? null, observed_at: new Date().toISOString() });
+  const alive = () => { try { process.kill(target, 0); return true; } catch (error) { if (error.code === "ESRCH") return false; if (error.code === "EPERM") return true; throw error; } };
+  const gone = async ms => {
+    const deadline = performance.now() + ms;
+    while (alive() && performance.now() < deadline) await new Promise(resolve => setTimeout(resolve, 25));
+    return !alive();
+  };
+  const signal = async name => {
+    // A retained raw PID is not a ChildProcess handle. Revalidate its birth at
+    // every signal, including escalation after the TERM grace period.
+    if (started !== null) {
+      if (typeof started !== 'string' || !started) throw Object.assign(new Error('process birth identity is invalid'), { code: 'process_identity_unproven' });
+      const observed = (await processSnapshot()).find(item => item.pid === child.pid);
+      if (!observed || observed.started !== started || observed.zombie) return false;
+    }
+    try { process.kill(target, name); return true; }
+    catch (error) {
+      if (error.code === "ESRCH") return false;
+      error.details = { ...error.details, ...details(name), syscall: error.syscall ?? "kill" };
+      // Permission denial is not permission to escalate. Observe only the retained target.
+      let absent = false;
+      try { absent = await gone(Math.min(250, Math.max(25, graceMs))); }
+      catch (observation) { error.details.observation_error = errorRecord(observation); }
+      if (absent) {
+        process.stderr.write(`${JSON.stringify({ cleanup_reobserved: errorRecord(error), physical_exit: "ESRCH" })}\n`);
+        return false;
+      }
+      throw error;
+    }
+  };
+  if (!alive()) return;
+  const leaderExited = () => child.exitCode != null || child.signalCode != null;
+  const observeLeaderless = async () => {
+    // Signal grace is not a drain deadline. Helpers can retire naturally after
+    // their leader exits; observing them does not authorize another signal.
+    if (await gone(Math.max(1_000, graceMs, settlementMs))) return;
+    throw Object.assign(new Error("process group remains after its leader exited; ownership cannot be proven from this child handle"), { code: "process_group_ownership_unknown", details: { ...details(null), settlement_ms: Math.max(1_000, graceMs, settlementMs) } });
+  };
+  if (leaderExited()) return await observeLeaderless();
+  if (!await signal("SIGTERM")) return;
+  if (await gone(graceMs)) return;
+  if (leaderExited()) return await observeLeaderless();
+  if (!await signal("SIGKILL")) return;
+  if (!await gone(Math.max(1_000, graceMs))) throw Object.assign(new Error(`process group ${child.pid} did not stop`), { code: "process_group_stop_incomplete", details: details("SIGKILL") });
+}
+
+/** A daemon acknowledgement is not a completed stop until its endpoint closes. */
+export async function confirmDaemonStopped(request, socket, timeoutMs = 30_000, { readState } = {}) {
+  const expected = await readState?.();
+  const identity = state => state?.daemon_id ?? state?.config?.daemonId;
+  if (readState && (!identity(expected) || !Number.isSafeInteger(expected?.pid) || expected.pid < 1)) throw Object.assign(new Error('daemon stop has no retained incarnation/PID proof'), { code: 'daemon_cleanup_identity_mismatch' });
+  const failure = async observer => {
+    const state = await readState?.();
+    if (expected && (identity(state) !== identity(expected) || state?.pid !== expected.pid)) throw Object.assign(new Error("daemon incarnation changed during stop"), { code: "daemon_identity_mismatch" });
+    const primary = state?.shutdown_state === "cleanup_failed" ? state.cleanup_error : null;
+    if (primary) throw Object.assign(new Error(primary.message), primary, { details: { ...primary.details, socket, phase: "confirm_daemon_stop", observer_error: errorRecord(observer) } });
+    throw observer;
+  };
+  let result, observationError;
+  try { result = await request(); }
+  catch (error) {
+    if (!expected || !["daemon_timeout", "daemon_not_running", "daemon_connection_closed", "operation_observation_lost"].includes(error.code)) return await failure(error);
+    observationError = error;
+  }
+  if (!observationError && result?.stopped !== true) return await failure(Object.assign(new Error("stop acknowledgement does not prove shutdown"), { code: "daemon_stop_incomplete" }));
+  // The RPC wait and physical retirement are separate phases. A lost ACK
+  // consumes the former, not the existing confirmation budget.
+  const deadline = performance.now() + timeoutMs;
+  for (;;) {
+    const state = await readState?.();
+    if (expected && (identity(state) !== identity(expected) || state?.pid !== expected.pid)) return await failure(Object.assign(new Error("daemon incarnation changed"), { code: "daemon_identity_mismatch" }));
+    if (state?.shutdown_state === "cleanup_failed") return await failure(observationError ?? Object.assign(new Error("daemon cleanup failed"), { code: "daemon_stop_incomplete" }));
+    let endpointPresent = true;
+    try { await lstat(socket); } catch (error) { if (error.code === "ENOENT") endpointPresent = false; else throw error; }
+    let daemonPresent = false;
+    if (expected?.pid) { try { process.kill(expected.pid, 0); daemonPresent = true; } catch (error) { if (error.code !== "ESRCH") daemonPresent = true; } }
+    if (!endpointPresent && !daemonPresent) {
+      if (observationError) {
+        if (state?.shutdown_state === "clean" && state.active_tree === false && !state.cleanup_error) return { stopped: true, clean: true };
+        return await failure(observationError);
+      }
+      if (readState && result.clean === true && (state?.shutdown_state !== 'clean' || state.active_tree !== false)) return await failure(Object.assign(new Error('daemon clean ACK conflicts with its retained state'), { code: 'daemon_stop_incomplete' }));
+      return result;
+    }
+    if (performance.now() >= deadline) return await failure(Object.assign(new Error("daemon stop could not confirm endpoint retirement and owned PID exit"), { code: "daemon_stop_incomplete", details: { socket, daemon_pid: expected?.pid ?? null, endpoint_present: endpointPresent, daemon_present: daemonPresent } }));
+    await new Promise(resolve => setTimeout(resolve, 25));
+  }
+}