@deksden-com/dd-flow-cli 0.9.0-beta.123 → 0.9.0-beta.127

package/dist/harness-runtime/lib/managed-daemon.mjs

@@ -5,8 +5,13 @@ import { commandJson } from "./process-json.mjs";
 import { errorRecord } from "./operation-errors.mjs";
 import { writeJsonAtomic } from "./runner-events.mjs";
 import { runBoundedCommand } from "./native-hook-command.mjs";
+import { createRenewalState, maintenanceRetryable, renewalConfirmed, renewalDelay, renewalFailure, renewalRemaining, RENEWAL_POLICY } from "./lease-renewal.mjs";
 
 const leases = new Map();
+// A rejected Promise is not physical exit. Do not start another writer for
+// this exact binding after an unconfirmed maintenance subtree cleanup.
+const maintenancePoison = new Map();
+const maintenanceFlights = new Map();
 
 /** Freeze resource ownership independently of whether Subject hooks are enabled. */
 export function bindRuntimeOwner(config, env = process.env) {
@@ -38,6 +43,7 @@ export function bindRuntimeOwner(config, env = process.env) {
 
 /** Reject rebinding before provider setup, including the already-running path. */
 export async function prepareRuntimeOwner(config, stateDir, env = process.env) {
+  config.stateDir = path.resolve(stateDir);
   const owner = bindRuntimeOwner(config, env);
   if (owner && path.resolve(owner.state_dir) !== path.resolve(stateDir)) {
     throw Object.assign(new Error("runtime owner belongs to another state directory"), { code: "runtime_owner_mismatch" });
@@ -52,27 +58,44 @@ export async function prepareRuntimeOwner(config, stateDir, env = process.env) {
   return owner;
 }
 
-export async function assertDaemonOwnership(stateDir, dispatch = {}, assertCurrent = () => {}) {
-  for (const lease of leases.values()) if (!lease.stateDir && lease.error) throw lease.error;
+export async function assertDaemonOwnership(stateDir, dispatch = {}, assertCurrent = () => {}, progress = async () => {}, waiting = async () => {}) {
+  const gate = async () => {
+    assertCurrent();
+    for (const lease of leases.values()) {
+      if (lease.stateDir !== path.resolve(stateDir)) continue;
+      await reconfirmLease(lease, assertCurrent);
+      assertCurrent();
+    }
+  };
+  await gate();
+  await progress('ownership_confirmed');
   const state = JSON.parse(await readFile(path.join(stateDir, "daemon.json"), "utf8"));
   const owner = bindRuntimeOwner({ runtime_owner: state.config?.runtime_owner ?? null }, {});
   if (!owner) return;
   const record = state.resource_process;
   if (owner.state_dir !== stateDir || !record?.id || !record.lease_token) throw Object.assign(new Error("owned daemon has no confirmed resource binding"), { code: "process_ownership_unknown" });
-  for (const lease of leases.values()) {
-    if (lease.stateDir !== stateDir) continue;
-    if (Date.now() >= lease.expiresAt) throw Object.assign(new Error("managed process renewal is overdue"), { code: "process_ownership_unknown", details: { process_id: lease.record.id, cause: lease.error ? errorRecord(lease.error) : null } });
-    // A failed renewal is not a lost lease. Reconfirm before productive work;
-    // never dispatch on the old lease merely because it has time remaining.
-    if (lease.error && !lease.lost) await heartbeatDaemonProcess(lease.config, lease.record);
-    if (lease.error) throw lease.error;
-  }
+  scheduleDaemonProcessHeartbeat(state.config, record);
+  const admissionRenewal = createRenewalState();
   for (;;) {
+    await gate();
+    admissionRenewal.uncertainSince ??= performance.now();
+    let result;
+    try { result = await runtimeProcess(state.config, "admission", { id: record.id, "lease-token": record.lease_token, owner: owner.owner_id, "state-dir": stateDir, "daemon-id": state.daemon_id ?? state.config?.daemonId, ...dispatch }, { timeoutMs: renewalRemaining(admissionRenewal) }); }
+    catch (error) {
+      if (!maintenanceRetryable(error) || !renewalFailure(admissionRenewal)) throw error;
+      await new Promise(resolve => setTimeout(resolve, Math.min(renewalDelay(admissionRenewal), renewalRemaining(admissionRenewal))));
+      continue;
+    }
     assertCurrent();
-    const result = await runtimeProcess(state.config, "admission", { id: record.id, "lease-token": record.lease_token, owner: owner.owner_id, "state-dir": stateDir, "daemon-id": state.daemon_id ?? state.config?.daemonId, ...dispatch });
-    assertCurrent();
-    if (result?.ok === true && result.admitted === true && result.process_id === record.id) return;
-    if (result?.ok !== true || result.reason !== "provider_capacity" || result.process_id !== record.id) throw Object.assign(new Error("productive runtime admission was not confirmed"), { code: "process_ownership_unknown" });
+    if (result?.ok !== true || result.process_id !== record.id || result.admitted !== true && result.reason !== "provider_capacity") throw Object.assign(new Error("productive runtime admission was not confirmed"), { code: "process_ownership_unknown" });
+    requireTimelyMaintenance(admissionRenewal, "admission", result, record);
+    // This bound ACK ends admission uncertainty. The post-ACK lease checks
+    // have their own episodes; successful renewal must not exhaust an already
+    // confirmed admission while the provider/daemon still retain ownership.
+    if (!renewalConfirmed(admissionRenewal)) requireTimelyMaintenance(admissionRenewal, "admission", result, record);
+    await gate();
+    await waiting({ ...result, lease_expires_at: result.lease_expires_at ?? record.lease_expires_at });
+    if (result.admitted === true) { await progress('admission_confirmed'); return; }
     const predecessor = result.blocking_turns?.find(turn => turn?.process_id === record.id
       && turn?.session_id === dispatch["native-session-id"]
       && turn?.operation_id !== dispatch["operation-id"]);
@@ -153,20 +176,92 @@ async function budgetState(stateDir) {
 
 async function releaseBudget(state, options) {
   const record = state.resource_process, owner = state.config.runtime_owner;
-  const receipt = await runtimeProcess(state.config, "release-turn", { id: record.id, "lease-token": record.lease_token, owner: owner.owner_id, "daemon-id": state.daemon_id ?? state.config?.daemonId, ...options });
+  const receipt = await retryBookkeeping(timeoutMs => runtimeProcess(state.config, "release-turn", { id: record.id, "lease-token": record.lease_token, owner: owner.owner_id, "daemon-id": state.daemon_id ?? state.config?.daemonId, ...options }, { timeoutMs }));
   if (receipt?.ok !== true) throw Object.assign(new Error("runtime budget release was not confirmed"), { code: "runtime_budget_release_unconfirmed" });
 }
 
-export async function runtimeProcess(config, action, options = {}) {
+export async function runtimeProcess(config, action, options = {}, { timeoutMs = RENEWAL_POLICY.budgetMs, signal } = {}) {
+  const resourceHome = config.runtime_owner?.resource_home ?? config.resourceHome;
+  if (action === "status" || !options.id || !resourceHome) return executeRuntimeProcess(config, action, options, { timeoutMs, signal });
+  const key = JSON.stringify([path.resolve(resourceHome), options.id, options["lease-token"] ?? null]);
+  const validBudget = Number.isFinite(timeoutMs) && timeoutMs > 0;
+  const deadline = performance.now() + (validBudget ? timeoutMs : 0);
+  const prior = maintenanceFlights.get(key);
+  let started = false;
+  let settle;
+  const flight = new Promise(resolve => { settle = resolve; });
+  maintenanceFlights.set(key, flight);
+  try {
+    if (!validBudget) throw Object.assign(new Error("Managed maintenance budget is invalid or exhausted before dispatch"), { code: "process_maintenance_timeout", details: { effect: "no_effect", native_dispatch_started: false } });
+    if (prior) {
+      let timer;
+      try {
+        await Promise.race([prior, new Promise((_, reject) => {
+          const expire = () => {
+            const remaining = deadline - performance.now();
+            // A timer wakeup is not proof that its monotonic deadline passed.
+            if (remaining > 0) { timer = setTimeout(expire, Math.min(2_147_483_647, Math.ceil(remaining))); return; }
+            reject(Object.assign(new Error("managed maintenance queue deadline expired"), { code: "process_maintenance_timeout", details: { effect: "no_effect" } }));
+          };
+          expire();
+        })]);
+      } finally { clearTimeout(timer); }
+    }
+    const remaining = deadline - performance.now();
+    if (remaining <= 0) throw Object.assign(new Error("managed maintenance queue deadline expired"), { code: "process_maintenance_timeout", details: { effect: "no_effect" } });
+    started = true;
+    return await executeRuntimeProcess(config, action, options, { timeoutMs: remaining, signal });
+  } catch (error) {
+    const lease = leases.get(key);
+    const text = value => typeof value === "string" ? value.slice(0, 128) : undefined;
+    const request = { phase: started ? "request" : "queue", action, process_id: text(options.id), operation_id: text(options["operation-id"] ?? config.runtime_owner?.operation_id), role: text(lease?.record.kind ?? config.runtime_owner?.role), attempt_kind: prior ? "join" : "fresh", attempt: (lease?.renewal.attempt ?? 0) + 1, remaining_ms: Math.max(0, Math.round(deadline - performance.now())), native_dispatch_started: false, maintenance_effect: started ? "unknown" : "no_effect" };
+    // Parent queue/import failures precede any child trace. Keep only bounded,
+    // credential-free context and the existing child phase allowlist.
+    const childPhases = Array.isArray(error.details?.hook_diagnostics) ? error.details.hook_diagnostics : [];
+    error.details = { ...error.details, hook_diagnostics: [request, ...childPhases.slice(-15)] };
+    throw error;
+  } finally {
+    // Even a waiter whose deadline expires must retain its predecessor's
+    // physical fence until that predecessor settles.
+    const finish = () => {
+      settle();
+      if (maintenanceFlights.get(key) === flight) maintenanceFlights.delete(key);
+    };
+    if (prior) void prior.then(finish); else finish();
+  }
+}
+
+async function executeRuntimeProcess(config, action, options = {}, { timeoutMs, signal }) {
   const owner = config.runtime_owner;
   const resourceHome = owner?.resource_home ?? config.resourceHome;
   const ddFlowHome = owner?.dd_flow_home ?? config.ddFlowHome;
   const ddFlowBin = owner?.dd_flow_bin ?? config.ddFlowBin;
   if (!resourceHome || !ddFlowHome || !ddFlowBin) return null;
+  const poisonKey = token => JSON.stringify([path.resolve(resourceHome), options.id, token ?? null]);
+  if (action !== "status" && options.id) {
+    const poison = maintenancePoison.get(poisonKey(options["lease-token"])) ?? maintenancePoison.get(poisonKey(null));
+    if (poison) throw poison;
+  }
   const args = ["runtime", "process", action];
   for (const [key, value] of Object.entries(options)) if (value !== null && value !== undefined) args.push(`--${key}`, String(value));
-  if (action === "heartbeat") return JSON.parse(await runBoundedCommand({ bin: ddFlowBin, args: [...args, "--json"], cwd: config.cwd,
-    env: { ...(config.env ?? {}), DD_FLOW_HOME: ddFlowHome, DD_FLOW_RESOURCE_HOME: resourceHome }, timeoutMs: 15_000, timeoutCode: "process_heartbeat_timeout" }));
+  if (["register", "confirm", "heartbeat", "admission", "check-admission", "finish", "release-turn", "status"].includes(action)) {
+    const maintenanceStarted = performance.now();
+    process.stderr.write(`${JSON.stringify({ kind: "process_maintenance_phase", phase: "request", observed_at: new Date().toISOString(), operation_id: options['operation-id'] ?? config.runtime_owner?.operation_id ?? null, action, process_id: options.id ?? null, remaining_ms: Math.round(timeoutMs), native_dispatch_started: false })}\n`);
+    let output;
+    try { output = await runBoundedCommand({ bin: ddFlowBin, args: [...args, "--json"], cwd: config.cwd,
+      env: { ...(config.env ?? {}), DD_FLOW_HOME: ddFlowHome, DD_FLOW_RESOURCE_HOME: resourceHome }, timeoutMs,
+      timeoutCode: action === "heartbeat" ? "process_heartbeat_timeout" : "process_maintenance_timeout", inheritDeadline: false, cleanupWithinBudget: true,
+      maintenanceProgress: { action, processId: options.id ?? null, inactivityMs: RENEWAL_POLICY.attemptMs }, signal }); }
+    catch (error) {
+      if (action !== "status" && options.id && error.details?.cleanup_unconfirmed) maintenancePoison.set(poisonKey(options["lease-token"]), error);
+      throw error;
+    }
+    let result;
+    try { result = JSON.parse(output); } catch { throw Object.assign(new Error("Invalid managed process maintenance response"), { code: "process_maintenance_receipt_invalid", details: { action } }); }
+    if (result?.ok === false && result.error?.code) throw Object.assign(new Error(result.error.message ?? "Managed process maintenance failed"), { code: result.error.code, details: result.error.details });
+    process.stderr.write(`${JSON.stringify({ kind: "process_maintenance_phase", phase: "ack", observed_at: new Date().toISOString(), duration_ms: Math.round(performance.now() - maintenanceStarted), operation_id: options['operation-id'] ?? config.runtime_owner?.operation_id ?? null, action, process_id: options.id ?? null, lease_expires_at: result.lease_expires_at ?? result.process?.lease_expires_at ?? null, native_dispatch_started: false })}\n`);
+    return result;
+  }
   return await commandJson(ddFlowBin, args, {
     cwd: config.cwd,
     env: { ...(config.env ?? {}), DD_FLOW_HOME: ddFlowHome, DD_FLOW_RESOURCE_HOME: resourceHome }
@@ -177,8 +272,38 @@ export async function registerDaemonProcess(config, input) {
   const owner = bindRuntimeOwner(config);
   const harness = ({ "codex-daemon": "codex-desktop", "codex-provider": "codex-desktop", "zcode-daemon": "zcode-acp", "zcode-provider": "zcode-acp", "grok-daemon": "grok-acp", "grok-provider": "grok-acp", "agy-daemon": "antigravity-cli", "opencode-daemon": "opencode-server", "droid-daemon": "droid-cli" })[input.kind];
   const scope = owner ? { owner: owner.owner_id, "project-id": owner.project_id, run: owner.run_id, work: owner.work_id, role: owner.role, "state-dir": owner.state_dir, "adapter-executable": owner.adapter_executable, ...(owner.budget ? { "budget-json": JSON.stringify(owner.budget), harness } : {}), ...(owner.project_id && owner.run_id || owner.budget ? { "dispatch-barrier": "run-generation@1" } : {}), operation: `${owner.operation_id}:${input.operation ?? input.kind}` } : {};
-  const result = await runtimeProcess(config, "register", { ...input, ...scope, "owner-pid": process.pid, "lease-ms": 300_000 });
-  return result?.process ? { id: result.process.id, lease_token: result.process.lease_token, kind: input.kind } : null;
+  const options = { ...input, ...scope, id: `PROC-${input.kind}-${randomUUID()}`, "owner-pid": process.pid, "lease-ms": 300_000 };
+  const renewal = createRenewalState();
+  renewal.uncertainSince = performance.now();
+  let result;
+  for (;;) {
+    try { result = await runtimeProcess(config, "register", options, { timeoutMs: renewalRemaining(renewal) }); break; }
+    catch (error) {
+      if (!maintenanceRetryable(error) || !renewalFailure(renewal)) throw error;
+      // The child has exited before a timeout is retryable. Reconcile the exact
+      // startup identity, not an arbitrary PID or another operation's owner.
+      let status;
+      try { status = await retryBookkeeping(timeoutMs => runtimeProcess(config, "status", {}, { timeoutMs }), renewal); }
+      catch (reconciliationError) {
+        // Retain the failed write's phases without changing the reconciliation
+        // failure's classification or authorizing another startup attempt.
+        reconciliationError.details = { ...reconciliationError.details, registration_error: errorRecord(error) };
+        throw reconciliationError;
+      }
+      const retained = status?.processes?.find(record => record.id === options.id);
+      if (retained) {
+        let metadata; try { metadata = JSON.parse(retained.metadata_json); } catch { /* Invalid binding below. */ }
+        if (retained.kind !== options.kind || retained.owner_id !== options.owner || retained.operation_id !== options.operation || retained.state !== "starting" || metadata?.owner_pid !== process.pid) throw Object.assign(new Error("Startup reconciliation changed managed process binding"), { code: "process_maintenance_receipt_invalid" });
+        result = { process: retained }; break;
+      }
+      if (!Array.isArray(status?.processes)) throw Object.assign(new Error("Startup reconciliation has no registry proof"), { code: "process_maintenance_receipt_invalid" });
+      await new Promise(resolve => setTimeout(resolve, Math.min(renewalDelay(renewal), renewalRemaining(renewal))));
+    }
+  }
+  if (!result) return null;
+  if (result.process?.id !== options.id || !result.process.lease_token) throw Object.assign(new Error("Managed startup receipt belongs to another registration"), { code: "process_maintenance_receipt_invalid" });
+  requireTimelyMaintenance(renewal, "register", result, result.process);
+  return { id: result.process.id, lease_token: result.process.lease_token, kind: input.kind, lease_expires_at: result.process.lease_expires_at };
 }
 
 export async function confirmDaemonProcess(config, record, child) {
@@ -187,7 +312,11 @@ export async function confirmDaemonProcess(config, record, child) {
     await finishDaemonProcess(config, record, "failed", "daemon_spawn_missing_pid");
     return;
   }
-  await runtimeProcess(config, "confirm", { id: record.id, "lease-token": record.lease_token, pid: child.pid, "process-group-id": child.pid, "owner-pid": record.kind?.endsWith("-daemon") ? child.pid : process.pid, "lease-ms": 300_000 });
+  const renewal = createRenewalState();
+  const result = await retryBookkeeping(timeoutMs => runtimeProcess(config, "confirm", { id: record.id, "lease-token": record.lease_token, pid: child.pid, "process-group-id": child.pid, "owner-pid": record.kind?.endsWith("-daemon") ? child.pid : process.pid, "lease-ms": 300_000 }, { timeoutMs }), renewal);
+  if (!result?.process || result.process.id !== record.id || result.process.lease_token !== record.lease_token) throw Object.assign(new Error("Managed process confirmation binding is invalid"), { code: "process_maintenance_receipt_invalid" });
+  retainReceipt(record, result.process);
+  requireTimelyMaintenance(renewal, "confirm", result, record);
 }
 
 /** Provider bridges use a different process group from their daemon. Publish
@@ -217,29 +346,40 @@ export async function closeManagedBridge(config, state, bridge) {
 
 export function scheduleDaemonProcessHeartbeat(config, record) {
   if (!record?.id || !record.lease_token) return;
-  if (leases.has(record.id)) return;
-  const lease = { pending: null, error: null, timer: null, lost: false, expiresAt: Date.now() + 300_000, stateDir: config.runtime_owner?.state_dir, config, record };
-  leases.set(record.id, lease);
-  lease.timer = setInterval(() => { void heartbeatDaemonProcess(config, record).catch(() => {}); }, 60_000);
+  const key = leaseKey(config, record);
+  if (leases.has(key)) return;
+  const lease = { pending: null, error: null, timer: null, lost: false, closing: false, expiresAt: Date.parse(record.lease_expires_at ?? ""), renewal: createRenewalState(), stateDir: path.resolve(config.runtime_owner?.state_dir ?? config.stateDir ?? config.cwd), config, record };
+  leases.set(key, lease);
+  // Retry inside the same uncertainty episode while native work is active.
+  // Waiting for the next 60s tick would exhaust the 30s budget after one BUSY.
+  lease.timer = setInterval(() => reconfirmLease(lease, () => {}).catch(() => {}), 60_000);
   lease.timer.unref();
 }
 
 export async function heartbeatDaemonProcess(config, record) {
   if (!record?.id || !record.lease_token) return;
   scheduleDaemonProcessHeartbeat(config, record);
-  const lease = leases.get(record.id);
+  const lease = leases.get(leaseKey(config, record));
+  if (lease.closing) throw Object.assign(new Error("Managed process is closing"), { code: "process_closing" });
   if (lease.lost) throw lease.error;
+  if (lease.error && !maintenanceRetryable(lease.error)) throw lease.error;
+  if (!renewalRemaining(lease.renewal)) throw lease.error;
   if (lease.pending) return await lease.pending;
   lease.pending = (async () => {
     try {
-      const result = await runtimeProcess(config, "heartbeat", { id: record.id, "lease-token": record.lease_token, "lease-ms": 300_000 });
-      if (result?.ok !== true) {
+      lease.renewal.uncertainSince ??= performance.now();
+      const result = await runtimeProcess(config, "heartbeat", { id: record.id, "lease-token": record.lease_token, "lease-ms": 300_000 }, { timeoutMs: renewalRemaining(lease.renewal) });
+      if (result?.ok === false) {
         lease.lost = true; clearInterval(lease.timer);
         throw Object.assign(new Error("managed process lease is no longer owned; new productive operations are blocked"), { code: "process_lease_lost", details: { process_id: record.id } });
       }
-      lease.error = null;
-      lease.expiresAt = Date.now() + 300_000;
+      if (result?.ok !== true || result.process_id !== record.id) throw Object.assign(new Error("Managed renewal response belongs to another process"), { code: "process_maintenance_receipt_invalid", details: { process_id: record.id } });
+      if (!lease.closing) {
+        if (!renewalConfirmed(lease.renewal)) requireTimelyMaintenance(lease.renewal, "heartbeat", result, record);
+        retainReceipt(record, result); lease.error = null; lease.expiresAt = Date.parse(record.lease_expires_at);
+      }
     } catch (cause) {
+      if (!lease.closing) renewalFailure(lease.renewal);
       lease.error = cause.code === "process_lease_lost" ? cause : Object.assign(new Error("cannot confirm managed process ownership"), { code: "process_ownership_unknown", cause, details: { process_id: record.id, cause: errorRecord(cause) } });
       throw lease.error;
     }
@@ -249,13 +389,59 @@ export async function heartbeatDaemonProcess(config, record) {
 
 export async function finishDaemonProcess(config, process, state = "stopped", reason = "daemon_stopped") {
   if (process?.id && process.lease_token) {
-    const lease = leases.get(process.id); if (lease) clearInterval(lease.timer);
-    const result = await runtimeProcess(config, "finish", { id: process.id, "lease-token": process.lease_token, state, reason });
+    const settlement = createRenewalState(); settlement.uncertainSince = performance.now();
+    const key = leaseKey(config, process), lease = leases.get(key);
+    if (lease) { lease.closing = true; clearInterval(lease.timer); await lease.pending?.catch(() => {}); if (lease.error?.cause?.details?.cleanup_unconfirmed) throw lease.error; }
+    const result = await retryBookkeeping(timeoutMs => runtimeProcess(config, "finish", { id: process.id, "lease-token": process.lease_token, state, reason }, { timeoutMs }), settlement);
     if (result?.ok !== true) throw Object.assign(new Error("managed process finish was not accepted"), { code: "process_lease_lost", details: { process_id: process.id } });
-    leases.delete(process.id);
+    leases.delete(key);
   }
 }
 
+function leaseKey(config, record) { return JSON.stringify([path.resolve(config.runtime_owner?.resource_home ?? config.resourceHome), record.id, record.lease_token]); }
+function retainReceipt(record, receipt) {
+  const expiresAt = Date.parse(receipt.lease_expires_at ?? "");
+  if (!Number.isFinite(expiresAt) || !/^[a-f0-9]{64}$/.test(receipt.registration_sha256 ?? "") || record.registration_sha256 && record.registration_sha256 !== receipt.registration_sha256) throw Object.assign(new Error("Managed lease receipt has no authoritative expiry/binding"), { code: "process_maintenance_receipt_invalid", details: { process_id: record.id } });
+  record.lease_expires_at = receipt.lease_expires_at;
+  record.registration_sha256 = receipt.registration_sha256;
+}
+async function reconfirmLease(lease, assertCurrent) {
+  for (;;) {
+    assertCurrent();
+    if (lease.closing || lease.lost) throw lease.error ?? Object.assign(new Error("Managed process is closing"), { code: "process_closing" });
+    try { await heartbeatDaemonProcess(lease.config, lease.record); }
+    catch (error) {
+      if (!maintenanceRetryable(error) || !renewalRemaining(lease.renewal)) throw error;
+      await new Promise(resolve => setTimeout(resolve, Math.min(renewalDelay(lease.renewal), renewalRemaining(lease.renewal))));
+      continue;
+    }
+    assertCurrent();
+    if (lease.closing || lease.error || !Number.isFinite(lease.expiresAt) || Date.now() >= lease.expiresAt) throw lease.error ?? Object.assign(new Error("Committed lease receipt expired before admission"), { code: "process_ownership_unknown" });
+    return;
+  }
+}
+async function retryBookkeeping(action, renewal = createRenewalState()) {
+  renewal.uncertainSince ??= performance.now();
+  for (;;) {
+    try { return await action(renewalRemaining(renewal)); }
+    catch (error) { if (!maintenanceRetryable(error) || !renewalFailure(renewal)) throw error; await new Promise(resolve => setTimeout(resolve, Math.min(renewalDelay(renewal), renewalRemaining(renewal)))); }
+  }
+}
+
+// A committed ACK is evidence, but cannot revive an expired authority episode.
+// Cleanup-only finish/release receipts remain facts, not productive grants.
+function requireTimelyMaintenance(renewal, action, receipt, record) {
+  if (renewalRemaining(renewal)) return;
+  const safeText = value => typeof value === "string" ? value.slice(0, 128) : undefined;
+  const hash = receipt?.process?.registration_sha256 ?? receipt?.registration_sha256 ?? record?.registration_sha256;
+  throw Object.assign(new Error("Managed ownership confirmation arrived after its uncertainty budget"), {
+    code: "process_ownership_unknown", details: { action, process_id: safeText(record?.id ?? receipt?.process_id),
+      registration_sha256: typeof hash === "string" && /^[a-f0-9]{64}$/.test(hash) ? hash : undefined,
+      lease_expires_at: safeText(receipt?.process?.lease_expires_at ?? receipt?.lease_expires_at ?? record?.lease_expires_at),
+      effect: action === "admission" && receipt?.admitted !== true ? "no_effect" : "committed", native_dispatch_started: false }
+  });
+}
+
 /** Preserve the startup error and retain the lease/resources if cleanup cannot
  * prove termination. A failed cleanup is not a terminal managed process. */
 export async function cleanupFailedStart(config, record, child, original) {
@@ -286,43 +472,113 @@ export async function cleanupFailedDaemonStart(config, daemonRecord, daemonChild
   }
 }
 
-export async function stopProcessGroup(child, graceMs = 1_000) {
+const processStops = new WeakMap();
+export async function closeBridgeProcess(bridge, drain, stopOptions = {}) {
+  if (bridge.closePending) return await bridge.closePending;
+  const pending = (async () => {
+    let primary;
+    try {
+      if (!bridge.physicallyClosed && bridge.child) {
+        const child = bridge.child;
+        if (!bridge.stdinEnded && !child.stdin?.destroyed) { bridge.stdinEnded = true; child.stdin?.end(); }
+        if (child.exitCode == null && child.signalCode == null) {
+          await new Promise(resolve => {
+            const done = () => { clearTimeout(timer); child.removeListener("exit", done); resolve(); };
+            const timer = setTimeout(done, 2_000); child.once("exit", done);
+          });
+        }
+        await stopProcessGroup(child, 2_000, stopOptions);
+        bridge.physicallyClosed = true;
+      }
+    } catch (error) { primary = error; }
+    let drainTimer;
+    try { await Promise.race([drain(), new Promise((_, reject) => { drainTimer = setTimeout(() => reject(Object.assign(new Error("bridge drain did not finish"), { code: "bridge_drain_timeout" })), 10_000); })]); }
+    catch (error) { if (primary) { primary.cleanup_error = errorRecord(error); primary.details = { ...primary.details, cleanup_error: primary.cleanup_error }; } else primary = error; }
+    finally { clearTimeout(drainTimer); }
+    if (primary) throw primary;
+  })();
+  bridge.closePending = pending;
+  try { return await pending; }
+  finally { bridge.closePending = null; }
+}
+
+export async function stopProcessGroup(child, graceMs = 1_000, { detached = true, settlementMs = 10_000 } = {}) {
   if (!child?.pid) return;
-  // Harness bridges are detached on Unix, so their own process group is the
-  // smallest reliable unit of cleanup. Windows has no equivalent negative-PID
-  // signal operation; there the direct child is the safe best effort.
-  const target = process.platform === "win32" ? child.pid : -child.pid;
+  if (processStops.has(child)) return await processStops.get(child);
+  const pending = stopOwnedChild(child, graceMs, detached, settlementMs);
+  processStops.set(child, pending);
+  try { return await pending; } finally { processStops.delete(child); }
+}
+async function stopOwnedChild(child, graceMs, detached, settlementMs) {
+  const target = process.platform === "win32" || !detached ? child.pid : -child.pid;
+  const details = signal => ({ pid: child.pid, pgid: target < 0 ? child.pid : null, signal, platform: process.platform, exit_code: child.exitCode ?? null, signal_code: child.signalCode ?? null, observed_at: new Date().toISOString() });
   const alive = () => { try { process.kill(target, 0); return true; } catch (error) { if (error.code === "ESRCH") return false; if (error.code === "EPERM") return true; throw error; } };
-  const signal = name => { try { process.kill(target, name); return true; } catch (error) { if (error.code === "ESRCH") return false; throw error; } };
-  if (!alive()) return;
-  if ((child.exitCode !== undefined && child.exitCode !== null) || (child.signalCode !== undefined && child.signalCode !== null)) {
-    // The leader can exit a few ticks before its helpers. Observe their normal
-    // shutdown before failing ownership; never signal a leaderless group.
-    const deadline = Date.now() + Math.max(1_000, graceMs);
-    while (alive() && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 25));
-    if (!alive()) return;
-    throw Object.assign(new Error("process group remains after its leader exited; ownership cannot be proven from this child handle"), { code: "process_group_ownership_unknown", details: { pid: child.pid } });
-  }
-  if (!signal("SIGTERM")) return; // The group may exit after the liveness read.
-  await new Promise((resolve) => setTimeout(resolve, graceMs));
+  const gone = async ms => {
+    const deadline = performance.now() + ms;
+    while (alive() && performance.now() < deadline) await new Promise(resolve => setTimeout(resolve, 25));
+    return !alive();
+  };
+  const signal = async name => {
+    try { process.kill(target, name); return true; }
+    catch (error) {
+      if (error.code === "ESRCH") return false;
+      error.details = { ...error.details, ...details(name), syscall: error.syscall ?? "kill" };
+      // Permission denial is not permission to escalate. Observe only the retained target.
+      let absent = false;
+      try { absent = await gone(Math.min(250, Math.max(25, graceMs))); }
+      catch (observation) { error.details.observation_error = errorRecord(observation); }
+      if (absent) {
+        process.stderr.write(`${JSON.stringify({ cleanup_reobserved: errorRecord(error), physical_exit: "ESRCH" })}\n`);
+        return false;
+      }
+      throw error;
+    }
+  };
   if (!alive()) return;
-  if (child.exitCode != null || child.signalCode != null) return stopProcessGroup(child, graceMs);
-  if (!signal("SIGKILL")) return;
-  // Grace controls escalation, not the OS acknowledgement budget. A caller
-  // testing immediate escalation must still allow the kernel to reap children.
-  const deadline = Date.now() + Math.max(1_000, graceMs);
-  while (alive() && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 25));
-  if (alive()) throw Object.assign(new Error(`process group ${child.pid} did not stop`), { code: "process_group_stop_incomplete" });
+  const leaderExited = () => child.exitCode != null || child.signalCode != null;
+  const observeLeaderless = async () => {
+    // Signal grace is not a drain deadline. Helpers can retire naturally after
+    // their leader exits; observing them does not authorize another signal.
+    if (await gone(Math.max(1_000, graceMs, settlementMs))) return;
+    throw Object.assign(new Error("process group remains after its leader exited; ownership cannot be proven from this child handle"), { code: "process_group_ownership_unknown", details: { ...details(null), settlement_ms: Math.max(1_000, graceMs, settlementMs) } });
+  };
+  if (leaderExited()) return await observeLeaderless();
+  if (!await signal("SIGTERM")) return;
+  if (await gone(graceMs)) return;
+  if (leaderExited()) return await observeLeaderless();
+  if (!await signal("SIGKILL")) return;
+  if (!await gone(Math.max(1_000, graceMs))) throw Object.assign(new Error(`process group ${child.pid} did not stop`), { code: "process_group_stop_incomplete", details: details("SIGKILL") });
 }
 
 /** A daemon acknowledgement is not a completed stop until its endpoint closes. */
-export async function confirmDaemonStopped(request, socket, timeoutMs = 30_000) {
-  const result = await request();
-  const deadline = Date.now() + timeoutMs;
-  while (Date.now() < deadline) {
-    try { await lstat(socket); }
-    catch (error) { if (error.code === "ENOENT") return result; throw error; }
-    await new Promise((resolve) => setTimeout(resolve, 25));
+export async function confirmDaemonStopped(request, socket, timeoutMs = 30_000, { readState } = {}) {
+  const deadline = performance.now() + timeoutMs;
+  const expected = await readState?.();
+  const identity = state => state?.daemon_id ?? state?.config?.daemonId;
+  if (readState && (!identity(expected) || !Number.isSafeInteger(expected?.pid) || expected.pid < 1)) throw Object.assign(new Error('daemon stop has no retained incarnation/PID proof'), { code: 'daemon_cleanup_identity_mismatch' });
+  const failure = async observer => {
+    const state = await readState?.();
+    if (expected && identity(state) !== identity(expected)) throw Object.assign(new Error("daemon incarnation changed during stop"), { code: "daemon_identity_mismatch" });
+    const primary = state?.shutdown_state === "cleanup_failed" ? state.cleanup_error : null;
+    if (primary) throw Object.assign(new Error(primary.message), primary, { details: { ...primary.details, socket, phase: "confirm_daemon_stop", observer_error: errorRecord(observer) } });
+    throw observer;
+  };
+  let result;
+  try { result = await request(); }
+  catch (error) { return await failure(error); }
+  if (result?.stopped !== true) return await failure(Object.assign(new Error("stop acknowledgement does not prove shutdown"), { code: "daemon_stop_incomplete" }));
+  for (;;) {
+    const state = await readState?.();
+    if (expected && identity(state) !== identity(expected)) return await failure(Object.assign(new Error("daemon incarnation changed"), { code: "daemon_identity_mismatch" }));
+    let endpointPresent = true;
+    try { await lstat(socket); } catch (error) { if (error.code === "ENOENT") endpointPresent = false; else throw error; }
+    let daemonPresent = false;
+    if (expected?.pid) { try { process.kill(expected.pid, 0); daemonPresent = true; } catch (error) { if (error.code !== "ESRCH") daemonPresent = true; } }
+    if (!endpointPresent && !daemonPresent) {
+      if (readState && result.clean === true && (state?.shutdown_state !== 'clean' || state.active_tree !== false)) return await failure(Object.assign(new Error('daemon clean ACK conflicts with its retained state'), { code: 'daemon_stop_incomplete' }));
+      return result;
+    }
+    if (performance.now() >= deadline) return await failure(Object.assign(new Error("daemon stop could not confirm endpoint retirement and owned PID exit"), { code: "daemon_stop_incomplete", details: { socket, daemon_pid: expected?.pid ?? null, endpoint_present: endpointPresent, daemon_present: daemonPresent } }));
+    await new Promise(resolve => setTimeout(resolve, 25));
   }
-  throw Object.assign(new Error("daemon acknowledged stop but its socket remained live"), { code: "daemon_stop_incomplete", details: { socket } });
 }

package/dist/harness-runtime/lib/native-children.d.mts

@@ -0,0 +1,13 @@
+export const NATIVE_CHILD_CONTRACT_VERSION: "native-children@1";
+export interface NativeChild {
+  session_id: string;
+  parent_session_id: string;
+  status: "running" | "unknown" | "completed" | "failed" | "cancelled" | "settled_by_root";
+  source: string;
+  provenance?: "root_child" | "parent_mismatch";
+  terminal_turn_id?: string;
+  terminal_observed_at?: string;
+  provider_error?: unknown;
+  native_turn_items?: { observed?: boolean; possible_effects?: boolean; pending?: boolean };
+}
+export function normalizeNativeChildren(receipt: Record<string, unknown>, rootSessionId: string): NativeChild[];

package/dist/harness-runtime/lib/native-children.mjs

@@ -0,0 +1,78 @@
+export const NATIVE_CHILD_CONTRACT_VERSION = "native-children@1";
+import { errorRecord } from "./operation-errors.mjs";
+
+function nativeFailure(code, message, exitCode, details) { return Object.assign(new Error(message), { code, exitCode, details }); }
+
+
+/** Normalize provider-observed topology only; never parse assistant prose. */
+export function normalizeNativeChildren(receipt, rootSessionId) {
+  const candidates = [];
+  const object = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : {};
+  receipt = object(receipt);
+  const list = (value) => Array.isArray(value) ? value : [];
+  const text = (value, keys) => { for (const key of keys) if (typeof value[key] === "string" && value[key]) return value[key]; return null; };
+  const settledTree = new Set(
+    list(receipt.tree_observations)
+      .filter(value => object(value).settled === true)
+      .map(value => text(object(value), ["provider_session_id", "session_id"]))
+      .filter(id => id !== null)
+  );
+  const add = (raw, source, scopedParent = null, fallback = "unknown") => {
+    const value = object(raw);
+    const id = text(value, ["provider_session_id", "session_id", "sessionId", "childSessionId", "conversation_id", "conversationId", "id", "subagentId"]);
+    const parent = text(value, ["parent_provider_session_id", "parent_session_id", "parentSessionId", "parentID", "parentId"]) ?? scopedParent;
+    if (!id) return;
+    const outcomeError = object(value.outcome_error);
+    if (typeof outcomeError.code === "string") {
+      const record = errorRecord(outcomeError);
+      throw Object.assign(nativeFailure(record.code, record.message, 1, record.details), record, { details: { ...object(record.details), session_id: id, source } });
+    }
+    const state = String(value.status ?? value.state ?? fallback).toLowerCase();
+    if (state === "lost") throw nativeFailure("native_child_outcome_lost", "Native child ended without a recoverable outcome", 1, { session_id: id, parent_session_id: parent, source, native_status: state });
+    // Provider idle means no current activity, not a Work result. Only an
+    // explicit terminal result or a separately proven root settlement may be
+    // terminal here; unknown states stay unknown for reconciliation.
+    const status = ["completed", "complete", "done", "success", "succeeded"].includes(state) ? "completed" : state === "settled_by_root" ? "settled_by_root" : ["failed", "error"].includes(state) ? "failed" : ["cancelled", "canceled", "interrupted"].includes(state) ? "cancelled" : ["running", "active", "pending"].includes(state) ? "running" : fallback;
+    const rootChild = parent === rootSessionId;
+    const settled = rootChild && status === "unknown" && settledTree.has(id) ? "settled_by_root" : status;
+    candidates.push({ session_id: id, parent_session_id: parent ?? "", status: settled, source, ...(rootChild ? {} : { provenance: "parent_mismatch" }),
+      ...(text(value, ["terminal_turn_id"]) ? { terminal_turn_id: text(value, ["terminal_turn_id"]) } : {}),
+      ...(text(value, ["terminal_observed_at"]) ? { terminal_observed_at: text(value, ["terminal_observed_at"]) } : {}),
+      ...(value.provider_error !== undefined ? { provider_error: value.provider_error } : {}),
+      ...(value.native_turn_items && typeof value.native_turn_items === "object" ? { native_turn_items: value.native_turn_items } : {}) });
+  };
+  for (const value of list(receipt.descendants)) add(value, "adapter.descendants");
+  for (const value of list(receipt.children)) add(value, "opencode.session.children");
+  const subagents = object(object(receipt.evidence).subagents ?? receipt.subagents);
+  for (const value of list(subagents.running)) add(value, "zcode/session/subagents", rootSessionId, "running");
+  for (const value of list(subagents.completed)) add(value, "zcode/session/subagents", rootSessionId, "completed");
+  for (const value of list(object(subagents.ended).items)) {
+    const status = String(object(value).status ?? "unknown");
+    if (!["success", "failed", "cancelled", "lost"].includes(status)) throw nativeFailure("native_child_outcome_unavailable", "ZCode ended child has no recognized native outcome", 1, { child: value });
+    add(value, "zcode/session/subagents", rootSessionId);
+  }
+  const unique = new Map();
+  const rank = { unknown: 0, running: 1, settled_by_root: 2, completed: 3, failed: 3, cancelled: 3 };
+  for (const child of candidates) {
+    const previous = unique.get(child.session_id);
+    if (previous && previous.parent_session_id !== child.parent_session_id) {
+      // Keep the contradictory parent visible to the existing provenance gate,
+      // independently of input order or which source happened to say success.
+      const foreign = [previous, child].filter(item => item.parent_session_id !== rootSessionId).sort((a, b) => a.parent_session_id.localeCompare(b.parent_session_id))[0];
+      unique.set(child.session_id, { ...foreign, provenance: "parent_mismatch" });
+      continue;
+    }
+    if (previous && rank[previous.status] === 3 && rank[child.status] === 3 && previous.status !== child.status) throw nativeFailure("native_child_outcome_conflict", "Native child has contradictory terminal outcomes", 1, { session_id: child.session_id, observations: [previous, child] });
+    if (previous && previous.status === child.status && previous.terminal_turn_id && child.terminal_turn_id && previous.terminal_turn_id !== child.terminal_turn_id)
+      throw nativeFailure("native_child_turn_conflict", "Native child has contradictory terminal Turn identities", 1, { session_id: child.session_id, observations: [previous, child] });
+    if (previous && previous.status === child.status) {
+      unique.set(child.session_id, { ...previous, ...child,
+        ...(child.terminal_turn_id ?? previous.terminal_turn_id ? { terminal_turn_id: child.terminal_turn_id ?? previous.terminal_turn_id } : {}),
+        ...(child.provider_error ?? previous.provider_error ? { provider_error: child.provider_error ?? previous.provider_error } : {}),
+        ...(child.native_turn_items ?? previous.native_turn_items ? { native_turn_items: child.native_turn_items ?? previous.native_turn_items } : {}) });
+      continue;
+    }
+    if (!previous || rank[child.status] > rank[previous.status]) unique.set(child.session_id, child);
+  }
+  return [...unique.values()].sort((left, right) => left.session_id.localeCompare(right.session_id));
+}

package/dist/harness-runtime/lib/native-hook-command.d.mts

@@ -1,3 +1,7 @@
 export function invokeNativeHook(input: { bin: string; args: string[]; payload: unknown; home?: string | undefined; cwd?: string; env?: NodeJS.ProcessEnv; timeoutMs?: number }): Promise<Record<string, unknown> & { timing?: Record<string, number> }>;
 export function hookBudget(timeoutMs?: number, deadlineAt?: number | null): { deadlineAt: number; remaining(phase: string, maximum?: number): number; run<T>(action: () => T | Promise<T>, phase: string): Promise<T> };
 export function readNativeHookInput(stream: NodeJS.ReadStream, budget: { remaining(phase: string): number }, maxBytes?: number): Promise<string>;
+export function acceptNativeHookRequest(stateDir: string, binding: Record<string, unknown>, current?: { requests: Map<string, { binding: Record<string, unknown> }>; accepting?: Map<string, Promise<Record<string, unknown>>> } | null): Promise<Record<string, unknown>>;
+export function writeNativeHookFailure(stateDir: string, binding: Record<string, unknown> | undefined, error: unknown, budget: ReturnType<typeof hookBudget>): Promise<void>;
+export function readNativeHookFailures(stateDir: string, requests: Map<string, { current: unknown; binding: Record<string, unknown> }>, current: unknown, reject: (error: Error & { code?: string; details?: Record<string, unknown> }) => void): Promise<void>;
+export function runBoundedCommand(input: { bin: string; args: string[]; input?: string | null; cwd?: string; env?: NodeJS.ProcessEnv; timeoutMs?: number; timeoutCode?: string; inheritDeadline?: boolean; cleanupWithinBudget?: boolean; maintenanceProgress?: { action: string; processId: string | null; inactivityMs: number } | null; signal?: AbortSignal }): Promise<string>;