@openrig/cli 0.5.10 → 0.5.12

package/dist/daemon-lifecycle.js

@@ -1,6 +1,8 @@
 import path from "node:path";
 import { existsSync } from "node:fs";
+import { DAEMON_STOP_WAIT_MS, DAEMON_SHUTDOWN_RECEIPT } from "@openrig/daemon/daemon-shutdown";
 import { ConfigStore } from "./config-store.js";
+import { readLocalOrigin } from "./local-origin.js";
 import { OPENRIG_HOME, LEGACY_RIGGED_HOME, readOpenRigEnv } from "./openrig-compat.js";
 import { ensureDefaultWorkspace, ensureOpenRigInstance, formatInstanceInitializationConflicts, } from "@openrig/daemon/instance-initialization";
 /** Build the daemon HTTP URL from status. Uses persisted host or defaults to 127.0.0.1. */
@@ -202,7 +204,7 @@ function readState(deps) {
     }
     catch {
         // Malformed daemon.json — treat as no state
-        console.error("Warning: malformed daemon.json, treating as stopped");
+        console.error("Warning: malformed daemon.json; target identity unavailable");
         return null;
     }
 }
@@ -385,7 +387,27 @@ export async function verifyRequiredListeners(input) {
     }
     return { ok: true, verified };
 }
+class StartupIdentityError extends Error {
+}
+class StartupChildPendingError extends Error {
+}
 export async function startDaemon(opts, deps) {
+    if (!deps.acquireStartLock)
+        throw new Error("Daemon startup requires a local launch reservation");
+    const lock = deps.acquireStartLock();
+    let preserve = false;
+    try {
+        return await startOwnedDaemon(opts, deps, lock);
+    }
+    catch (error) {
+        preserve = error instanceof StartupChildPendingError;
+        throw error;
+    }
+    finally {
+        lock.release(preserve);
+    }
+}
+async function startOwnedDaemon(opts, deps, lock) {
     const port = opts.port ?? DEFAULT_PORT;
     const db = opts.db ?? DEFAULT_DB;
     // bug-fix slice auth-bearer-tailscale-trust: preserve the
@@ -437,151 +459,285 @@ export async function startDaemon(opts, deps) {
         throw new Error(`OpenRig instance initialization blocked: ${formatInstanceInitializationConflicts(initialization)}`);
     }
     const logFd = deps.openForAppend(LOG_FILE);
-    const child = deps.spawn(process.execPath, [daemonEntry], {
-        env: buildDaemonEnv(process.env, {
-            port,
-            host: explicitHost,
-            db,
-            transcriptsEnabled: opts.transcriptsEnabled,
-            transcriptsPath: opts.transcriptsPath,
-            transcriptsLines: opts.transcriptsLines,
-            transcriptsPollIntervalSeconds: opts.transcriptsPollIntervalSeconds,
-            skipKernelBoot: opts.skipKernelBoot,
-        }),
-        stdio: ["ignore", logFd, logFd],
-        detached: true,
-    });
+    let child;
+    try {
+        child = deps.spawn(process.execPath, [daemonEntry], {
+            env: buildDaemonEnv(process.env, {
+                port,
+                host: explicitHost,
+                db,
+                transcriptsEnabled: opts.transcriptsEnabled,
+                transcriptsPath: opts.transcriptsPath,
+                transcriptsLines: opts.transcriptsLines,
+                transcriptsPollIntervalSeconds: opts.transcriptsPollIntervalSeconds,
+                skipKernelBoot: opts.skipKernelBoot,
+            }),
+            stdio: ["ignore", logFd, logFd],
+            detached: true,
+        });
+    }
+    finally {
+        deps.closeFile?.(logFd);
+    }
+    let childFailure;
+    let rejectExit;
+    let resolveExit;
+    const exited = new Promise((resolve) => { resolveExit = resolve; });
+    const failed = new Promise((_, reject) => { rejectExit = reject; });
+    // An error can arrive during synchronous publication/cleanup, between awaits.
+    void failed.catch(() => { });
+    const onError = (error) => {
+        childFailure = new Error(`Daemon child failed to spawn: ${error.message}`);
+        rejectExit(childFailure);
+    };
+    const onExit = (code, signal) => {
+        childFailure = new Error(`Daemon child ${child.pid ?? "unknown"} exited before startup completed (code ${code}, signal ${signal ?? "none"})`);
+        rejectExit(childFailure);
+        resolveExit();
+    };
+    child.once("error", onError);
+    child.once("exit", onExit);
     child.unref();
     const pid = child.pid;
-    // Poll healthz against the configured probe host (loopback default
-    // when operator didn't opt in to a specific bind).
+    const hasExited = () => child.exitCode != null || child.signalCode != null;
+    const assertChild = () => {
+        if (childFailure)
+            throw childFailure;
+        if (!Number.isSafeInteger(pid) || pid <= 0)
+            throw new Error("Daemon spawn returned no valid child PID");
+        if (hasExited())
+            throw new Error(`Daemon child ${pid} exited before startup completed`);
+    };
     const healthzUrl = `http://${probeHost}:${port}/healthz`;
-    let healthy = false;
-    for (let i = 0; i < HEALTHZ_RETRIES; i++) {
+    const readOwnedHealth = async (url) => {
+        assertChild();
+        let timer;
         try {
-            const res = await fetchDaemonProbe(deps, healthzUrl, HEALTHZ_PROBE_TIMEOUT_MS);
-            if (res.ok) {
-                healthy = true;
-                break;
-            }
-        }
-        catch {
-            // Not ready yet
+            return await Promise.race([
+                failed,
+                new Promise((_, reject) => {
+                    timer = setTimeout(() => reject(new HealthProbeTimeoutError(url)), HEALTHZ_PROBE_TIMEOUT_MS);
+                }),
+                (async () => {
+                    const res = await deps.fetch(url);
+                    if (!res.ok)
+                        return null;
+                    let body;
+                    try {
+                        body = res.json ? await res.json() : null;
+                    }
+                    catch {
+                        throw new StartupIdentityError(`Daemon startup identity unavailable at ${url}: expected child PID ${pid}; invalid health response`);
+                    }
+                    if (body?.pid !== pid) {
+                        throw new StartupIdentityError(`Daemon startup identity mismatch at ${url}: expected child PID ${pid}, observed ${typeof body?.pid === "number" ? body.pid : "unknown"}; state not published`);
+                    }
+                    assertChild();
+                    return body;
+                })(),
+            ]);
         }
-        await new Promise((r) => setTimeout(r, HEALTHZ_DELAY_MS));
-    }
-    if (!healthy) {
-        try {
-            deps.kill(pid, "SIGTERM");
+        finally {
+            if (timer)
+                clearTimeout(timer);
         }
-        catch { /* best effort */ }
-        const logContent = deps.readFile(resolveLifecycleFile(deps, "daemon.log"));
-        throw new Error(summarizeDaemonStartFailure(healthzUrl, logContent));
-    }
-    // S20 — the restored adoption/upgrade gate: when the daemon reports its bind plan,
-    // derive the required listener set from the EFFECTIVE mode and prove every listener
-    // by probing its own /healthz. A silently dropped listener fails the start LOUDLY
-    // (the 0.5.3-receipt regression accepted loopback health alone and missed exactly
-    // this). Older daemons without the payload skip the gate unchanged (additive).
+    };
     try {
-        const healthRes = await fetchDaemonProbe(deps, healthzUrl, HEALTHZ_PROBE_TIMEOUT_MS);
-        const healthBody = (await (healthRes.json?.() ?? Promise.resolve(null)).catch(() => null));
-        if (healthBody?.bind) {
-            const gate = await verifyRequiredListeners({
-                bind: healthBody.bind,
-                port,
-                // r2 repair — error PROVENANCE preserved: an explicit non-OK answer or a
-                // connection refusal is positive evidence ("unhealthy"); a timeout or any
-                // other transient probe exception is "indeterminate" and can never kill.
-                probe: async (url) => {
-                    try {
-                        const r = await fetchDaemonProbe(deps, url, HEALTHZ_PROBE_TIMEOUT_MS);
-                        return r.ok ? "healthy" : "unhealthy";
+        assertChild();
+        lock.recordChild(pid);
+        let healthy = false;
+        let lastProbe = "health endpoint not responding";
+        for (let i = 0; i < HEALTHZ_RETRIES; i++) {
+            try {
+                const body = await readOwnedHealth(healthzUrl);
+                if (body) {
+                    const bind = body.bind;
+                    if (!bind || !["explicit", "default"].includes(bind.mode) || !Array.isArray(bind.hosts)
+                        || bind.hosts.length === 0 || !bind.hosts.every((host) => typeof host === "string" && host.length > 0)
+                        || typeof bind.tailscaleDetected !== "boolean") {
+                        throw new StartupIdentityError(`Daemon startup listener identity unavailable for child PID ${pid}; state not published`);
                     }
-                    catch (err) {
-                        const code = err.code
-                            ?? err.cause?.code;
-                        return code === "ECONNREFUSED" ? "unhealthy" : "indeterminate";
+                    const gate = await verifyRequiredListeners({ bind, port, probe: async (url) => {
+                            try {
+                                return await readOwnedHealth(url) ? "healthy" : "unhealthy";
+                            }
+                            catch (error) {
+                                if (error instanceof StartupIdentityError || childFailure)
+                                    throw error;
+                                const code = error.code
+                                    ?? error.cause?.code;
+                                return code === "ECONNREFUSED" ? "unhealthy" : "indeterminate";
+                            }
+                        } });
+                    if (gate.ok === false)
+                        throw new StartupIdentityError(`Daemon child ${pid} FAILED the listener adoption gate: ${gate.reason}`);
+                    if (gate.ok === true) {
+                        healthy = true;
+                        break;
                     }
-                },
-            });
-            if (gate.ok === false) {
-                try {
-                    deps.kill(pid, "SIGTERM");
+                    lastProbe = gate.reason;
                 }
-                catch { /* best effort */ }
-                throw new Error(`daemon started but FAILED the listener adoption gate: ${gate.reason}`);
             }
-            // gate.ok === "indeterminate" falls through healthy: no evidence, no action.
+            catch (error) {
+                assertChild();
+                if (error instanceof StartupIdentityError)
+                    throw error;
+                lastProbe = error instanceof Error ? error.message : String(error);
+            }
+            await Promise.race([new Promise((resolve) => setTimeout(resolve, HEALTHZ_DELAY_MS)), failed]);
+        }
+        if (!healthy) {
+            throw new Error(`${summarizeDaemonStartFailure(healthzUrl, deps.readFile(resolveLifecycleFile(deps, "daemon.log")))}. Last probe: ${lastProbe}`);
+        }
+        assertChild();
+        if (!deps.isProcessAlive(pid))
+            throw new Error(`Daemon child ${pid} liveness could not be confirmed; state not published`);
+        const state = { pid: pid, port, host: probeHost, db, startedAt: new Date().toISOString() };
+        try {
+            deps.writeFile(STATE_FILE, JSON.stringify(state, null, 2));
+            assertChild();
+            // A synchronous writer can outlast the process while exit events remain
+            // undelivered. Recheck physical evidence on the other side of publication.
+            if (!deps.isProcessAlive(pid))
+                throw new Error(`Daemon child ${pid} liveness could not be confirmed at state publication; startup not accepted`);
+        }
+        catch (error) {
+            removeMatchingState(deps, STATE_FILE, state);
+            throw error;
         }
+        return state;
     }
-    catch (err) {
-        if (err instanceof Error && err.message.includes("listener adoption gate"))
-            throw err;
-        // A parse/probe hiccup on the gate read must not kill an otherwise healthy start;
-        // the gate acts only on POSITIVE evidence of a bad bind (never on its own failure).
-    }
-    const state = {
-        pid,
-        port,
-        // Persist the probe host (operator-explicit when set, loopback
-        // default otherwise) so subsequent status reads can reach the
-        // daemon regardless of whether tailscale was added at multi-bind.
-        host: probeHost,
-        db,
-        startedAt: new Date().toISOString(),
-    };
-    deps.writeFile(STATE_FILE, JSON.stringify(state, null, 2));
-    return state;
+    catch (error) {
+        // Only this launch's child may be cleaned up. A retained child also retains
+        // the reservation so a retry cannot run another pre-bind initialization.
+        // A failed ps is not exit evidence. This is our unreaped child: signal only
+        // that PID, then release exclusion only after its own exit is observed.
+        if (Number.isSafeInteger(pid) && pid > 0 && !hasExited()) {
+            try {
+                deps.kill(pid, "SIGTERM");
+            }
+            catch { /* confirm exit below */ }
+            let timer;
+            try {
+                const gone = await Promise.race([
+                    exited.then(() => true),
+                    new Promise((resolve) => { timer = setTimeout(() => resolve(false), DAEMON_STOP_WAIT_MS); }),
+                ]);
+                if (!gone)
+                    throw new StartupChildPendingError(`${error instanceof Error ? error.message : error}. Child PID ${pid} exit is unconfirmed; startup reservation retained. Inspect daemon-start.lock and daemon.log before recovery.`);
+            }
+            finally {
+                if (timer)
+                    clearTimeout(timer);
+            }
+        }
+        throw error;
+    }
+    finally {
+        // A native spawn failure with no PID emits its error on the next turn.
+        if (pid !== undefined || childFailure)
+            child.off("error", onError);
+        child.off("exit", onExit);
+    }
+}
+function readShutdownReceipt(deps, stateFile) {
+    const receiptPath = path.join(path.dirname(stateFile), DAEMON_SHUTDOWN_RECEIPT);
+    const raw = deps.readFile(receiptPath);
+    let receipt;
+    try {
+        receipt = JSON.parse(raw ?? "null") ?? undefined;
+    }
+    catch { /* unavailable */ }
+    if (receipt?.schema !== "openrig.daemon-shutdown/v1" || !Number.isSafeInteger(receipt.pid) || receipt.pid <= 0
+        || !Number.isFinite(Date.parse(receipt.startedAt)) || !(Date.parse(receipt.completedAt) >= Date.parse(receipt.startedAt))
+        || !(Date.parse(receipt.completedAt) <= Date.now()) || typeof receipt.phase !== "string"
+        || !Array.isArray(receipt.failures) || !["clean", "failed", "timed-out"].includes(receipt.outcome))
+        receipt = undefined;
+    return { receipt, present: raw !== null || deps.exists(receiptPath), receiptPath };
+}
+function receiptMatchesState(receipt, state, notBefore = Date.parse(state.startedAt)) {
+    return receipt?.pid === state.pid && Date.parse(receipt.startedAt) >= notBefore;
+}
+function isCleanShutdown(receipt) {
+    return receipt?.outcome === "clean" && receipt.phase === "complete" && receipt.failures.length === 0;
+}
+function removeMatchingState(deps, stateFile, state) {
+    // A concurrent start/rebind must not have its state removed by an old stop/status.
+    const current = readState(deps);
+    if (current?.pid === state.pid && current.startedAt === state.startedAt && current.port === state.port
+        && current.host === state.host && current.db === state.db)
+        deps.removeFile(stateFile);
 }
 export async function stopDaemon(deps) {
     const state = readState(deps);
-    if (!state) {
-        const configured = resolveConfiguredDaemonTarget();
-        let recoveredRunning = false;
+    const configured = resolveConfiguredDaemonTarget();
+    const explicitUrl = readOpenRigEnv("OPENRIG_URL", "RIGGED_URL");
+    const stateUrl = state ? `http://${state.host ?? DEFAULT_HOST}:${state.port}` : undefined;
+    const target = stateUrl ?? explicitUrl?.replace(/\/+$/, "") ?? `http://${configured.host}:${configured.port}`;
+    if (stateUrl && explicitUrl && new URL(explicitUrl).origin !== new URL(stateUrl).origin) {
+        throw new Error(`Cannot stop safely: addressed ${explicitUrl}/healthz does not match local PID ${state.pid} at ${stateUrl}/healthz; no signal sent.`);
+    }
+    const check = `${target}/healthz`;
+    const listener = async () => {
         try {
-            await fetchDaemonProbe(deps, `http://${configured.host}:${configured.port}/healthz`, HEALTHZ_PROBE_TIMEOUT_MS);
-            recoveredRunning = true;
+            await fetchDaemonProbe(deps, check, HEALTHZ_PROBE_TIMEOUT_MS);
+            return "responding";
         }
-        catch (err) {
-            if (err instanceof HealthProbeTimeoutError) {
-                throw new Error(`Daemon state is missing and port ${configured.port} is unresponsive — cannot stop safely.`);
-            }
+        catch (error) {
+            return isRefusedError(error) ? "refused" : "unavailable";
         }
-        if (recoveredRunning) {
-            throw new Error(`Daemon is running on port ${configured.port}, but daemon state is missing — cannot stop safely.`);
+    };
+    if (!state) {
+        const effect = await listener();
+        const sibling = findSiblingHome(deps);
+        if (effect !== "refused" || sibling) {
+            throw new Error(`Daemon state is missing — cannot stop safely. Checked ${check}; listener ${effect}.` +
+                (sibling ? ` Resolved home ${sibling.resolvedHome}; live sibling ${sibling.siblingHome}.` : ""));
         }
-        return;
+        const stateFile = resolveLifecycleFile(deps, "daemon.json");
+        if (deps.exists(stateFile)) {
+            throw new Error(`Daemon target state is unreadable; drain completion unverified. Checked ${check}; listener refused; no signal sent. Inspect ${stateFile}.`);
+        }
+        const prior = readShutdownReceipt(deps, stateFile);
+        if (prior.present && !isCleanShutdown(prior.receipt)) {
+            throw new Error(`Drain completion unverified: local shutdown evidence exists but no target state is recorded; cannot attribute it to ${target}. Checked ${check}; listener refused. Inspect ${prior.receiptPath}.`);
+        }
+        // No target is not a clean-drain verdict, including after an earlier clean stop.
+        return "no-target";
     }
     const stateFile = resolveLifecycleFile(deps, "daemon.json");
     const pidState = await checkPid(state, deps);
-    if (pidState === "dead") {
-        // Already dead — clean up stale state
-        deps.removeFile(stateFile);
-        return;
-    }
     if (pidState === "not_openrig") {
-        // PID alive but not our daemon (reused PID) — clean up state, don't kill
-        deps.removeFile(stateFile);
-        return;
+        throw new Error(`Cannot stop safely: PID ${state.pid} is present but identity is not confirmed at ${check}; no signal sent, state preserved.`);
     }
-    // pidState === "openrig" or "unresponsive" — safe to SIGTERM
-    deps.kill(state.pid, "SIGTERM");
-    // Wait briefly for process to exit
-    let exited = false;
-    for (let i = 0; i < 20; i++) {
-        if (!deps.isProcessAlive(state.pid)) {
-            exited = true;
-            break;
+    let notBefore = Date.parse(state.startedAt);
+    if (pidState !== "dead") {
+        notBefore = Date.now();
+        deps.kill(state.pid, "SIGTERM");
+        // One signal. An already-exited target goes straight to the same judgment.
+        const deadline = Date.now() + DAEMON_STOP_WAIT_MS;
+        while (deps.isProcessAlive(state.pid) && Date.now() < deadline) {
+            await new Promise((r) => setTimeout(r, 100));
         }
-        await new Promise((r) => setTimeout(r, 100));
     }
-    if (exited) {
-        deps.removeFile(stateFile);
+    const present = deps.isProcessAlive(state.pid);
+    const effect = await listener();
+    const observation = `targeted ${target}; checked ${check}; PID ${state.pid} ${present ? "present" : "absent"}; listener ${effect}`;
+    if (present) {
+        throw new Error(`Daemon did not exit after SIGTERM within ${DAEMON_STOP_WAIT_MS}ms; ${observation}` +
+            (effect === "responding" ? " (still listening)" : "") + "; state file preserved.");
     }
-    else {
-        throw new Error(`Daemon (pid ${state.pid}) did not exit after SIGTERM — state file preserved`);
+    if (effect !== "refused")
+        throw new Error(`Daemon stop verification is unverified: ${observation}; state preserved.`);
+    const { receipt, receiptPath } = readShutdownReceipt(deps, stateFile);
+    if (!receiptMatchesState(receipt, state, notBefore)) {
+        throw new Error(`Daemon process stopped; ${observation}; drain completion unverified (no matching shutdown receipt); state preserved. Inspect ${path.join(path.dirname(stateFile), "daemon.log")}.`);
+    }
+    if (!isCleanShutdown(receipt)) {
+        throw new Error(`Daemon stopped with incomplete shutdown: ${receipt.outcome}; phase=${receipt.phase}; ${observation}. Pending effects are unverified; state preserved; inspect ${receiptPath}.`);
     }
+    removeMatchingState(deps, stateFile, state);
+    return "stopped";
 }
 /** RULING 1ae863d2 — positive-down evidence classifier (lockstep with the daemon's
  *  crash-cart-detect semantics): ONLY a connection refusal is strong down evidence;
@@ -675,9 +831,11 @@ export async function getDaemonStatus(deps, options = {}) {
         }
     }
     if (!deps.isProcessAlive(state.pid)) {
-        // Process dead — stale state
-        if (options.cleanupStaleState !== false) {
-            deps.removeFile(resolveLifecycleFile(deps, "daemon.json"));
+        // A status read must not erase the identity needed to judge a failed stop.
+        const stateFile = resolveLifecycleFile(deps, "daemon.json");
+        const { receipt } = readShutdownReceipt(deps, stateFile);
+        if (options.cleanupStaleState !== false && receiptMatchesState(receipt, state) && isCleanShutdown(receipt)) {
+            removeMatchingState(deps, stateFile, state);
         }
         return { state: "stale" };
     }
@@ -818,32 +976,9 @@ export async function fetchSelfHostIdentity(deps, url) {
         return undefined;
     }
 }
-/**
- * A4 HTTP-path origin-triple carry: resolve THIS host's `selfHostId` for a REMOTE-targeting client,
- * so the stamped X-OpenRig-Session becomes the origin TRIPLE (member@rig@selfHostId) and the remote
- * daemon renders the ORIGIN host. The id comes from the LOCAL daemon's /healthz (env OPENRIG_URL,
- * else the running local daemon), via the shipped fail-open `fetchSelfHostId`. C1 fail-open all the
- * way down: no env URL and no running local daemon ⇒ `undefined` ⇒ the 2-part header stamps (today's
- * behavior, no new failure mode). This is THIS host's own derived id, never a caller-supplied string.
- */
-export async function resolveOriginSelfHostId(deps) {
-    // C1 fail-open around the WHOLE resolution, not just fetchSelfHostId: a remote op must NEVER depend on
-    // local daemon health. getDaemonStatus can throw (down / mid-restart / minimal test deps); ANY failure
-    // ⇒ undefined ⇒ the 2-part header stamps (today's behavior). Pin 5: no new failure mode on the remote path.
-    try {
-        let localUrl = readOpenRigEnv("OPENRIG_URL", "RIGGED_URL");
-        if (!localUrl) {
-            const status = await getDaemonStatus(deps);
-            if (status.state === "running" && status.port !== undefined)
-                localUrl = getDaemonUrl(status);
-        }
-        if (!localUrl)
-            return undefined;
-        return await fetchSelfHostId(deps, localUrl);
-    }
-    catch {
-        return undefined;
-    }
+/** Resolve this instance's persisted identity even during an outage; never read it from the target. */
+export async function resolveOriginSelfHostId(_deps) {
+    return readLocalOrigin();
 }
 const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /**