@sentropic/h2a-runtime 0.98.6 → 0.98.7-rc.10

package/dist/native-terminal/op.js

@@ -34,16 +34,35 @@ import { dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import { NativeTerminalClient } from "./client.js";
 import { loadRegistry, registryEntriesForNativeTarget } from "../registry.js";
-import { NATIVE_TERMINAL_INTERACTIVE_READ_TIMEOUT_MS, NATIVE_TERMINAL_PROTOCOL_VERSION, NativeTerminalRemoteError, } from "./protocol.js";
+import { NATIVE_TERMINAL_INTERACTIVE_READ_TIMEOUT_MS, NATIVE_TERMINAL_HEALTH_TIMEOUT_MS, NATIVE_TERMINAL_PROTOCOL_VERSION, NativeTerminalRemoteError, } from "./protocol.js";
 import { NativeTerminalHostSupervisor } from "./supervisor.js";
-import { proveNativeTerminalEndpointAbsent } from "./server.js";
+import { proveNativeTerminalEndpointAbsent, withNativeTerminalStartupLock } from "./server.js";
 import { defaultNativeTerminalSocketPath, knownNativeTerminalSocketPaths, inspectPrivateNativeTerminalSocket, sameNativeTerminalSocket } from "./socket-path.js";
-import { collectNativeInventory, resolveNativeOwner } from "./fleet.js";
+import { collectNativeInventory, publicationContenderObserved, readNativeTerminalSuccessor, resolveNativeOwner } from "./fleet.js";
 import { renderTerminalScreen } from "./screen.js";
 import { TerminalModeTracker } from "./terminal-modes.js";
 import { publishAttachReset } from "./attach-recovery.js";
 import { TerminalResetRelay } from "./reset-relay.js";
+import { appendCrashLog, crashSignalName, installCrashHandlers } from "../crash-log.js";
 const DETACH_BYTE = 0x1c; // Ctrl-\
+/**
+ * Build the crash-log record for an observed session exit. Only observed
+ * values are recorded: `exitCode` comes from the session's exit report and
+ * `signal` from the stop order (`stopSignal`) or the pty exit signal. Either
+ * field is omitted when unknown — never defaulted to 0/1.
+ */
+export function sessionExitCrashEntry(state, options) {
+    const exitCode = state.exit?.exitCode;
+    const signal = state.stopSignal ?? crashSignalName(state.exit?.signal ?? undefined);
+    return {
+        event: "sessionExit",
+        sessionId: options.sessionId,
+        ...(options.reason ? { reason: options.reason } : {}),
+        ...(options.detail ? { detail: options.detail } : {}),
+        ...(typeof exitCode === "number" && Number.isSafeInteger(exitCode) ? { exitCode } : {}),
+        ...(signal ? { signal } : {}),
+    };
+}
 function parseArgv(argv) {
     const op = argv[2];
     if (!op)
@@ -77,7 +96,7 @@ function required(parsed, name) {
         throw new Error(`--${name} is required`);
     return value;
 }
-function emit(payload) {
+function emitResult(payload) {
     process.stdout.write(`${JSON.stringify(payload)}\n`);
 }
 export function stripAnsi(text) {
@@ -104,12 +123,17 @@ export function closeAllNativeTerminalOpClients() {
         client.close();
     openClients.clear();
 }
-async function connectExisting(socketPath, tracked = true) {
+async function connectExisting(socketPath, tracked = true, signal) {
+    signal?.throwIfAborted();
     const identity = await inspectPrivateNativeTerminalSocket(socketPath);
+    signal?.throwIfAborted();
     const client = await NativeTerminalClient.connect(socketPath);
     if (tracked)
         openClients.add(client);
+    const abort = () => client.close();
+    signal?.addEventListener("abort", abort, { once: true });
     try {
+        signal?.throwIfAborted();
         const ping = await client.ping();
         if (ping.protocolVersion !== NATIVE_TERMINAL_PROTOCOL_VERSION || !ping.generation?.trim() ||
             !Number.isSafeInteger(ping.hostPid) || ping.hostPid <= 0 ||
@@ -119,8 +143,12 @@ async function connectExisting(socketPath, tracked = true) {
     }
     catch (error) {
         client.close();
+        openClients.delete(client);
         throw error;
     }
+    finally {
+        signal?.removeEventListener("abort", abort);
+    }
     return client;
 }
 class NativeOwnerError extends Error {
@@ -134,6 +162,34 @@ async function inventoryFor(parsed) {
     const pinned = parsed.flags.get("socket");
     return collectNativeInventory(pinned ? [pinned] : knownNativeTerminalSocketPaths(), connectExisting, proveNativeTerminalEndpointAbsent);
 }
+/** Same three-state proof for in-process callers and the one-shot probe.
+ * Own these connections locally: an async caller must close them even when
+ * another endpoint makes the inventory incomplete. */
+export async function probeNativeSession(id, socketPath, ensureHost = false) {
+    // A run may prepare its host before the read-only guards. Otherwise every
+    // relauncher first queues multiple absence proofs, delaying the publisher
+    // until those same callers' publication/readiness budgets expire.
+    if (ensureHost) {
+        const client = socketPath ? await launchClient(socketPath) : (await selectLaunchClient(true)).client;
+        client.close();
+        openClients.delete(client);
+    }
+    const inventory = await collectNativeInventory(socketPath ? [socketPath] : knownNativeTerminalSocketPaths(), path => connectExisting(path, false), proveNativeTerminalEndpointAbsent);
+    try {
+        const owner = resolveNativeOwner(id, inventory);
+        if (owner.state === "found")
+            return { verdict: owner.session.status === "running" ? "live" : "dead",
+                state: { ...owner.session, socketPath: owner.socketPath } };
+        if (owner.state === "absent")
+            return { verdict: "dead" };
+        return { verdict: "unknown", code: owner.state,
+            ...(owner.state === "unknown" ? { reason: owner.reason } : { sockets: owner.sockets, reason: "ambiguous-owner" }) };
+    }
+    finally {
+        for (const host of inventory.hosts)
+            host.client?.close();
+    }
+}
 async function owningClient(parsed, id) {
     const owner = resolveNativeOwner(id, await inventoryFor(parsed));
     if (owner.state !== "found")
@@ -153,31 +209,71 @@ async function selectLaunchClient(fenced) {
     }
     const historical = paths[0];
     const client = await launchClient(historical);
-    const ping = await client.ping();
-    if (!fenced || paths.length === 1)
-        return { client, socketPath: historical };
-    const compatible = paths[1];
-    // Initialize the second known endpoint so admission obtains a complete
-    // inventory even when the historical host already provides the fence.
-    const second = await launchClient(compatible);
-    return ping.launchFence === true ? { client, socketPath: historical } : { client: second, socketPath: compatible };
+    try {
+        const ping = await client.ping();
+        if (!fenced || paths.length === 1)
+            return { client, socketPath: historical };
+        const compatible = paths[1];
+        // Initialize the second known endpoint so admission obtains a complete
+        // inventory even when the historical host already provides the fence.
+        const second = await launchClient(compatible);
+        const unused = ping.launchFence === true ? second : client;
+        unused.close();
+        openClients.delete(unused);
+        return ping.launchFence === true ? { client, socketPath: historical } : { client: second, socketPath: compatible };
+    }
+    catch (error) {
+        client.close();
+        openClients.delete(client);
+        throw error;
+    }
 }
 async function launchClient(socketPath) {
+    let failure;
     try {
         return await connectExisting(socketPath);
     }
     catch (error) {
-        let absent = false;
-        try {
-            absent = await proveNativeTerminalEndpointAbsent(socketPath, error);
-        }
-        catch (proofError) {
-            error = new Error(`${error instanceof Error ? error.message : String(error)}; absence proof failed: ${proofError instanceof Error ? proofError.message : String(proofError)}`);
+        failure = error;
+    }
+    try {
+        return await withNativeTerminalStartupLock(socketPath, () => launchClientUnderLock(socketPath, failure), signal => connectExisting(socketPath, true, signal));
+    }
+    catch (error) {
+        // Election expiry is contention, just like publication-lock expiry. The
+        // elected publisher can finish as our acquisition deadline expires. Spend
+        // the existing successor-health budget on read-only adoption; never use
+        // this failure as an absence proof or permission to spawn another host.
+        if (error?.name === "TimeoutError" || publicationContenderObserved(error)) {
+            const successorSignal = AbortSignal.timeout(NATIVE_TERMINAL_HEALTH_TIMEOUT_MS);
+            try {
+                return await readNativeTerminalSuccessor(() => connectExisting(socketPath, true, successorSignal), true);
+            }
+            catch { /* Preserve the original fail-closed election diagnostic. */ }
         }
-        if (absent)
-            return ensureClient(socketPath);
-        throw Object.assign(new Error(`${socketPath}: ${error instanceof Error ? error.message : String(error)}; owner death is unproven. Inspect endpoint permissions and the owning process identity, restore probe access, then retry; do not remove a socket or lock while ownership is unproven.`), { socketPath });
+        throw error;
+    }
+}
+async function launchClientUnderLock(socketPath, error) {
+    let absent = false;
+    let publisherObserved = false;
+    // Launch-only reclamation: inventories never remove the stale inode.
+    try {
+        absent = await proveNativeTerminalEndpointAbsent(socketPath, error, undefined, undefined, true);
+    }
+    catch (proofError) {
+        publisherObserved = publicationContenderObserved(proofError);
+        error = new Error(`${error instanceof Error ? error.message : String(error)}; absence proof failed: ${proofError instanceof Error ? proofError.message : String(proofError)}`);
+    }
+    if (absent)
+        return ensureClient(socketPath);
+    // Publication may have completed while the absence proof held/waited for
+    // its lock. A fresh handshake can adopt that winner without any takeover.
+    try {
+        return await readNativeTerminalSuccessor(() => connectExisting(socketPath), publisherObserved);
     }
+    catch { /* Still uncertain: retain the original proof failure below. */ }
+    throw Object.assign(new Error(`${socketPath}: ${error instanceof Error ? error.message : String(error)}; owner death is unproven. Inspect endpoint permissions and the owning process identity, restore probe access, then retry; do not remove a socket or lock while ownership is unproven.`), { socketPath });
 }
 async function ensureClient(socketPath) {
     const supervisor = new NativeTerminalHostSupervisor({
@@ -375,6 +471,10 @@ export async function runAttach(id, runtime = defaultAttachRuntime(socketPathFro
                     : "controller released or session replaced";
         stdout.write(`\r\n[h2a] ${reason}; attach detached\r\n`);
         exitCode = state.status === "exited" && !state.stopSignal ? state.exit?.exitCode ?? 0 : 0;
+        if (state.status === "exited")
+            appendCrashLog("attach", sessionExitCrashEntry(state, { sessionId: id, detail: reason }));
+        else
+            appendCrashLog("attach", { event: "refusal", sessionId: id, reason: "controller-changed", detail: reason });
         terminalEnded = true;
         return true;
     };
@@ -408,6 +508,7 @@ export async function runAttach(id, runtime = defaultAttachRuntime(socketPathFro
             return;
         if (error instanceof NativeTerminalRemoteError && error.message === "stale terminal controller lease") {
             stdout.write("\r\n[h2a] controller revoked by takeover or stop; attach detached\r\n");
+            appendCrashLog("attach", { event: "refusal", sessionId: id, reason: "stale-controller-lease" });
             terminalEnded = true;
             closeActive();
             return;
@@ -458,6 +559,7 @@ export async function runAttach(id, runtime = defaultAttachRuntime(socketPathFro
         if (pendingInputBytes + chunk.length > ATTACH_PENDING_INPUT_MAX_BYTES) {
             stdout.write("\r\n[h2a] native attach input buffer full during reconnect; detaching " +
                 "without stopping the session\r\n");
+            appendCrashLog("attach", { event: "refusal", sessionId: id, reason: "input-buffer-full" });
             detached = true;
             closeActive();
             return;
@@ -506,9 +608,22 @@ export async function runAttach(id, runtime = defaultAttachRuntime(socketPathFro
         if (isRawCapable)
             stdin.setRawMode(true);
         stdin.resume();
-        const initial = await connectAndAcquire();
-        if (initial.status === "exited")
+        let initial;
+        try {
+            initial = await connectAndAcquire();
+        }
+        catch (error) {
+            // Attaching to a provably absent session is an observed session end,
+            // not a crash without a record. Anything else still throws.
+            if (!isDefinitiveNativeTerminalAbsence(error))
+                throw error;
+            appendCrashLog("attach", sessionExitCrashEntry({ exit: null, stopSignal: null }, { sessionId: id, reason: "definitive-absence", detail: "session absent at attach; exit code unknown" }));
+            return 1;
+        }
+        if (initial.status === "exited") {
+            appendCrashLog("attach", sessionExitCrashEntry(initial, { sessionId: id, detail: "session already exited at attach" }));
             return initial.exit?.exitCode ?? 0;
+        }
         let idlePollMs = ATTACH_IDLE_POLL_MIN_MS;
         let nextStatePollAt = runtime.now() + ATTACH_STATE_POLL_MS;
         for (;;) {
@@ -530,6 +645,7 @@ export async function runAttach(id, runtime = defaultAttachRuntime(socketPathFro
                         const state = await connectAndAcquire();
                         if (state.status === "exited") {
                             exitCode = state.exit?.exitCode ?? 0;
+                            appendCrashLog("attach", sessionExitCrashEntry(state, { sessionId: id, detail: "session exited during reconnect" }));
                             terminalEnded = true;
                             break;
                         }
@@ -543,10 +659,12 @@ export async function runAttach(id, runtime = defaultAttachRuntime(socketPathFro
                     catch (error) {
                         if (error instanceof NativeTerminalRemoteError && error.message.includes("already has a controller")) {
                             stdout.write(`\r\n[h2a] ${error.message}; attach detached\r\n`);
+                            appendCrashLog("attach", { event: "refusal", sessionId: id, reason: "already-has-controller", detail: error.message });
                             terminalEnded = true;
                             break;
                         }
                         if (isDefinitiveNativeTerminalAbsence(error)) {
+                            appendCrashLog("attach", sessionExitCrashEntry({ exit: null, stopSignal: null }, { sessionId: id, reason: "definitive-absence", detail: "session proven absent during reconnect; exit code unknown" }));
                             exitCode = 1;
                             terminalEnded = true;
                             break;
@@ -596,6 +714,7 @@ export async function runAttach(id, runtime = defaultAttachRuntime(socketPathFro
                         break;
                     if (state.status === "exited") {
                         exitCode = state.exit?.exitCode ?? 0;
+                        appendCrashLog("attach", sessionExitCrashEntry(state, { sessionId: id, detail: "session exited during attach" }));
                         break;
                     }
                 }
@@ -605,6 +724,7 @@ export async function runAttach(id, runtime = defaultAttachRuntime(socketPathFro
                     break;
                 if (isDefinitiveNativeTerminalAbsence(error)) {
                     stdout.write("\r\n[h2a] native session is no longer available; attach stopped\r\n");
+                    appendCrashLog("attach", sessionExitCrashEntry({ exit: null, stopSignal: null }, { sessionId: id, reason: "definitive-absence", detail: "session proven absent during attach; exit code unknown" }));
                     exitCode = 1;
                     break;
                 }
@@ -639,7 +759,13 @@ export async function runAttach(id, runtime = defaultAttachRuntime(socketPathFro
     }
     return exitCode;
 }
-export async function runNativeTerminalOp(argv) {
+export async function runNativeTerminalOp(argv, emit = emitResult, signal, deadline = Infinity) {
+    const checkDeadline = () => {
+        signal?.throwIfAborted();
+        if (Date.now() >= deadline)
+            throw new Error("native host operation deadline exceeded");
+    };
+    checkDeadline();
     const parsed = parseArgv(argv);
     const socketPath = parsed.flags.get("socket") ?? socketPathFromEnv();
     switch (parsed.op) {
@@ -684,15 +810,7 @@ export async function runNativeTerminalOp(argv) {
             //  - unknown: an incomplete inventory without a found owner; even
             //             transport errors without an independent owner-death proof.
             const id = required(parsed, "id");
-            const owner = resolveNativeOwner(id, await inventoryFor(parsed));
-            if (owner.state === "found")
-                emit({ verdict: owner.session.status === "running" ? "live" : "dead",
-                    state: { ...owner.session, socketPath: owner.socketPath } });
-            else if (owner.state === "absent")
-                emit({ verdict: "dead" });
-            else
-                emit({ verdict: "unknown", code: owner.state,
-                    ...(owner.state === "unknown" ? { reason: owner.reason } : { sockets: owner.sockets, reason: "ambiguous-owner" }) });
+            emit(await probeNativeSession(id, parsed.flags.get("socket")));
             return 0;
         }
         case "admit": {
@@ -744,6 +862,88 @@ export async function runNativeTerminalOp(argv) {
             emit(state);
             return 0;
         }
+        case "launch": {
+            // R3 P1 single-fork fast path: ensure-host + admit + probe + create
+            // share ONE client and ONE inventory in ONE Node process. Admission
+            // and identity proofs stay under the host lock: an incomplete
+            // inventory still refuses, and a fenced create re-checks admission
+            // atomically like "create" does. Emits the host identity together
+            // with the session state so callers skip their pid re-probes.
+            const fenced = parsed.flags.has("incarnation") || parsed.flags.has("fenced");
+            const selected = parsed.flags.has("socket")
+                ? { client: await connectExisting(socketPath), socketPath }
+                : await selectLaunchClient(fenced);
+            const client = selected.client;
+            const launchSocket = selected.socketPath;
+            try {
+                checkDeadline();
+                const ping = await client.ping();
+                if (fenced && ping.launchFence !== true) {
+                    emit({ code: "native-host-capability-mismatch", socketPath: launchSocket,
+                        hostPid: ping.hostPid, generation: ping.generation, launchFence: false });
+                    return 0;
+                }
+                const inventory = await inventoryFor({ ...parsed, flags: new Map() });
+                const id = required(parsed, "id");
+                const ids = [id, ...(parsed.flags.has("sidecar") ? [required(parsed, "sidecar")]
+                        : !id.endsWith(".h2a") ? [`${id}.h2a`] : [])];
+                const collision = inventory.sessions.find(session => ids.includes(session.id) &&
+                    (session.status !== "exited" || session.socketPath !== launchSocket));
+                if (collision) {
+                    emit({ code: "native-name-collision", id: collision.id, socketPath: collision.socketPath,
+                        hostPid: ping.hostPid, generation: ping.generation, launchFence: ping.launchFence === true });
+                    return 0;
+                }
+                if (!inventory.complete) {
+                    emit({ code: "native-inventory-unknown",
+                        hosts: inventory.hosts.map(({ socketPath: p, reason }) => ({ socketPath: p, reason })),
+                        hostPid: ping.hostPid, generation: ping.generation, launchFence: ping.launchFence === true });
+                    return 0;
+                }
+                const existing = inventory.sessions.find(session => session.id === id && session.status !== "exited" &&
+                    session.socketPath === launchSocket);
+                if (existing && parsed.flags.get("refuse-existing") !== "true") {
+                    const { socketPath: _droppedSocket, ...existingRest } = existing;
+                    emit({ ...existingRest, hostPid: ping.hostPid, socketPath: launchSocket, generation: ping.generation,
+                        launchFence: ping.launchFence === true, adopted: true });
+                    return 0;
+                }
+                if (existing && parsed.flags.get("refuse-existing") === "true") {
+                    emit({ code: "native-name-collision", id: existing.id, socketPath: launchSocket,
+                        hostPid: ping.hostPid, generation: ping.generation, launchFence: ping.launchFence === true });
+                    return 0;
+                }
+                const cwd = required(parsed, "cwd");
+                const cols = Number(parsed.flags.get("cols") ?? 160);
+                const rows = Number(parsed.flags.get("rows") ?? 48);
+                const envFile = parsed.flags.get("env-file");
+                const env = envFile !== undefined
+                    ? JSON.parse(readFileSync(envFile, "utf8"))
+                    : Object.fromEntries(Object.entries(process.env).filter((entry) => entry[1] !== undefined));
+                if (parsed.command.length === 0)
+                    throw new Error("launch requires -- command");
+                checkDeadline();
+                const state = await client.create({
+                    ...(parsed.flags.has("incarnation") ? { launchFence: {
+                            generation: required(parsed, "generation"), incarnation: required(parsed, "incarnation"),
+                        } } : {}),
+                    id,
+                    command: parsed.command[0],
+                    args: parsed.command.slice(1),
+                    cwd,
+                    env,
+                    cols,
+                    rows,
+                });
+                const { socketPath: _droppedStateSocket, generation: _droppedStateGeneration, ...stateRest } = state;
+                emit({ ...stateRest, hostPid: ping.hostPid, socketPath: launchSocket, generation: ping.generation,
+                    launchFence: ping.launchFence === true });
+                return 0;
+            }
+            finally {
+                client.close();
+            }
+        }
         case "drive": {
             const target = parsed.flags.has("id")
                 ? { state: "found", id: required(parsed, "id") } : nativeDriveSessionId(required(parsed, "target"));
@@ -853,10 +1053,14 @@ export async function runNativeTerminalOp(argv) {
             const fenced = parsed.op.endsWith("-if-incarnation");
             const generation = fenced ? required(parsed, "generation") : before.generation;
             const incarnation = fenced ? required(parsed, "incarnation") : before.incarnation;
-            if (before.generation !== generation)
+            if (before.generation !== generation) {
+                appendCrashLog("cli", { event: "refusal", sessionId: id, instanceId: generation, reason: "stale-terminal-host-generation" });
                 throw new Error("stale terminal host generation");
-            if (before.incarnation !== incarnation)
+            }
+            if (before.incarnation !== incarnation) {
+                appendCrashLog("cli", { event: "refusal", sessionId: id, instanceId: generation, reason: "stale-terminal-session-incarnation" });
                 throw new Error("stale terminal session incarnation");
+            }
             if (before.status !== "exited") {
                 await client.stopIfIncarnation(id, generation, incarnation, signal);
                 const deadline = Date.now() + 4_000;
@@ -878,17 +1082,28 @@ export async function runNativeTerminalOp(argv) {
             const after = await client.state(id);
             client.close();
             if (after.generation !== generation || after.incarnation !== incarnation) {
+                appendCrashLog("cli", { event: "refusal", sessionId: id, instanceId: generation, reason: "incarnation-changed-while-stopping" });
                 throw new Error("native session changed incarnation while stopping; replacement was not stopped");
             }
             if (after.status !== "exited") {
+                appendCrashLog("cli", { event: "exit", sessionId: id, instanceId: generation, reason: "kill-escalation-failed", detail: `status=${after.status}` });
                 throw new Error(`native session ${id} pid=${after.pid} did not exit after ${signal} and SIGKILL (status=${after.status})`);
             }
+            appendCrashLog("cli", { event: "sessionExit", sessionId: id, instanceId: generation, reason: `killed-${signal}`, signal });
             emit(after);
             return 0;
         }
         case "attach": {
             const id = required(parsed, "id");
-            const owner = await owningClient(parsed, id);
+            let owner;
+            try {
+                owner = await owningClient(parsed, id);
+            }
+            catch (error) {
+                const refusal = error instanceof NativeOwnerError ? error.refusal : undefined;
+                appendCrashLog("attach", { event: "refusal", sessionId: id, reason: refusal?.code ?? "owner-unproven", detail: refusal?.reason });
+                throw error;
+            }
             owner.client.close();
             // Reconnects retain this owner for the entire interactive operation.
             return runAttach(id, defaultAttachRuntime(owner.socketPath), { takeover: parsed.flags.get("takeover") === "true" });
@@ -926,12 +1141,13 @@ function isEntryPoint() {
     }
 }
 if (isEntryPoint()) {
+    installCrashHandlers("cli");
     runNativeTerminalOp(process.argv)
         .then((code) => {
         process.exitCode = code;
     }, (error) => {
         if (error instanceof NativeOwnerError)
-            emit(error.refusal);
+            emitResult(error.refusal);
         process.stderr.write(`[h2a-native-op] ${error instanceof Error ? error.message : String(error)}\n`);
         process.exitCode = 1;
     })