@aztec-foundation/ipc-runtime 0.0.1-commit.b66364b

package/dest/spawned_backend.js

@@ -0,0 +1,342 @@
+import { spawn } from "node:child_process";
+import { open, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { threadId } from "node:worker_threads";
+import { IpcError, IpcProcessExitedError, IpcSpawnError } from "./errors.js";
+import { createNapiShmAsyncClient } from "./shm_client.js";
+import { UdsIpcClient } from "./uds_client.js";
+// Backstop for a spawned server that is alive but never reaches listen().
+// This is a broken-process detector, not a performance expectation: servers
+// create their socket before any heavy initialization, so the timed window
+// covers only exec + linking + minimal init, and requests issued before the
+// server is fully initialized simply wait in the socket buffer. When the
+// backstop fires, the wedged process is killed rather than orphaned.
+const DEFAULT_CONNECT_TIMEOUT_MS = 60_000;
+// After destroy() sends SIGTERM, how long to wait before escalating to
+// SIGKILL so teardown cannot hang on a server that is stuck before its signal
+// handlers were installed (or is wedged inside them).
+const SIGTERM_GRACE_MS = 5_000;
+// How long a failed call() waits for the child's 'exit' event before deciding
+// the process is still alive. The socket usually breaks before 'exit' lands,
+// so without this grace a death would be misreported as a bare transport error.
+const EXIT_ATTRIBUTION_GRACE_MS = 250;
+let instanceCounter = 0;
+/**
+ * An IpcClientAsync backed by a spawned server process. Owns the process
+ * lifecycle end to end: spawn, connect (raced against child death, with a
+ * kill-on-expiry backstop), death detection, optional lazy respawn, and
+ * teardown with SIGTERM→SIGKILL escalation. Failures surface as IpcError
+ * subclasses whose `retry` property tells callers whether the operation may
+ * be retried; no process or transport state is exposed on the API.
+ */
+export class SpawnedProcessBackend {
+    options;
+    current;
+    starting;
+    destroying = false;
+    /** Set when the process died and respawn is disabled; fails all later calls. */
+    exitError;
+    respawn;
+    ipcPath;
+    logPath;
+    constructor(options) {
+        this.options = options;
+        this.respawn = options.respawn ?? false;
+        const instanceId = `${options.instancePrefix}-${process.pid}-${threadId}-${instanceCounter++}`;
+        // The ipc path is per-backend, not per-incarnation, so getIpcPath() stays
+        // stable across respawns for anyone who was handed the path.
+        this.ipcPath =
+            options.transport === "shm"
+                ? `${instanceId}.shm`
+                : join(tmpdir(), `${instanceId}.sock`);
+        // Same for the log file: respawns append, preserving the death's last words.
+        this.logPath = options.logger
+            ? undefined
+            : join(tmpdir(), `${instanceId}.log`);
+    }
+    static async spawn(options) {
+        if (options.respawn && options.transport === "shm") {
+            throw new IpcError(`respawn is not supported over the shm transport`, 
+            /*retry=*/ false);
+        }
+        const backend = new SpawnedProcessBackend(options);
+        backend.current = await backend.spawnIncarnation();
+        return backend;
+    }
+    getIpcPath() {
+        return this.ipcPath;
+    }
+    async call(input) {
+        const incarnation = await this.ensureUp();
+        try {
+            return await incarnation.client.call(input);
+        }
+        catch (err) {
+            throw await this.attributeCallError(incarnation, err);
+        }
+    }
+    sendProcessSignal(signal) {
+        const child = this.current?.child;
+        if (child && child.exitCode === null && child.signalCode === null) {
+            child.kill(signal);
+        }
+    }
+    async destroy() {
+        // Mark intentional teardown so the exit handler doesn't report it as an
+        // unexpected death (or trigger a respawn).
+        this.destroying = true;
+        // A respawn may be mid-flight; let it settle so its child can't leak.
+        await this.starting?.then((incarnation) => {
+            this.current = incarnation;
+        }, () => { });
+        const incarnation = this.current;
+        this.current = undefined;
+        if (!incarnation) {
+            return;
+        }
+        await incarnation.client.destroy();
+        const { child } = incarnation;
+        let killTimer;
+        if (child.exitCode === null && child.signalCode === null) {
+            child.kill("SIGTERM");
+            killTimer = setTimeout(() => {
+                if (child.exitCode === null && child.signalCode === null) {
+                    child.kill("SIGKILL");
+                }
+            }, SIGTERM_GRACE_MS);
+        }
+        await incarnation.exitPromise;
+        if (killTimer !== undefined) {
+            clearTimeout(killTimer);
+        }
+        child.stdout?.destroy();
+        child.stderr?.destroy();
+        child.removeAllListeners();
+        await this.cleanupIpcPath();
+    }
+    /** Return the live incarnation, lazily respawning one when allowed. */
+    async ensureUp() {
+        if (this.destroying) {
+            throw new IpcError(`${this.options.binaryName} backend destroyed`, 
+            /*retry=*/ false);
+        }
+        if (this.current) {
+            return this.current;
+        }
+        if (this.exitError) {
+            throw this.exitError;
+        }
+        // Lazy, shared respawn: the first caller after a death starts it, everyone
+        // else awaits the same attempt. Lazy (rather than eager-on-exit) so a
+        // crashing binary cannot respawn-loop with no one asking for it.
+        this.starting ??= this.spawnIncarnation()
+            .then((incarnation) => {
+            if (this.destroying) {
+                // destroy() raced us and already awaited this promise; it owns teardown.
+                return incarnation;
+            }
+            this.current = incarnation;
+            return incarnation;
+        })
+            .finally(() => {
+            this.starting = undefined;
+        });
+        return await this.starting;
+    }
+    /**
+     * Convert a failed call into the death of the process when that is what
+     * actually happened: the socket breaks before the child's 'exit' event
+     * lands, so wait a short grace for exit attribution before giving up and
+     * rethrowing the transport error as-is.
+     */
+    async attributeCallError(incarnation, err) {
+        if (!incarnation.exitInfo && !this.destroying) {
+            await Promise.race([
+                incarnation.exitPromise,
+                new Promise((resolve) => setTimeout(resolve, EXIT_ATTRIBUTION_GRACE_MS)),
+            ]);
+        }
+        if (incarnation.exitInfo && !this.destroying) {
+            const { code, signal } = incarnation.exitInfo;
+            return new IpcProcessExitedError(`${this.options.binaryName} exited unexpectedly (code=${code}, signal=${signal})` +
+                (this.logPath !== undefined ? `; see logs: ${this.logPath}` : ""), code, signal, this.logPath);
+        }
+        return err;
+    }
+    /** Spawn the server process and connect to it; kills the child on any failure. */
+    async spawnIncarnation() {
+        const { options } = this;
+        if (options.transport === "uds") {
+            await rm(this.ipcPath, { force: true });
+        }
+        // Without a live logger, capture the child's stdout/stderr to the backend's
+        // log file (a plain fd, not a pipe — a pipe would keep the libuv loop
+        // referenced and break clean process exit). The path is surfaced on
+        // failures so the child's errors are recoverable instead of vanishing.
+        // Async fs throughout: this runs on respawn under exactly the machine
+        // conditions where a sync open against a struggling disk could stall the
+        // whole event loop.
+        const logFile = this.logPath !== undefined ? await open(this.logPath, "a") : undefined;
+        const child = spawn(options.binaryPath, [
+            ...options.ipcPathArgs.map((arg) => arg === "{path}" ? this.ipcPath : arg),
+            ...(options.extraArgs ?? []),
+        ], {
+            stdio: [
+                "ignore",
+                options.logger ? "pipe" : logFile.fd,
+                options.logger ? "pipe" : logFile.fd,
+            ],
+            env: { ...process.env, ...(options.env ?? {}) },
+        });
+        if (options.logger) {
+            child.stdout?.on("data", (data) => options.logger?.(`[${options.binaryName} stdout] ${data.toString().trimEnd()}`));
+            child.stderr?.on("data", (data) => options.logger?.(`[${options.binaryName} stderr] ${data.toString().trimEnd()}`));
+        }
+        const incarnation = {
+            child,
+        };
+        const exitPromise = new Promise((resolve) => {
+            child.on("exit", (code, signal) => {
+                incarnation.exitInfo = { code, signal };
+                this.onIncarnationExit(incarnation, code, signal);
+                resolve();
+            });
+        });
+        incarnation.exitPromise = exitPromise;
+        const childReadyFailure = new Promise((_, reject) => {
+            // Spawn syscall failures are the one place errno distinguishes a
+            // configuration error (missing/non-executable binary → retrying cannot
+            // help) from an environmental one.
+            child.once("error", (err) => {
+                const code = err.code;
+                const retry = code !== "ENOENT" && code !== "EACCES";
+                reject(new IpcSpawnError(`Failed to spawn ${options.binaryName}: ${err.message}`, retry, { cause: err }));
+            });
+            child.once("exit", (code, signal) => {
+                reject(new IpcSpawnError(`${options.binaryName} exited before IPC connection was ready (code=${code}, signal=${signal})`, 
+                /*retry=*/ true));
+            });
+        });
+        // Observe immediately: childReadyFailure can reject during the awaits
+        // below (spawn failures land on nextTick), before Promise.race attaches
+        // its handler — without this it would count as an unhandled rejection.
+        childReadyFailure.catch(() => { });
+        if (logFile !== undefined) {
+            // spawn() dups the fd synchronously; the parent's handle isn't needed.
+            // Closed only now, after the 'error'/'exit' listeners are attached: spawn
+            // failures are emitted on nextTick, so an await between spawn() and the
+            // listeners would let them fire unhandled.
+            await logFile.close();
+        }
+        // Liveness-based startup: wait on the connect for as long as the process
+        // is alive (the connect path retries socket-not-ready errors internally),
+        // and fail immediately with the real cause if the process dies first. On
+        // any failure, reap the child and its ipc path so a failed spawn cannot
+        // leak an orphan process still holding sockets or database locks.
+        try {
+            incarnation.client = await Promise.race([
+                this.connectClient(),
+                childReadyFailure,
+            ]);
+        }
+        catch (err) {
+            if (child.pid !== undefined) {
+                // SIGKILL, not SIGTERM: the process never became ready, so it has no
+                // state to flush, and a wedged process may not honour SIGTERM.
+                if (child.exitCode === null && child.signalCode === null) {
+                    child.kill("SIGKILL");
+                }
+                await exitPromise;
+            }
+            child.stdout?.destroy();
+            child.stderr?.destroy();
+            child.removeAllListeners();
+            await this.cleanupIpcPath();
+            throw this.asSpawnError(err);
+        }
+        return incarnation;
+    }
+    /** Handles an incarnation's death: reject/replace state so later calls behave per the respawn policy. */
+    onIncarnationExit(incarnation, code, signal) {
+        // Break the connection so in-flight calls reject rather than wait forever
+        // (matters over SHM, where there is no socket to break).
+        void incarnation.client?.destroy();
+        if (this.destroying || this.current !== incarnation) {
+            return;
+        }
+        this.current = undefined;
+        if (!this.respawn) {
+            this.exitError = new IpcProcessExitedError(`${this.options.binaryName} exited unexpectedly (code=${code}, signal=${signal})` +
+                (this.logPath !== undefined ? `; see logs: ${this.logPath}` : ""), code, signal, this.logPath);
+            console.error(this.exitError.message);
+        }
+        else {
+            console.error(`${this.options.binaryName} exited unexpectedly (code=${code}, signal=${signal}); will respawn on next call`);
+        }
+    }
+    async connectClient() {
+        const { options } = this;
+        const timeoutMs = options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS;
+        if (options.transport === "uds") {
+            // UdsIpcClient.connect retries socket-not-ready errors (path not yet
+            // created, server not yet accepting, backlog momentarily full) until the
+            // budget expires, and fails immediately on hard errors. Process death is
+            // raced against this by the caller, so a dead server short-circuits the
+            // wait with its real exit cause.
+            return await UdsIpcClient.connect(this.ipcPath, {
+                connectTimeoutMs: timeoutMs,
+            });
+        }
+        // The SHM client attaches to server-created rings, so creation can race
+        // server startup; retry until the backstop expires.
+        const deadline = Date.now() + timeoutMs;
+        let lastError;
+        while (Date.now() <= deadline) {
+            try {
+                return createNapiShmAsyncClient(this.ipcPath.replace(/\.shm$/, ""), {
+                    clientId: options.clientId,
+                    customAddonPath: options.napiPath,
+                });
+            }
+            catch (err) {
+                lastError = err;
+                await new Promise((resolve) => setTimeout(resolve, 50));
+            }
+        }
+        const message = lastError instanceof Error ? lastError.message : String(lastError);
+        throw new IpcSpawnError(`Timed out connecting to ${this.options.binaryName}: ${message}`, 
+        /*retry=*/ true, { cause: lastError });
+    }
+    /** Wrap a spawn/connect failure as IpcSpawnError and point at the captured log. */
+    asSpawnError(err) {
+        const logHint = this.logPath !== undefined ? `; see logs: ${this.logPath}` : "";
+        if (err instanceof IpcSpawnError) {
+            // Already classified at its source (child spawn 'error', exit-before-ready).
+            return new IpcSpawnError(err.message + logHint, err.retry, {
+                cause: err.cause ?? err,
+            });
+        }
+        // Everything else reaching here (connect backstop expiry, transient
+        // transport errors) is environmental.
+        const message = err instanceof Error ? err.message : String(err);
+        return new IpcSpawnError(`Failed to start ${this.options.binaryName}: ${message}${logHint}`, 
+        /*retry=*/ true, { cause: err });
+    }
+    async cleanupIpcPath() {
+        try {
+            if (this.options.transport === "uds") {
+                await rm(this.ipcPath, { force: true });
+            }
+            if (this.options.transport === "shm") {
+                const shmName = this.ipcPath.replace(/\.shm$/, "");
+                for (const suffix of ["_request", "_response"]) {
+                    await rm(`/dev/shm/${shmName}${suffix}`, { force: true });
+                }
+            }
+        }
+        catch {
+            // Cleanup is best-effort; the paths live under tmpdir.
+        }
+    }
+}

package/dest/spawned_backend.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dest/spawned_backend.test.js

@@ -0,0 +1,173 @@
+import assert from "node:assert/strict";
+import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { after, test } from "node:test";
+import { IpcProcessExitedError, IpcSpawnError } from "./errors.js";
+import { SpawnedProcessBackend } from "./spawned_backend.js";
+const scratch = mkdtempSync(join(tmpdir(), "spawned-backend-test-"));
+after(() => rmSync(scratch, { recursive: true, force: true }));
+// A minimal length-prefixed echo server speaking the UdsIpcClient wire format.
+// Replies to each request with its own pid as a decimal string, so tests can
+// tell process incarnations apart.
+const PID_ECHO_SERVER = join(scratch, "pid_echo_server.cjs");
+writeFileSync(PID_ECHO_SERVER, `
+const net = require('node:net');
+const server = net.createServer(conn => {
+  let buf = Buffer.alloc(0);
+  conn.on('data', chunk => {
+    buf = Buffer.concat([buf, chunk]);
+    while (buf.length >= 4) {
+      const len = buf.readUInt32LE(0);
+      if (buf.length < 4 + len) return;
+      const requestId = buf.readBigUInt64LE(4); // frame = [len][8B id][payload]
+      buf = buf.subarray(4 + len);
+      const payload = Buffer.from(String(process.pid));
+      const out = Buffer.alloc(12 + payload.length);
+      out.writeUInt32LE(payload.length + 8, 0);
+      out.writeBigUInt64LE(requestId, 4);
+      payload.copy(out, 12);
+      conn.write(out);
+    }
+  });
+});
+server.listen(process.argv[2]);
+`);
+function shellScript(name, body) {
+    const path = join(scratch, name);
+    writeFileSync(path, `#!/bin/sh\n${body}\n`);
+    chmodSync(path, 0o755);
+    return path;
+}
+function spawnPidEcho(opts = {}) {
+    return SpawnedProcessBackend.spawn({
+        binaryPath: process.execPath,
+        binaryName: "pid_echo_server",
+        instancePrefix: "pid-echo-test",
+        ipcPathArgs: [PID_ECHO_SERVER, "{path}"],
+        transport: "uds",
+        ...opts,
+    });
+}
+async function callPid(backend) {
+    return Buffer.from(await backend.call(Uint8Array.from([1]))).toString();
+}
+function processAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+async function waitFor(cond, timeoutMs) {
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        if (cond())
+            return true;
+        await new Promise((resolve) => setTimeout(resolve, 25));
+    }
+    return cond();
+}
+test("spawns, round-trips calls, and destroys cleanly", async () => {
+    const backend = await spawnPidEcho();
+    const pid = await callPid(backend);
+    assert.match(pid, /^\d+$/);
+    await backend.destroy();
+    assert.equal(processAlive(parseInt(pid, 10)), false);
+});
+test("without respawn, death rejects in-flight and later calls with a retryable exit error", async () => {
+    const backend = await spawnPidEcho();
+    const pid = parseInt(await callPid(backend), 10);
+    process.kill(pid, "SIGKILL");
+    await waitFor(() => !processAlive(pid), 2_000);
+    const err = await callPid(backend).then(() => undefined, (e) => e);
+    assert.ok(err instanceof IpcProcessExitedError, `expected IpcProcessExitedError, got ${err}`);
+    assert.equal(err.retry, true);
+    assert.equal(err.signal, "SIGKILL");
+    // Still down: same classified error, no zombie respawn.
+    await assert.rejects(callPid(backend), IpcProcessExitedError);
+    await backend.destroy();
+});
+test("with respawn, the next call after a death transparently gets a fresh process", async () => {
+    const backend = await spawnPidEcho({ respawn: true });
+    const firstPid = parseInt(await callPid(backend), 10);
+    process.kill(firstPid, "SIGKILL");
+    await waitFor(() => !processAlive(firstPid), 2_000);
+    // The first call after the death may race the exit event; it is allowed to
+    // fail (with a retryable error) or succeed against the respawned process.
+    let secondPid;
+    try {
+        secondPid = parseInt(await callPid(backend), 10);
+    }
+    catch (err) {
+        assert.equal(err.retry, true);
+        secondPid = parseInt(await callPid(backend), 10);
+    }
+    assert.notEqual(secondPid, firstPid);
+    assert.equal(processAlive(secondPid), true);
+    // getIpcPath is stable across incarnations.
+    const path = backend.getIpcPath();
+    await backend.destroy();
+    assert.equal(processAlive(secondPid), false);
+    assert.equal(backend.getIpcPath(), path);
+});
+test("a missing binary fails with retry=false", async () => {
+    const err = await SpawnedProcessBackend.spawn({
+        binaryPath: join(scratch, "does_not_exist"),
+        binaryName: "ghost",
+        instancePrefix: "ghost-test",
+        ipcPathArgs: ["{path}"],
+        transport: "uds",
+    }).then(() => undefined, (e) => e);
+    assert.ok(err instanceof IpcSpawnError, `expected IpcSpawnError, got ${err}`);
+    assert.equal(err.retry, false);
+});
+test("a server dying before listen fails promptly with retry=true and the exit code", async () => {
+    const dying = shellScript("dying_server.sh", "exit 7");
+    const started = Date.now();
+    const err = await SpawnedProcessBackend.spawn({
+        binaryPath: dying,
+        binaryName: "dying_server",
+        instancePrefix: "dying-test",
+        ipcPathArgs: ["{path}"],
+        transport: "uds",
+    }).then(() => undefined, (e) => e);
+    assert.ok(err instanceof IpcSpawnError, `expected IpcSpawnError, got ${err}`);
+    assert.equal(err.retry, true);
+    assert.match(err.message, /code=7/);
+    assert.ok(Date.now() - started < 5_000, "failed before the connect backstop");
+});
+test("a wedged server is killed when the connect backstop fires", async () => {
+    const pidFile = join(scratch, "wedged.pid");
+    const wedged = shellScript("wedged_server.sh", `echo $$ > "${pidFile}"\nexec sleep 600`);
+    const err = await SpawnedProcessBackend.spawn({
+        binaryPath: wedged,
+        binaryName: "wedged_server",
+        instancePrefix: "wedged-test",
+        ipcPathArgs: ["{path}"],
+        transport: "uds",
+        connectTimeoutMs: 1_000,
+    }).then(() => undefined, (e) => e);
+    assert.ok(err instanceof IpcSpawnError, `expected IpcSpawnError, got ${err}`);
+    assert.equal(err.retry, true);
+    const { readFileSync } = await import("node:fs");
+    const pid = parseInt(readFileSync(pidFile, "utf-8").trim(), 10);
+    assert.equal(await waitFor(() => !processAlive(pid), 5_000), true, "wedged process was reaped");
+});
+test("destroy during a pending respawn does not leak the fresh process", async () => {
+    const backend = await spawnPidEcho({ respawn: true });
+    const firstPid = parseInt(await callPid(backend), 10);
+    process.kill(firstPid, "SIGKILL");
+    await waitFor(() => !processAlive(firstPid), 2_000);
+    // Trigger the lazy respawn, then destroy while it may still be in flight.
+    const pending = callPid(backend).catch(() => undefined);
+    await backend.destroy();
+    const result = await pending;
+    if (result !== undefined) {
+        // The respawn won the race and served the call; destroy() must still have
+        // reaped the fresh process afterwards.
+        assert.equal(await waitFor(() => !processAlive(parseInt(result, 10)), 2_000), true);
+    }
+});

package/dest/types.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Minimal byte-in / byte-out interface that the ipc-codegen-emitted
+ * <Service>Api types consume. Both UDS and SHM transports satisfy this.
+ */
+export interface IpcClientAsync {
+    call(input: Uint8Array): Promise<Uint8Array>;
+    destroy(): Promise<void>;
+}
+export interface IpcClientSync {
+    call(input: Uint8Array): Uint8Array;
+    destroy(): void;
+}
+/**
+ * Maximum length-prefix value accepted on receive. A frame claiming more
+ * than this is treated as corruption and the connection is closed instead
+ * of buffering the claimed size.
+ */
+export declare const MAX_FRAME_SIZE: number;
+/**
+ * Total budget (ms) for connect() retry loops, covering the window where
+ * the server process is still starting up.
+ */
+export declare const CONNECT_RETRY_BUDGET_MS = 5000;
+/** Default ring size for SHM transports (per direction, per client). */
+export declare const DEFAULT_RING_SIZE: number;
+/** Default listen backlog for UDS servers. */
+export declare const SOCKET_BACKLOG = 10;
+/** Default per-call timeout: 0 = infinite (matches the C++ client APIs). */
+export declare const DEFAULT_CALL_TIMEOUT_NS = 0;

package/dest/types.js

@@ -0,0 +1,19 @@
+// Shared transport constants, mirroring cpp/ipc_runtime/constants.hpp —
+// keep the two in sync.
+/**
+ * Maximum length-prefix value accepted on receive. A frame claiming more
+ * than this is treated as corruption and the connection is closed instead
+ * of buffering the claimed size.
+ */
+export const MAX_FRAME_SIZE = 256 * 1024 * 1024; // 256 MiB
+/**
+ * Total budget (ms) for connect() retry loops, covering the window where
+ * the server process is still starting up.
+ */
+export const CONNECT_RETRY_BUDGET_MS = 5000;
+/** Default ring size for SHM transports (per direction, per client). */
+export const DEFAULT_RING_SIZE = 4 * 1024 * 1024; // 4 MiB
+/** Default listen backlog for UDS servers. */
+export const SOCKET_BACKLOG = 10;
+/** Default per-call timeout: 0 = infinite (matches the C++ client APIs). */
+export const DEFAULT_CALL_TIMEOUT_NS = 0;

package/dest/uds.test.d.ts

@@ -0,0 +1 @@
+export {};