@aztec-foundation/ipc-runtime 0.0.1-commit.b66364b

package/src/native_loader.ts

@@ -0,0 +1,97 @@
+// Locate the prebuilt ipc_runtime_napi.node addon shipped with this package.
+//
+// The addon is built by `ipc-runtime/bootstrap.sh` (CMake target
+// `ipc_runtime_napi`) and copied into `build/<arch>-<os>/` next to this
+// package's `package.json`. Resolution walks up from this file's URL to the
+// first `package.json` adjacent to a `build/` directory — that's the
+// package root in both `file:`-linked and published consumption.
+
+import { createRequire } from "node:module";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { fileURLToPath } from "node:url";
+
+export type Platform =
+  | "x86_64-linux"
+  | "x86_64-darwin"
+  | "aarch64-linux"
+  | "aarch64-darwin";
+
+const PLATFORM_TO_BUILD_DIR: Record<Platform, string> = {
+  "x86_64-linux": "amd64-linux",
+  "x86_64-darwin": "amd64-macos",
+  "aarch64-linux": "arm64-linux",
+  "aarch64-darwin": "arm64-macos",
+};
+
+function detectPlatform(): Platform | null {
+  const arch = process.arch;
+  const platform = process.platform;
+  if (arch === "x64" && platform === "linux") return "x86_64-linux";
+  if (arch === "x64" && platform === "darwin") return "x86_64-darwin";
+  if (arch === "arm64" && platform === "linux") return "aarch64-linux";
+  if (arch === "arm64" && platform === "darwin") return "aarch64-darwin";
+  return null;
+}
+
+function findPackageRoot(): string | null {
+  // `import.meta.url` after tsc compile points at the .js file under
+  // <pkg>/dest/...; climb until we find package.json with a sibling build/.
+  let currentDir = path.dirname(fileURLToPath(import.meta.url));
+  const root = path.parse(currentDir).root;
+  while (currentDir !== root) {
+    const packageJsonPath = path.join(currentDir, "package.json");
+    if (fs.existsSync(packageJsonPath)) {
+      const buildDir = path.join(currentDir, "build");
+      if (fs.existsSync(buildDir)) {
+        return currentDir;
+      }
+    }
+    currentDir = path.dirname(currentDir);
+  }
+  return null;
+}
+
+/**
+ * Resolve the path of `ipc_runtime_napi.node` for the current platform.
+ * Returns null if either the platform is unsupported or the artifact is
+ * absent (typical reason: ipc-runtime/bootstrap.sh hasn't run yet).
+ */
+export function findIpcRuntimeNapi(customPath?: string): string | null {
+  if (customPath) {
+    return fs.existsSync(customPath) ? path.resolve(customPath) : null;
+  }
+  const platform = detectPlatform();
+  if (!platform) return null;
+  const packageRoot = findPackageRoot();
+  if (!packageRoot) return null;
+  const buildDir = PLATFORM_TO_BUILD_DIR[platform];
+  const candidate = path.join(
+    packageRoot,
+    "build",
+    buildDir,
+    "ipc_runtime_napi.node",
+  );
+  return fs.existsSync(candidate) ? candidate : null;
+}
+
+/**
+ * Load `ipc_runtime_napi.node` and return its native exports
+ * (`MsgpackClient`, `MsgpackClientAsync`). Throws a descriptive error when
+ * the addon cannot be located or fails to dlopen.
+ */
+export function loadIpcRuntimeNapi(customPath?: string): {
+  MsgpackClient: new (shmName: string, clientId?: number) => any;
+  MsgpackClientAsync: new (shmName: string, clientId?: number) => any;
+} {
+  const addonPath = findIpcRuntimeNapi(customPath);
+  if (!addonPath) {
+    throw new Error(
+      "Could not locate ipc_runtime_napi.node. Build with `ipc-runtime/bootstrap.sh` " +
+        "or set the optional `customPath` argument to point at a prebuilt addon.",
+    );
+  }
+  // createRequire so this works in both ESM and CJS callers.
+  const require = createRequire(import.meta.url);
+  return require(addonPath);
+}

package/src/shm_client.test.ts

@@ -0,0 +1,72 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { NapiShmAsyncClient, NapiMsgpackClientAsync } from "./shm_client.js";
+
+/**
+ * Drives NapiShmAsyncClient through a mock addon, so the id-pairing logic is
+ * testable without the native module or a live server.
+ */
+class MockAddon implements NapiMsgpackClientAsync {
+  public deliver!: (requestId: bigint, response: Buffer) => void;
+  public sent: Array<{ requestId: bigint; input: Buffer }> = [];
+  public acquires = 0;
+  public releases = 0;
+  public closed = false;
+
+  setResponseCallback(cb: (requestId: bigint, response: Buffer) => void): void {
+    this.deliver = cb;
+  }
+  call(requestId: bigint, input: Buffer): void {
+    this.sent.push({ requestId, input });
+  }
+  acquire(): void {
+    this.acquires++;
+  }
+  release(): void {
+    this.releases++;
+  }
+  close(): void {
+    this.closed = true;
+  }
+}
+
+test("shm async client discards a stale frame and still resolves the live call", async () => {
+  const addon = new MockAddon();
+  const client = new NapiShmAsyncClient(addon);
+
+  const pending = client.call(new Uint8Array([1, 2, 3]));
+  assert.equal(addon.sent.length, 1);
+  const liveId = addon.sent[0].requestId;
+
+  // A leftover frame from a ring's previous occupant: unknown id. Must be
+  // discarded — not resolve the live call, not reject anything.
+  addon.deliver(liveId ^ 0xdeadbeefn, Buffer.from([0xba, 0xad]));
+
+  // The real response still pairs and resolves.
+  addon.deliver(liveId, Buffer.from([9, 9]));
+  assert.deepEqual(await pending, new Uint8Array([9, 9]));
+
+  // Refcount stayed balanced: one acquire for the call, one release when the
+  // live response drained the map (the stale frame must not release).
+  assert.equal(addon.acquires, 1);
+  assert.equal(addon.releases, 1);
+
+  await client.destroy();
+});
+
+test("shm async client pairs out-of-order responses to the right callers", async () => {
+  const addon = new MockAddon();
+  const client = new NapiShmAsyncClient(addon);
+
+  const a = client.call(new Uint8Array([0xaa]));
+  const b = client.call(new Uint8Array([0xbb]));
+  const [idA, idB] = addon.sent.map((s) => s.requestId);
+
+  // Complete in reverse order; each caller must get its own payload.
+  addon.deliver(idB, Buffer.from([2]));
+  addon.deliver(idA, Buffer.from([1]));
+  assert.deepEqual(await b, new Uint8Array([2]));
+  assert.deepEqual(await a, new Uint8Array([1]));
+
+  await client.destroy();
+});

package/src/shm_client.ts

@@ -0,0 +1,194 @@
+import { loadIpcRuntimeNapi } from "./native_loader.js";
+import { IpcClientAsync, IpcClientSync } from "./types.js";
+
+/**
+ * Minimum surface a NAPI msgpack client must expose. Satisfied by the
+ * `MsgpackClient` / `MsgpackClientAsync` classes exported from this
+ * package's own `ipc_runtime_napi.node` addon (see ipc-runtime/cpp/napi/),
+ * which wraps the C++ ipc::IpcClient.
+ *
+ * The interface is exposed for tests / consumers that want to inject a
+ * mock or alternative implementation; the standard production path is the
+ * `createNapiShm{Sync,Async}Client` factories below, which load the
+ * prebuilt addon shipped in this package's `build/<arch>-<os>/` directory.
+ *
+ * Note on the async contract: `MsgpackClientAsync.call` is *fire and
+ * forget*. Responses arrive via `setResponseCallback` in COMPLETION order on
+ * a background-thread → main-thread bridge (Napi::ThreadSafeFunction), each
+ * carrying its echoed request id. The TS wrapper below owns the pending map
+ * and pairs responses to callers by id.
+ */
+export interface NapiMsgpackClientSync {
+  call(input: Buffer): Buffer;
+  close(): void;
+}
+
+export interface NapiMsgpackClientAsync {
+  setResponseCallback(cb: (requestId: bigint, response: Buffer) => void): void;
+  call(requestId: bigint, input: Buffer): void;
+  acquire(): void;
+  release(): void;
+  /** Stop the native poll thread, release any held TSFN ref, close the client. */
+  close(): void;
+}
+
+/** Wraps a sync NAPI msgpack client behind the IpcClientSync interface. */
+export class NapiShmSyncClient implements IpcClientSync {
+  constructor(private inner: NapiMsgpackClientSync) {}
+
+  call(input: Uint8Array): Uint8Array {
+    const buf = Buffer.isBuffer(input)
+      ? input
+      : Buffer.from(input.buffer, input.byteOffset, input.byteLength);
+    const resp = this.inner.call(buf);
+    return new Uint8Array(resp.buffer, resp.byteOffset, resp.byteLength);
+  }
+
+  destroy(): void {
+    this.inner.close();
+  }
+}
+
+interface PendingCallback {
+  resolve: (data: Uint8Array) => void;
+  reject: (error: Error) => void;
+}
+
+/**
+ * Wraps the fire-and-forget async NAPI msgpack client behind the
+ * `IpcClientAsync` interface. Owns a map of pending calls keyed by request
+ * id; the C++ background polling thread invokes `setResponseCallback` once
+ * per response (in completion order), and this wrapper pairs it to its
+ * caller by the echoed id. Ids start at a random point per client so a
+ * stale frame left in a recycled SHM ring slot by a previous occupant
+ * cannot pair with a live call.
+ *
+ * `acquire` / `release` are reference-count hooks the NAPI exposes so the
+ * libuv loop is kept alive only while requests are outstanding — without
+ * them a `node script.js` would never exit naturally.
+ */
+export class NapiShmAsyncClient implements IpcClientAsync {
+  private readonly pending = new Map<bigint, PendingCallback>();
+  private nextRequestId =
+    (BigInt(Math.floor(Math.random() * 0xffffffff)) << 16n) + 1n;
+  private destroyed = false;
+
+  constructor(private inner: NapiMsgpackClientAsync) {
+    this.inner.setResponseCallback((requestId: bigint, response: Buffer) => {
+      if (this.destroyed) {
+        // Late response delivered after destroy(); the native close already
+        // balanced the TSFN reference.
+        return;
+      }
+      const cb = this.pending.get(requestId);
+      if (cb) {
+        this.pending.delete(requestId);
+        cb.resolve(new Uint8Array(response));
+        if (this.pending.size === 0) {
+          this.inner.release();
+        }
+      } else {
+        // SHM rings persist across occupants (slot reclaim / reattach), so a
+        // frame addressed to a previous occupant's id is an anticipated
+        // leftover — discard it and keep serving live calls. Log it so that if
+        // a genuinely lost pairing ever hangs a caller, the evidence is in the
+        // log rather than silently dropped. Don't release: no acquire was
+        // taken for an orphan response.
+        console.warn(
+          `NapiShmAsyncClient: discarding response for unknown request id ${requestId} ` +
+            "(stale frame from a previous ring occupant?)",
+        );
+      }
+    });
+  }
+
+  call(input: Uint8Array): Promise<Uint8Array> {
+    if (this.destroyed) {
+      return Promise.reject(
+        new Error("NapiShmAsyncClient: call() after destroy()"),
+      );
+    }
+    const buf = Buffer.isBuffer(input)
+      ? input
+      : Buffer.from(input.buffer, input.byteOffset, input.byteLength);
+    return new Promise<Uint8Array>((resolve, reject) => {
+      const requestId = this.nextRequestId++;
+      if (this.pending.size === 0) {
+        this.inner.acquire();
+      }
+      this.pending.set(requestId, { resolve, reject });
+      try {
+        this.inner.call(requestId, buf);
+      } catch (err: any) {
+        // Send failed — unwind the map entry we just added.
+        this.pending.delete(requestId);
+        if (this.pending.size === 0) {
+          this.inner.release();
+        }
+        reject(
+          err instanceof Error
+            ? err
+            : new Error(`SHM async call failed: ${String(err)}`),
+        );
+      }
+    });
+  }
+
+  async destroy(): Promise<void> {
+    if (this.destroyed) {
+      return;
+    }
+    this.destroyed = true;
+    // Reject anything still in flight.
+    const err = new Error("ipc-runtime SHM client destroyed before response");
+    for (const cb of this.pending.values()) {
+      cb.reject(err);
+    }
+    this.pending.clear();
+    // Stops the native poll thread and releases the TSFN reference taken
+    // when the map went 0 → 1 — without this, Node never exits when
+    // destroyed with calls in flight.
+    this.inner.close();
+  }
+}
+
+export interface CreateNapiShmOptions {
+  /** MPSC client slot id (default 0). Distinct clients on the same shmName must use distinct slots. */
+  clientId?: number;
+  /** Override addon path lookup. Rarely needed; useful for tests / unusual deployments. */
+  customAddonPath?: string;
+}
+
+/**
+ * Factories that load the bundled `ipc_runtime_napi.node` addon and
+ * construct an MPSC-SHM client wrapped behind the `IpcClient*` interface.
+ * Matches the transport used by `ipc::make_server` on the C++ side, so any
+ * server started via that helper accepts these clients directly.
+ */
+export function createNapiShmSyncClient(
+  shmName: string,
+  options: CreateNapiShmOptions = {},
+): NapiShmSyncClient {
+  const napi = loadIpcRuntimeNapi(options.customAddonPath);
+  return new NapiShmSyncClient(
+    // Omit the slot id when not given so the native side self-allocates a free
+    // slot (kAutoClientId) instead of aliasing every client onto slot 0.
+    options.clientId === undefined
+      ? new napi.MsgpackClient(shmName)
+      : new napi.MsgpackClient(shmName, options.clientId),
+  );
+}
+
+export function createNapiShmAsyncClient(
+  shmName: string,
+  options: CreateNapiShmOptions = {},
+): NapiShmAsyncClient {
+  const napi = loadIpcRuntimeNapi(options.customAddonPath);
+  return new NapiShmAsyncClient(
+    // Omit the slot id when not given so the native side self-allocates a free
+    // slot (kAutoClientId) instead of aliasing every client onto slot 0.
+    options.clientId === undefined
+      ? new napi.MsgpackClientAsync(shmName)
+      : new napi.MsgpackClientAsync(shmName, options.clientId),
+  );
+}

package/src/spawned_backend.test.ts

@@ -0,0 +1,225 @@
+import assert from "node:assert/strict";
+import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { after, test } from "node:test";
+import { IpcProcessExitedError, IpcSpawnError } from "./errors.js";
+import { SpawnedProcessBackend } from "./spawned_backend.js";
+
+const scratch = mkdtempSync(join(tmpdir(), "spawned-backend-test-"));
+after(() => rmSync(scratch, { recursive: true, force: true }));
+
+// A minimal length-prefixed echo server speaking the UdsIpcClient wire format.
+// Replies to each request with its own pid as a decimal string, so tests can
+// tell process incarnations apart.
+const PID_ECHO_SERVER = join(scratch, "pid_echo_server.cjs");
+writeFileSync(
+  PID_ECHO_SERVER,
+  `
+const net = require('node:net');
+const server = net.createServer(conn => {
+  let buf = Buffer.alloc(0);
+  conn.on('data', chunk => {
+    buf = Buffer.concat([buf, chunk]);
+    while (buf.length >= 4) {
+      const len = buf.readUInt32LE(0);
+      if (buf.length < 4 + len) return;
+      const requestId = buf.readBigUInt64LE(4); // frame = [len][8B id][payload]
+      buf = buf.subarray(4 + len);
+      const payload = Buffer.from(String(process.pid));
+      const out = Buffer.alloc(12 + payload.length);
+      out.writeUInt32LE(payload.length + 8, 0);
+      out.writeBigUInt64LE(requestId, 4);
+      payload.copy(out, 12);
+      conn.write(out);
+    }
+  });
+});
+server.listen(process.argv[2]);
+`,
+);
+
+function shellScript(name: string, body: string): string {
+  const path = join(scratch, name);
+  writeFileSync(path, `#!/bin/sh\n${body}\n`);
+  chmodSync(path, 0o755);
+  return path;
+}
+
+function spawnPidEcho(
+  opts: { respawn?: boolean; connectTimeoutMs?: number } = {},
+) {
+  return SpawnedProcessBackend.spawn({
+    binaryPath: process.execPath,
+    binaryName: "pid_echo_server",
+    instancePrefix: "pid-echo-test",
+    ipcPathArgs: [PID_ECHO_SERVER, "{path}"],
+    transport: "uds",
+    ...opts,
+  });
+}
+
+async function callPid(backend: SpawnedProcessBackend): Promise<string> {
+  return Buffer.from(await backend.call(Uint8Array.from([1]))).toString();
+}
+
+function processAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function waitFor(
+  cond: () => boolean,
+  timeoutMs: number,
+): Promise<boolean> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (cond()) return true;
+    await new Promise((resolve) => setTimeout(resolve, 25));
+  }
+  return cond();
+}
+
+test("spawns, round-trips calls, and destroys cleanly", async () => {
+  const backend = await spawnPidEcho();
+  const pid = await callPid(backend);
+  assert.match(pid, /^\d+$/);
+  await backend.destroy();
+  assert.equal(processAlive(parseInt(pid, 10)), false);
+});
+
+test("without respawn, death rejects in-flight and later calls with a retryable exit error", async () => {
+  const backend = await spawnPidEcho();
+  const pid = parseInt(await callPid(backend), 10);
+
+  process.kill(pid, "SIGKILL");
+  await waitFor(() => !processAlive(pid), 2_000);
+
+  const err = await callPid(backend).then(
+    () => undefined,
+    (e: Error) => e,
+  );
+  assert.ok(
+    err instanceof IpcProcessExitedError,
+    `expected IpcProcessExitedError, got ${err}`,
+  );
+  assert.equal((err as IpcProcessExitedError).retry, true);
+  assert.equal((err as IpcProcessExitedError).signal, "SIGKILL");
+
+  // Still down: same classified error, no zombie respawn.
+  await assert.rejects(callPid(backend), IpcProcessExitedError);
+  await backend.destroy();
+});
+
+test("with respawn, the next call after a death transparently gets a fresh process", async () => {
+  const backend = await spawnPidEcho({ respawn: true });
+  const firstPid = parseInt(await callPid(backend), 10);
+
+  process.kill(firstPid, "SIGKILL");
+  await waitFor(() => !processAlive(firstPid), 2_000);
+
+  // The first call after the death may race the exit event; it is allowed to
+  // fail (with a retryable error) or succeed against the respawned process.
+  let secondPid: number | undefined;
+  try {
+    secondPid = parseInt(await callPid(backend), 10);
+  } catch (err) {
+    assert.equal((err as { retry?: boolean }).retry, true);
+    secondPid = parseInt(await callPid(backend), 10);
+  }
+  assert.notEqual(secondPid, firstPid);
+  assert.equal(processAlive(secondPid!), true);
+
+  // getIpcPath is stable across incarnations.
+  const path = backend.getIpcPath();
+  await backend.destroy();
+  assert.equal(processAlive(secondPid!), false);
+  assert.equal(backend.getIpcPath(), path);
+});
+
+test("a missing binary fails with retry=false", async () => {
+  const err = await SpawnedProcessBackend.spawn({
+    binaryPath: join(scratch, "does_not_exist"),
+    binaryName: "ghost",
+    instancePrefix: "ghost-test",
+    ipcPathArgs: ["{path}"],
+    transport: "uds",
+  }).then(
+    () => undefined,
+    (e: Error) => e,
+  );
+  assert.ok(err instanceof IpcSpawnError, `expected IpcSpawnError, got ${err}`);
+  assert.equal((err as IpcSpawnError).retry, false);
+});
+
+test("a server dying before listen fails promptly with retry=true and the exit code", async () => {
+  const dying = shellScript("dying_server.sh", "exit 7");
+  const started = Date.now();
+  const err = await SpawnedProcessBackend.spawn({
+    binaryPath: dying,
+    binaryName: "dying_server",
+    instancePrefix: "dying-test",
+    ipcPathArgs: ["{path}"],
+    transport: "uds",
+  }).then(
+    () => undefined,
+    (e: Error) => e,
+  );
+  assert.ok(err instanceof IpcSpawnError, `expected IpcSpawnError, got ${err}`);
+  assert.equal((err as IpcSpawnError).retry, true);
+  assert.match((err as IpcSpawnError).message, /code=7/);
+  assert.ok(Date.now() - started < 5_000, "failed before the connect backstop");
+});
+
+test("a wedged server is killed when the connect backstop fires", async () => {
+  const pidFile = join(scratch, "wedged.pid");
+  const wedged = shellScript(
+    "wedged_server.sh",
+    `echo $$ > "${pidFile}"\nexec sleep 600`,
+  );
+  const err = await SpawnedProcessBackend.spawn({
+    binaryPath: wedged,
+    binaryName: "wedged_server",
+    instancePrefix: "wedged-test",
+    ipcPathArgs: ["{path}"],
+    transport: "uds",
+    connectTimeoutMs: 1_000,
+  }).then(
+    () => undefined,
+    (e: Error) => e,
+  );
+  assert.ok(err instanceof IpcSpawnError, `expected IpcSpawnError, got ${err}`);
+  assert.equal((err as IpcSpawnError).retry, true);
+  const { readFileSync } = await import("node:fs");
+  const pid = parseInt(readFileSync(pidFile, "utf-8").trim(), 10);
+  assert.equal(
+    await waitFor(() => !processAlive(pid), 5_000),
+    true,
+    "wedged process was reaped",
+  );
+});
+
+test("destroy during a pending respawn does not leak the fresh process", async () => {
+  const backend = await spawnPidEcho({ respawn: true });
+  const firstPid = parseInt(await callPid(backend), 10);
+  process.kill(firstPid, "SIGKILL");
+  await waitFor(() => !processAlive(firstPid), 2_000);
+
+  // Trigger the lazy respawn, then destroy while it may still be in flight.
+  const pending = callPid(backend).catch(() => undefined);
+  await backend.destroy();
+  const result = await pending;
+
+  if (result !== undefined) {
+    // The respawn won the race and served the call; destroy() must still have
+    // reaped the fresh process afterwards.
+    assert.equal(
+      await waitFor(() => !processAlive(parseInt(result, 10)), 2_000),
+      true,
+    );
+  }
+});