@kabmo-labs/kabmo 0.0.0-stage → 0.0.1

package/src/server/lib/lifecycle/bun.ts

@@ -0,0 +1,20 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * The Bun binary to spawn child processes with.
+ *
+ * The server is usually itself Bun, so `process.execPath` is the most
+ * trustworthy answer — it is the exact runtime this install was started with.
+ * A host that is not Bun falls back to a PATH lookup, and finally to the
+ * literal `bun`, which lets the child resolve it from its own environment.
+ */
+export function resolveBunBin(): string {
+  const execPath = process.execPath;
+  // `bun.exe` on Windows: the launcher carries the extension there, and without
+  // it this fell through to a PATH lookup for a runtime already running.
+  if (/(^|[\\/])bun(\.exe)?$/.test(execPath)) return execPath;
+  return Bun.which('bun') ?? 'bun';
+}

package/src/server/lib/lifecycle/fd-pressure.test.ts

@@ -0,0 +1,110 @@
+import { describe, expect, test } from 'bun:test';
+import fs from 'node:fs';
+import {
+  SPAWN_CLIFF,
+  countOpenFileDescriptors,
+  describeFdPressure,
+  describeSpawnFailure,
+  descriptorTableSize,
+} from '@/server/lib/lifecycle/fd-pressure';
+
+describe('descriptorTableSize', () => {
+  // The bug this pins: the limit used to be the constant 10240, so on this host
+  // (getdtablesize() = 61440) the probe reported "near the cliff" from 9216 —
+  // six times the budget still free — and then never stopped reporting it.
+  test('reports the kernel ceiling, which is what allocation actually enforces', () => {
+    if (process.platform === 'win32') {
+      expect(descriptorTableSize()).toBe(SPAWN_CLIFF);
+      return;
+    }
+    const size = descriptorTableSize();
+    expect(size).toBeGreaterThanOrEqual(SPAWN_CLIFF);
+    // 61440 on the machine this was measured on; never the shell's `ulimit -n`,
+    // which reads 1048576 there and would understate pressure seventeenfold.
+    expect(Number.isInteger(size)).toBe(true);
+  });
+});
+
+describe('countOpenFileDescriptors', () => {
+  test('sweeps this process own descriptors on a POSIX host', () => {
+    const pressure = countOpenFileDescriptors();
+    if (process.platform === 'win32') {
+      expect(pressure).toBeNull();
+      return;
+    }
+    expect(pressure).not.toBeNull();
+    // Stdio is always open.
+    expect(pressure?.open).toBeGreaterThanOrEqual(3);
+    expect(pressure?.highest).toBeGreaterThanOrEqual(2);
+    // The limit is the real ceiling (bounded by the sweep cap), never the old
+    // constant — a limit of 10240 is exactly the wrong answer on this host.
+    expect(pressure?.limit).toBeGreaterThanOrEqual(SPAWN_CLIFF);
+    expect(pressure?.nearCliff).toBe(false);
+  });
+
+  test('counts descriptors opened and closed around it', () => {
+    const before = countOpenFileDescriptors();
+    if (!before) return; // Host cannot answer; nothing to assert.
+    const opened = [0, 1, 2, 3, 4].map(() => fs.openSync('/dev/null', 'r'));
+    try {
+      const during = countOpenFileDescriptors();
+      expect(during!.open - before.open).toBeGreaterThanOrEqual(opened.length);
+      expect(during!.highest).toBeGreaterThanOrEqual(Math.max(...opened));
+    } finally {
+      for (const fd of opened) fs.closeSync(fd);
+    }
+  });
+});
+
+describe('describeFdPressure', () => {
+  test('is silent when the table has room, so no caller blames the wrong thing', () => {
+    expect(describeFdPressure(null)).toBeNull();
+    expect(describeFdPressure({ open: 40, highest: 42, limit: SPAWN_CLIFF, nearCliff: false })).toBeNull();
+  });
+
+  test('names the count, the limit and the fix when the table is near the cliff', () => {
+    const sentence = describeFdPressure({ open: 10_239, highest: 10_239, limit: SPAWN_CLIFF, nearCliff: true });
+    expect(sentence).toContain('out of file descriptors');
+    expect(sentence).toContain('10239 of 10240');
+    expect(sentence).toContain('restart the instance');
+  });
+});
+
+describe('describeSpawnFailure', () => {
+  test('passes an unrelated failure through untouched', () => {
+    // The helper must not relabel a genuine bad-descriptor or ENOENT bug as
+    // resource pressure; only an observable full table earns the extra text.
+    expect(describeSpawnFailure(new Error('spawn ENOENT'))).toBe('spawn ENOENT');
+    expect(describeSpawnFailure('boom')).toBe('boom');
+  });
+
+  test('names exhaustion only when the table really is full', () => {
+    const opened: number[] = [];
+    try {
+      // Filling to the REAL ceiling would mean holding ~55k descriptors in the
+      // test process and would put every other suite at risk, so the fill stops
+      // at the old constant: on a host whose table is larger (this one reports
+      // 61440) the gate is untestable here and the synthetic-table test above
+      // is what covers it. The loop still degrades to that skip rather than
+      // asserting a pressure it did not create.
+      while (opened.length < SPAWN_CLIFF + 16) {
+        try {
+          opened.push(fs.openSync('/dev/null', 'r'));
+        } catch {
+          break; // Host limit below the fill target; the gate is untestable here.
+        }
+      }
+      if (!countOpenFileDescriptors()?.nearCliff) return;
+
+      const raw = "EBADF: bad file descriptor, posix_spawn '/bin/sh'";
+      const described = describeSpawnFailure(new Error(raw));
+      expect(described.startsWith(raw)).toBe(true);
+      expect(described).toContain('out of file descriptors');
+      // The symptom gate still holds under pressure: an unrelated failure keeps
+      // its own message instead of being blamed on the descriptor table.
+      expect(describeSpawnFailure(new Error('spawn ENOENT'))).toBe('spawn ENOENT');
+    } finally {
+      for (const fd of opened) fs.closeSync(fd);
+    }
+  });
+});

package/src/server/lib/lifecycle/fd-pressure.ts

@@ -0,0 +1,240 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * File-descriptor pressure, and the honest name for a spawn that dies from it.
+ *
+ * When the descriptor table is full, `posix_spawn` fails with a message that
+ * names the shim it tried to exec — `EBADF: bad file descriptor, posix_spawn
+ * '/bin/sh'` on macOS, `EMFILE: too many open files, socketpair` on Linux — and
+ * says nothing about the cause. A terminal panel reads "Failed to start
+ * /bin/zsh" while the real condition is that the whole server process has run
+ * out of descriptors, so this module answers "how close?" and names the real
+ * failure when it happens.
+ *
+ * A dev server reaches that state on its own, without a leak in this codebase:
+ * `Bun.serve({ development: true })` holds the client module graph open, and a
+ * macOS reload never releases the previous generation's (bun#40706, bun#40907 —
+ * the pre-exec sweep exists only on Linux). Measured on this app: 13
+ * descriptors on a freshly started server, 3354 after the first page load, and
+ * never released. The ceiling it climbs toward is the kernel's own
+ * (`getdtablesize()`: 61440 here), NOT Darwin's documented `OPEN_MAX` of 10240
+ * — see `descriptorTableSize`, which is what makes `nearCliff` mean something.
+ *
+ * `countOpenFileDescriptors()` answers with `fcntl(F_GETFD)` over the table —
+ * the same `bun:ffi` route `flock.ts` takes. It returns null where the symbols
+ * are unavailable (and on Windows), and every caller must read null as
+ * "unknown", never as "fine": `describeSpawnFailure` only names exhaustion when
+ * it can see the pressure, so a genuine bad-descriptor bug still reports itself
+ * plainly.
+ */
+
+import { dlopen, FFIType } from 'bun:ffi';
+import type { FFIFunction, Library } from 'bun:ffi';
+
+/**
+ * Darwin's documented `OPEN_MAX`, kept only as the fallback for a host whose
+ * table size cannot be read.
+ *
+ * It is NOT the live limit, and treating it as one is what let a real
+ * exhaustion go unwarned. Measured on this machine: `getdtablesize()` answers
+ * **61440** (`kern.maxfilesperproc`), so the process runs happily past 10240 —
+ * while the probe capped at 10240 and reported "near the cliff" from 9216, a
+ * warning that fired with six times the budget still free and then stayed on
+ * forever. The ceiling that matters is the one the kernel reports, which is
+ * what `descriptorTableSize` now returns.
+ */
+export const SPAWN_CLIFF = 10_240;
+
+/**
+ * Upper bound on the sweep, so the probe stays fast on a host with a huge table
+ * (Linux commonly reports 1,048,576, and one `fcntl` per descriptor there would
+ * be a fifth of a second on every health check). Measured: 61440 sweeps in
+ * 12 ms, so this covers every real ceiling while keeping the cost bounded.
+ *
+ * When the table is larger than this, `open` is a LOWER BOUND and `nearCliff`
+ * cannot be established — which is the honest answer, and why the cap is
+ * documented here rather than silently clipping the number.
+ */
+const SWEEP_CAP = 65_536;
+
+/**
+ * Pressure is reported as "near" from 90% of the real ceiling. The remaining
+ * tenth is what a burst of spawns needs (an omp child takes several), and the
+ * failure arrives without a warning of its own: once the table is full there is
+ * room for neither a retry nor a log line.
+ */
+const NEAR_CLIFF_RATIO = 0.9;
+
+const FD_SYMBOLS = {
+  fcntl: { args: [FFIType.i32, FFIType.i32, FFIType.i32], returns: FFIType.i32 },
+  getdtablesize: { args: [], returns: FFIType.i32 },
+} satisfies Record<string, FFIFunction>;
+
+/** Library names to try, in the order each platform resolves them. */
+const LIBC_NAMES = ['libc.dylib', 'libSystem.B.dylib', 'libc.so.6'] as const;
+
+/** `F_GETFD` — the cheapest question that distinguishes open from closed. */
+const F_GETFD = 1;
+
+declare global {
+  // eslint-disable-next-line no-var
+  var __kabmoFdLib: Library<typeof FD_SYMBOLS> | null | undefined;
+}
+
+/** libc with `fcntl`/`getdtablesize`, or null where this host has neither. */
+function libc(): Library<typeof FD_SYMBOLS> | null {
+  if (globalThis.__kabmoFdLib !== undefined) return globalThis.__kabmoFdLib;
+  globalThis.__kabmoFdLib = null;
+  if (process.platform === 'win32') return null;
+  for (const name of LIBC_NAMES) {
+    try {
+      globalThis.__kabmoFdLib = dlopen(name, FD_SYMBOLS);
+      return globalThis.__kabmoFdLib;
+    } catch {
+      // Try the next name; a host with none of them keeps the null answer.
+    }
+  }
+  return null;
+}
+
+export interface FdPressure {
+  /** Descriptors open below `limit`. */
+  open: number;
+  /** Highest descriptor number seen, or -1 when the table looked empty. */
+  highest: number;
+  /** Where the sweep stopped: `getdtablesize()`, capped at the cliff. */
+  limit: number;
+  /** Whether the next spawn is at risk. See `NEAR_CLIFF_RATIO`. */
+  nearCliff: boolean;
+}
+
+/**
+ * The kernel's descriptor table size for this process, or `SPAWN_CLIFF` when
+ * the host cannot report one.
+ *
+ * `getdtablesize()` is the syscall that answers this on Darwin and Linux alike,
+ * and it is the number the allocation path itself uses — so a limit read here
+ * is the same one an `open` or a `posix_spawn` will hit. It is deliberately NOT
+ * `ulimit -n`: on this machine the shell reports 1,048,576 while the kernel
+ * refuses the 61441st descriptor, so the shell's number would understate the
+ * pressure by a factor of seventeen.
+ */
+export function descriptorTableSize(): number {
+  const symbols = libc();
+  if (!symbols) return SPAWN_CLIFF;
+  try {
+    const reported = symbols.symbols.getdtablesize();
+    return Number.isFinite(reported) && reported > 0 ? reported : SPAWN_CLIFF;
+  } catch {
+    return SPAWN_CLIFF;
+  }
+}
+
+/**
+ * Sweep the descriptor table, or null when this host cannot answer.
+ *
+ * Descriptors are allocated lowest-first, so the number of open descriptors and
+ * the highest one agree on a saturated table — which is the only state that
+ * matters here, and the state a dev server reaches by degrees.
+ *
+ * `limit` is the REAL ceiling (see `descriptorTableSize`), not a constant: the
+ * whole point of this probe is to say how much room is left, and a limit that
+ * disagrees with the kernel reports the wrong amount of it. On a host whose
+ * table exceeds `SWEEP_CAP` the sweep stops at the cap, which makes `open` a
+ * lower bound — read as "at least this many", never as the total.
+ *
+ * Never throws: `GET /api/health` is the CLI's readiness probe and the client's
+ * liveness check, so a host where the symbol resolves but the call does not
+ * must degrade to "unknown" rather than take those down.
+ */
+export function countOpenFileDescriptors(): FdPressure | null {
+  const symbols = libc();
+  if (!symbols) return null;
+
+  try {
+    const table = descriptorTableSize();
+    const limit = Math.min(table, SWEEP_CAP);
+
+    let open = 0;
+    let highest = -1;
+    for (let fd = 0; fd < limit; fd++) {
+      if (symbols.symbols.fcntl(fd, F_GETFD, 0) !== -1) {
+        open += 1;
+        highest = fd;
+      }
+    }
+    return { open, highest, limit, nearCliff: highest >= limit * NEAR_CLIFF_RATIO };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * What descriptor exhaustion looks like in Bun's own words. macOS names the
+ * failing syscall (`EBADF … posix_spawn '/bin/sh'`), Linux the errno
+ * (`EMFILE: too many open files, socketpair`) — measured on both.
+ */
+const EXHAUSTION_SYMPTOM = /\bEBADF\b|\bEMFILE\b|too many open files/i;
+
+/**
+ * The sentence that names descriptor exhaustion, or null when there is none to
+ * name. Every caller that would otherwise blame the operation it was running
+ * needs this: a truncated table makes a bundle, a spawn and a plain file read
+ * fail identically, and the operation's own message never carries the cause.
+ */
+export function describeFdPressure(pressure = countOpenFileDescriptors()): string | null {
+  if (!pressure || !pressure.nearCliff) return null;
+  return (
+    `The Kabmo server is out of file descriptors ` +
+    `(${pressure.open} of ${pressure.limit} open, highest #${pressure.highest}); restart the instance. ` +
+    `A dev server holds the whole client module graph open and a macOS reload never releases ` +
+    `the previous generation (bun#40706), so the count only climbs.`
+  );
+}
+
+/**
+ * What a burst of spawns needs, kept free at all times.
+ *
+ * A terminal PTY, an omp child and `git`/`rg` can all be asked for at once, and
+ * once the table is full the failure arrives without a warning of its own:
+ * there is room for neither a retry nor a log line.
+ */
+const MIN_SPAWN_HEADROOM = 1_024;
+
+/**
+ * Whether `cost` more descriptors can be opened without walking into the cliff.
+ *
+ * This is the difference between a hard stop and a cascade. When the table is
+ * nearly full, an operation that needs thousands of descriptors — a client
+ * bundle, whose graph a dev server then HOLDS — does not merely fail: it fails
+ * partway, and the failure is what the reader sees instead of the cause. Asking
+ * first turns that into one honest answer.
+ *
+ * `null` (the host cannot answer) returns true: refusing work on a guess would
+ * break every host where the probe is unavailable, and the cost of being wrong
+ * is the failure we already had.
+ */
+export function hasDescriptorHeadroom(pressure: FdPressure | null, cost: number): boolean {
+  if (!pressure) return true;
+  return pressure.open + cost + MIN_SPAWN_HEADROOM <= pressure.limit;
+}
+
+/**
+ * The message to surface for a failed spawn: Bun's own, plus what it takes to
+ * act on it when the cause is descriptor exhaustion.
+ *
+ * Two conditions, both required. The message must carry the symptom, and the
+ * sweep must see real pressure — a truncated table is not the only way `EBADF`
+ * reaches here, and appending "out of descriptors" to a genuine bad-descriptor
+ * bug (or to an unrelated `ENOENT` that failed while the table happened to be
+ * full) sends the next reader down the wrong path.
+ */
+export function describeSpawnFailure(error: unknown): string {
+  const raw = error instanceof Error ? error.message : String(error);
+  if (!EXHAUSTION_SYMPTOM.test(raw)) return raw;
+  const pressure = describeFdPressure();
+  return pressure ? `${raw} — ${pressure}` : raw;
+}

package/src/server/lib/lifecycle/flock.test.ts

@@ -0,0 +1,166 @@
+/**
+ * @license
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * `flock.ts` is the only thing between two starters and a bind race, so these
+ * tests pin its kernel-level semantics rather than its API shape: a free lock
+ * is granted and names its holder, a lock already held by ANOTHER process is
+ * refused, a holder that dies without releasing frees the lease (the kernel
+ * drops it — which is why no stale-holder recovery exists here), re-entry
+ * within one process is refcounted, and releasing really releases.
+ *
+ * The cross-process case is the point of the module, so it is exercised
+ * against a real second process (a throwaway `bun -e`, never an `omp` server)
+ * instead of a fake. The same-inode/different-path case uses a symlink: it
+ * bypasses the module's re-entrancy shortcut and asks `flock(2)` directly.
+ *
+ * POSIX-only (`describePosix`): `flock(2)` has no Windows equivalent, so every
+ * lease semantics the module exposes is unreachable there.
+ */
+
+import { afterEach, beforeEach, expect, test } from 'bun:test';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import { flockAvailable, tryFlock } from '@/server/lib/lifecycle/flock';
+import type { FlockHandle } from '@/server/lib/lifecycle/flock';
+import { describePosix } from '@/test-support/platform';
+
+/** Without `flock(2)` every lease test would assert the unavailability path. */
+const flockWorks = flockAvailable();
+
+let tempDir: string;
+const leases: FlockHandle[] = [];
+const children: Bun.Subprocess[] = [];
+
+/** `tryFlock` plus cleanup bookkeeping, so no lease outlives its test. */
+function acquire(lockPath: string): FlockHandle | null {
+  const handle = tryFlock(lockPath);
+  if (handle) leases.push(handle);
+  return handle;
+}
+
+/**
+ * A real second process holding `lockPath`, confirmed by its own stdout before
+ * returning. It never releases: the test decides whether it exits.
+ */
+async function spawnLockHolder(lockPath: string): Promise<Bun.Subprocess> {
+  const modulePath = path.join(import.meta.dir, 'flock.ts');
+  // The subprocess loads the module by path at runtime — the specifier cannot
+  // be a static import in a program assembled as a string for `bun -e`.
+  const code = `
+    const mod = await import(${JSON.stringify(modulePath)});
+    const handle = mod.tryFlock(${JSON.stringify(lockPath)});
+    console.log(handle ? 'held' : 'denied');
+    await new Promise(() => {});
+  `;
+  const child = Bun.spawn({ cmd: ['bun', '-e', code], stdout: 'pipe', stderr: 'ignore' });
+  children.push(child);
+  const reader = (child.stdout as ReadableStream<Uint8Array>).getReader();
+  const first = await reader.read();
+  const said = new TextDecoder().decode(first.value ?? new Uint8Array());
+  expect(said).toContain('held');
+  return child;
+}
+
+beforeEach(() => {
+  tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kabmo-test-flock-'));
+});
+
+afterEach(async () => {
+  for (const child of children.splice(0)) {
+    try {
+      child.kill('SIGKILL');
+      await child.exited;
+    } catch {
+      // Already gone.
+    }
+  }
+  for (const lease of leases.splice(0)) lease.release();
+  // `held` and the probe result live on a globalThis singleton shared with the
+  // rest of the run; leaked state would hold a lock against a sibling suite.
+  // The singleton is untyped on `globalThis`, so the shape is named once here.
+  const host = globalThis as unknown as { __kabmoFlock?: unknown };
+  delete host.__kabmoFlock;
+  fs.rmSync(tempDir, { recursive: true, force: true });
+});
+
+describePosix('flockAvailable', () => {
+  test('answers with a boolean and caches the answer', () => {
+    const first = flockAvailable();
+    expect(typeof first).toBe('boolean');
+    // The probe writes a file in TMPDIR; answering it per call would be a
+    // filesystem round-trip on every startup.
+    expect(flockAvailable()).toBe(first);
+  });
+});
+
+describePosix('tryFlock', () => {
+  test('grants a free lock and records this process in the file', () => {
+    if (!flockWorks) return;
+    const file = path.join(tempDir, 'port.lock');
+    const lease = acquire(file);
+
+    expect(lease).not.toBeNull();
+    expect(fs.readFileSync(file, 'utf8')).toBe(String(process.pid));
+  });
+
+  test('refuses a lock held by another process, and reclaims it when that process dies', async () => {
+    if (!flockWorks) return;
+    const file = path.join(tempDir, 'port.lock');
+    const holder = await spawnLockHolder(file);
+
+    // The whole reason the port handover uses flock: another live holder is a
+    // refusal, not a wait, and no PID file can go stale here.
+    expect(tryFlock(file)).toBeNull();
+
+    holder.kill('SIGKILL');
+    await holder.exited;
+
+    const reclaimed = acquire(file);
+    expect(reclaimed).not.toBeNull();
+  });
+
+  test('an unopenable lock path is refused rather than thrown', () => {
+    if (!flockWorks) return;
+    // A run directory that vanished under us must not take the server down.
+    expect(tryFlock(path.join(tempDir, 'missing-dir', 'port.lock'))).toBeNull();
+  });
+
+  test('re-acquiring the same path is re-entrant and refcounted', () => {
+    if (!flockWorks) return;
+    const file = path.join(tempDir, 'port.lock');
+    const link = path.join(tempDir, 'alias.lock');
+    fs.symlinkSync(file, link);
+
+    const first = acquire(file);
+    const second = acquire(file);
+    expect(first).not.toBeNull();
+    expect(second).not.toBeNull();
+
+    // Different path, same inode: `flock(2)` is asked directly here, so this
+    // proves the kernel lock is still held after one of two refs was dropped.
+    first!.release();
+    expect(tryFlock(link)).toBeNull();
+
+    second!.release();
+    expect(acquire(link)).not.toBeNull();
+  });
+
+  test('releasing frees the lock and is idempotent', () => {
+    if (!flockWorks) return;
+    const file = path.join(tempDir, 'port.lock');
+    const link = path.join(tempDir, 'alias.lock');
+    fs.symlinkSync(file, link);
+
+    const lease = acquire(file);
+    expect(lease).not.toBeNull();
+    lease!.release();
+    lease!.release(); // A second release must not throw or double-close the fd.
+
+    expect(acquire(link)).not.toBeNull();
+  });
+});