@openparachute/hub 0.6.5-rc.6 → 0.6.5-rc.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openparachute/hub",
-  "version": "0.6.5-rc.6",
+  "version": "0.6.5-rc.8",
   "description": "parachute — the local hub for the Parachute ecosystem (discovery, ports, lifecycle, soon OAuth).",
   "license": "AGPL-3.0",
   "publishConfig": {

package/src/__tests__/hub-db-liveness.test.ts

@@ -214,18 +214,23 @@ describe("DbHolder.probePath (#610 proactive detection)", () => {
     h.cleanup();
   });
 
-  test("path GONE (ENOENT) → reopen attempted; reopen verify fails → exit(1)", () => {
-    // Reopen returns a closed handle (the dir is still gone) → SELECT 1 throws
-    // → exit. This is the genuine `rm -rf ~/.parachute` field shape.
-    const dead = new Database(":memory:");
-    dead.close();
+  test("path GONE (ENOENT) → exit(1) directly, NO reopen (#619 follow-up)", () => {
+    // The genuine `rm -rf ~/.parachute` field shape. We must NOT reopen here:
+    // reopen is openHubDb, which mkdir-recursive's the dir back + opens a fresh
+    // EMPTY db, so its SELECT-1 verify would PASS and the hub would "heal" into a
+    // half-recovered state (empty db, stale in-memory state, wiped well-known,
+    // un-respawned modules). A full wipe must exit so the platform manager does a
+    // clean restart that re-bootstraps everything. `onReopen` throws to PROVE the
+    // reopen path is never taken — if it were, this test would surface the throw.
     const h = makeHolder({
       initialInode: INODE_A,
       statInode: () => undefined, // ENOENT
-      onReopen: () => dead,
+      onReopen: () => {
+        throw new Error("reopen must NOT be called on a gone verdict");
+      },
     });
     expect(h.holder.probePath()).toBe("gone");
-    expect(h.stats().reopens).toBe(1);
+    expect(h.stats().reopens).toBe(0);
     expect(h.stats().exits).toBe(1);
     expect(h.stats().exitCode).toBe(1);
     h.cleanup();

package/src/__tests__/serve.test.ts

@@ -4,6 +4,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { _resetBootstrapTokenForTests, getBootstrapToken } from "../bootstrap-token.ts";
 import {
+  armServeDbWatchdog,
   formatBootstrapTokenBanner,
   formatListeningBanner,
   hubPortConflictMessage,
@@ -463,3 +464,74 @@ describe("resolveStartupIssuer — expose-state fallback (#531)", () => {
     expect(resolveStartupIssuer({}, {}, throwing)).toBeUndefined();
   });
 });
+
+describe("armServeDbWatchdog — #610/#619 ghost-fd watchdog wiring on the serve path", () => {
+  let tmp: string;
+  let realDbPath: string;
+
+  beforeEach(() => {
+    tmp = mkdtempSync(join(tmpdir(), "serve-watchdog-"));
+    realDbPath = join(tmp, "hub.db");
+  });
+  afterEach(() => {
+    rmSync(tmp, { recursive: true, force: true });
+  });
+
+  test("starts the liveness timer (without it, a wipe is never noticed)", () => {
+    let tick: (() => void) | undefined;
+    const { livenessTimer } = armServeDbWatchdog(realDbPath, {
+      openDb: () => openHubDb(realDbPath),
+      statInode: () => ({ dev: 1, ino: 42 }),
+      setIntervalFn: (cb) => {
+        tick = cb;
+        return 0;
+      },
+      clearIntervalFn: () => {},
+    });
+    // The timer must actually be armed — the captured tick callback proves
+    // startDbPathLivenessTimer ran (the #619 bug was that it never did on this path).
+    expect(tick).toBeInstanceOf(Function);
+    expect(livenessTimer).toBeDefined();
+  });
+
+  test("opens the db BEFORE snapshotting the inode, so a wipe tick self-exits (#619 ordering)", () => {
+    // The load-bearing invariant: `initialInode` must be a DEFINED baseline so
+    // a later "gone" verdict fires reopen-or-exit. If the helper statted before
+    // opening (the bug), a fresh path would yield ENOENT → undefined baseline →
+    // probe stuck at "unknown" → NEVER exits on a wipe.
+    let opened = false;
+    let wiped = false;
+    let tick: (() => void) | undefined;
+    const exitCodes: number[] = [];
+    armServeDbWatchdog(realDbPath, {
+      openDb: () => {
+        if (wiped) throw new Error("ENOENT: state dir wiped");
+        opened = true;
+        return openHubDb(realDbPath);
+      },
+      statInode: () => {
+        if (wiped) return undefined; // path gone
+        // Proves ordering: if the helper statted before opening, this throws and
+        // the helper's catch leaves initialInode undefined (watchdog disabled).
+        if (!opened) throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
+        return { dev: 1, ino: 42 };
+      },
+      setIntervalFn: (cb) => {
+        tick = cb;
+        return 0;
+      },
+      clearIntervalFn: () => {},
+      exit: (code) => exitCodes.push(code),
+    });
+
+    // Simulate the wipe, then drive one watchdog tick.
+    wiped = true;
+    expect(tick).toBeInstanceOf(Function);
+    tick?.();
+
+    // The probe saw "gone" against a real baseline → reopen threw (dir gone) →
+    // exit(1). A non-zero exitCodes proves `initialInode` was a defined baseline,
+    // which proves the db was opened before the inode snapshot.
+    expect(exitCodes).toEqual([1]);
+  });
+});

package/src/commands/serve.ts

@@ -34,7 +34,7 @@ import { generateBootstrapToken } from "../bootstrap-token.ts";
 // path isolation.
 import { CONFIG_DIR, SERVICES_MANIFEST_PATH } from "../config.ts";
 import { readExposeState } from "../expose-state.ts";
-import { createDbHolder } from "../hub-db-liveness.ts";
+import { createDbHolder, defaultStatInode, startDbPathLivenessTimer } from "../hub-db-liveness.ts";
 import { hubDbPath, openHubDb } from "../hub-db.ts";
 import { hubFetch } from "../hub-server.ts";
 import { writeHubFile } from "../hub.ts";
@@ -304,6 +304,80 @@ export function formatBootstrapTokenBanner(token: string, hubUrl?: string): stri
   ].join("\n");
 }
 
+/**
+ * Injectable seams for {@link armServeDbWatchdog} (test-only). Generic on the
+ * timer handle `H` so the scheduler seams never name `setInterval` in type
+ * position — mirrors `DbLivenessTimerDeps<H>` in hub-db-liveness.ts, which
+ * keeps the public interface portable to a types-less tsc environment.
+ */
+export interface ServeDbWatchdogDeps<H = unknown> {
+  log?: (line: string) => void;
+  /** Open a db handle (default {@link openHubDb}). Tests inject a fake that creates a fixture. */
+  openDb?: (path: string) => ReturnType<typeof openHubDb>;
+  /** Path stat for the inode snapshot + proactive probe (default {@link defaultStatInode}). */
+  statInode?: typeof defaultStatInode;
+  /** Injectable scheduler threaded to the liveness timer (default `setInterval`). */
+  setIntervalFn?: (cb: () => void, ms: number) => H;
+  /** Injectable clear threaded to the liveness timer (default `clearInterval`). */
+  clearIntervalFn?: (handle: H) => void;
+  /** Process-exit fn threaded into the holder's reopen-or-exit (default `process.exit`). */
+  exit?: (code: number) => void;
+}
+
+/**
+ * Build the self-heal DB holder (#594) + start the proactive ghost-fd watchdog
+ * (#610) for the `parachute serve` path, returning both so the caller wires
+ * `getDb`/`onDbError`/`probeDbPath` and stops the timer on shutdown.
+ *
+ * Extracted + exported so the wiring is unit-testable WITHOUT binding a real
+ * port (#619): a serve()-level test would have to `Bun.serve` and risk the
+ * hub#535 launchd-bootout hazard, so this pure helper carries the load-bearing
+ * invariants instead — (1) the db is OPENED before the inode is snapshotted, so
+ * a fresh-install first boot gets a defined baseline (an ENOENT snapshot would
+ * silently disable the proactive probe for the whole process lifetime), and
+ * (2) the liveness timer is actually started. Both were absent on this path
+ * before #619 — the watchdog was wired only into `createHubServer`.
+ */
+export function armServeDbWatchdog<H = unknown>(
+  dbPath: string,
+  deps: ServeDbWatchdogDeps<H> = {},
+): {
+  dbHolder: ReturnType<typeof createDbHolder>;
+  livenessTimer: ReturnType<typeof startDbPathLivenessTimer>;
+} {
+  const openDb = deps.openDb ?? openHubDb;
+  const statInode = deps.statInode ?? defaultStatInode;
+  // Open FIRST — `openHubDb` mkdir's + creates the file when absent, so the
+  // stat below sees a real inode on a fresh-install first boot. Reversing this
+  // would leave `initialInode` undefined (ENOENT) and the probe at "unknown"
+  // for the process lifetime. Mirrors `createHubServer`'s ordering.
+  const db = openDb(dbPath);
+  let initialInode: ReturnType<typeof defaultStatInode> | undefined;
+  try {
+    initialInode = statInode(dbPath);
+  } catch {
+    initialInode = undefined;
+  }
+  const dbHolder = createDbHolder(db, {
+    reopen: () => openDb(dbPath),
+    dbPath,
+    statInode,
+    initialInode,
+    ...(deps.log !== undefined ? { log: deps.log } : {}),
+    ...(deps.exit !== undefined ? { exit: deps.exit } : {}),
+  });
+  // The active `parachute serve` path (systemd / launchd / container ExecStart)
+  // MUST start the watchdog here, not only in `createHubServer` — else a
+  // `rm -rf ~/.parachute` under a running unit leaves a ghost fd that keeps
+  // SELECT 1 succeeding with no thrown error, the reactive path never fires,
+  // and the hub never self-recovers (#619).
+  const livenessTimer = startDbPathLivenessTimer<H>(dbHolder, {
+    ...(deps.setIntervalFn !== undefined ? { setIntervalFn: deps.setIntervalFn } : {}),
+    ...(deps.clearIntervalFn !== undefined ? { clearIntervalFn: deps.clearIntervalFn } : {}),
+  });
+  return { dbHolder, livenessTimer };
+}
+
 /**
  * Run the hub fetch loop in the foreground. Resolves when `Bun.serve` is
  * bound; the returned `stop()` shuts the server down for tests.
@@ -355,15 +429,8 @@ export async function serve(opts: ServeOpts = {}): Promise<{
   writeHubFile(hubHtmlPath);
 
   const dbPath = hubDbPath();
-  // Self-heal-or-die DB holder (#594). The handle lives behind a mutable
-  // holder so a request that hits the persistent-corruption class (disk I/O
-  // error / malformed image — e.g. the state dir deleted under a running hub)
-  // can reopen the handle once, or exit(1) for the platform manager to restart
-  // us with a fresh one. `getDb` reads the current handle from the holder.
-  const dbHolder = createDbHolder(openHubDb(dbPath), {
-    reopen: () => openHubDb(dbPath),
-    log,
-  });
+  // Self-heal-or-die DB holder (#594) + proactive ghost-fd watchdog (#610/#619).
+  const { dbHolder, livenessTimer } = armServeDbWatchdog(dbPath, { log });
   const adminBootstrap = await seedInitialAdminIfNeeded(dbHolder.get(), env, log);
 
   if (adminBootstrap === "needs-setup") {
@@ -401,6 +468,9 @@ export async function serve(opts: ServeOpts = {}): Promise<{
       fetch: hubFetch(WELL_KNOWN_DIR, {
         getDb: () => dbHolder.get(),
         onDbError: (err) => dbHolder.healOrExit(err),
+        // #610: /health's db check probes the path so monitoring + the #591
+        // adoption probe see a wipe instead of the ghost-fd lie.
+        probeDbPath: () => dbHolder.probePath(),
         issuer,
         loopbackPort: port,
         supervisor,
@@ -486,6 +556,7 @@ export async function serve(opts: ServeOpts = {}): Promise<{
       for (const state of supervisor.list()) {
         await supervisor.stop(state.short);
       }
+      livenessTimer.stop();
       await server.stop();
       dbHolder.get().close();
     },

package/src/hub-db-liveness.ts

@@ -383,25 +383,41 @@ export function createDbHolder(initial: Database, deps: DbHolderDeps): DbHolder
       const verdict = classifyPathLiveness({ expected: currentInode, current: pathInode });
       if (verdict === "ok" || verdict === "unknown") return verdict;
 
-      // Genuine wipe signal: the on-disk DB the handle points at is gone
-      // ("gone") or was replaced underneath us ("replaced"). Trigger the SAME
-      // reopen-or-exit machinery. When the path is gone, reopen's SELECT-1
-      // verify fails → exit → platform manager restarts with a fresh on-disk
-      // handle (seconds, not "never"). When replaced, we adopt the fresh inode.
+      if (verdict === "gone") {
+        // The whole state dir was wiped under the running hub (`rm -rf
+        // ~/.parachute`). We must NOT reopen-in-place here: `reopen` is
+        // `openHubDb`, which `mkdirSync`'s the dir back + opens a fresh EMPTY db,
+        // so its SELECT-1 verify would PASS and we'd "heal" into a half-recovered
+        // hub — empty db, but stale in-memory state, wiped well-known files, and
+        // supervised modules whose own state dirs are gone yet never re-spawned
+        // (#619 follow-up). The correct recovery for a full wipe is a clean
+        // process exit so the platform manager (systemd / launchd / container)
+        // restarts `parachute serve`, which re-bootstraps everything (well-known,
+        // admin seed, supervisor re-spawn). This restores the #610 design intent
+        // ("we exit, letting the platform manager restart") that the shared
+        // reopen-or-exit path silently defeated via openHubDb's mkdir-recursive.
+        log(
+          `parachute hub: db path ${deps.dbPath} no longer exists (state dir wiped under a running hub, #610); exiting so the platform manager restarts the hub with a freshly bootstrapped state dir.`,
+        );
+        exit(1);
+        return verdict;
+      }
+
+      // "replaced": the db FILE was swapped underneath us (e.g. a restore copied
+      // a new file over the same path) while the rest of the state dir is intact.
+      // Adopting the fresh inode in-place via reopen-or-exit is correct here — a
+      // process restart would be heavier than needed.
       //
-      // ONE-TICK /health ANOMALY (intentional): on a "replaced" verdict the
-      // reopenOrExit below heals SYNCHRONOUSLY, but we still RETURN "replaced"
-      // for this one call — so the /health request that drove this probe reports
-      // `db:"error: path-replaced"` even though the handle is now healthy; the
-      // very next request reads `ok`. We don't mask it (returning "ok" here would
-      // hide that a heal just happened, which is exactly what monitoring wants to
-      // see). It's safe because #591's adoption probe checks only HTTP 200
-      // (`res.ok`), not the specific `db` string, so a single transient error
-      // string can't cascade.
+      // ONE-TICK /health ANOMALY (intentional): the reopenOrExit below heals
+      // SYNCHRONOUSLY, but we still RETURN "replaced" for this one call — so the
+      // /health request that drove this probe reports `db:"error: path-replaced"`
+      // even though the handle is now healthy; the very next request reads `ok`.
+      // We don't mask it (returning "ok" here would hide that a heal just
+      // happened, which is exactly what monitoring wants to see). It's safe
+      // because #591's adoption probe checks only HTTP 200 (`res.ok`), not the
+      // specific `db` string, so a single transient error string can't cascade.
       reopenOrExit(
-        verdict === "gone"
-          ? `db path ${deps.dbPath} no longer exists (state dir wiped under a running hub, #610)`
-          : `db path ${deps.dbPath} now resolves to a different inode (DB file replaced underneath the open handle, #610)`,
+        `db path ${deps.dbPath} now resolves to a different inode (DB file replaced underneath the open handle, #610)`,
       );
       return verdict;
     },

package/src/hub-server.ts

@@ -1627,8 +1627,11 @@ export function hubFetch(
             // succeeding, so `probeDbLiveness` alone would report `db:"ok"` on a
             // database that's gone from disk (the /health lie the issue calls
             // out). `probeDbPath` stat()s the path + compares inodes; on a
-            // gone/replaced verdict it ALSO self-heals (reopen-or-exit) and we
-            // surface the fault so the #591 adoption probe + monitoring see it.
+            // "replaced" verdict it self-heals in-place (reopen-or-exit, adopt
+            // the new inode); on a "gone" verdict it exits the process directly
+            // (#621 — a full wipe needs a clean platform-manager restart, not an
+            // empty-db reopen). Either way we surface the fault so the #591
+            // adoption probe + monitoring see it.
             const pathVerdict = deps?.probeDbPath?.();
             if (pathVerdict === "gone" || pathVerdict === "replaced") {
               // One-request anomaly on "replaced": probeDbPath already healed the