@openparachute/hub 0.6.5-rc.5 → 0.6.5-rc.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openparachute/hub",
-  "version": "0.6.5-rc.5",
+  "version": "0.6.5-rc.7",
   "description": "parachute — the local hub for the Parachute ecosystem (discovery, ports, lifecycle, soon OAuth).",
   "license": "AGPL-3.0",
   "publishConfig": {

package/src/__tests__/serve.test.ts

@@ -4,6 +4,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { _resetBootstrapTokenForTests, getBootstrapToken } from "../bootstrap-token.ts";
 import {
+  armServeDbWatchdog,
   formatBootstrapTokenBanner,
   formatListeningBanner,
   hubPortConflictMessage,
@@ -463,3 +464,74 @@ describe("resolveStartupIssuer — expose-state fallback (#531)", () => {
     expect(resolveStartupIssuer({}, {}, throwing)).toBeUndefined();
   });
 });
+
+describe("armServeDbWatchdog — #610/#619 ghost-fd watchdog wiring on the serve path", () => {
+  let tmp: string;
+  let realDbPath: string;
+
+  beforeEach(() => {
+    tmp = mkdtempSync(join(tmpdir(), "serve-watchdog-"));
+    realDbPath = join(tmp, "hub.db");
+  });
+  afterEach(() => {
+    rmSync(tmp, { recursive: true, force: true });
+  });
+
+  test("starts the liveness timer (without it, a wipe is never noticed)", () => {
+    let tick: (() => void) | undefined;
+    const { livenessTimer } = armServeDbWatchdog(realDbPath, {
+      openDb: () => openHubDb(realDbPath),
+      statInode: () => ({ dev: 1, ino: 42 }),
+      setIntervalFn: (cb) => {
+        tick = cb;
+        return 0;
+      },
+      clearIntervalFn: () => {},
+    });
+    // The timer must actually be armed — the captured tick callback proves
+    // startDbPathLivenessTimer ran (the #619 bug was that it never did on this path).
+    expect(tick).toBeInstanceOf(Function);
+    expect(livenessTimer).toBeDefined();
+  });
+
+  test("opens the db BEFORE snapshotting the inode, so a wipe tick self-exits (#619 ordering)", () => {
+    // The load-bearing invariant: `initialInode` must be a DEFINED baseline so
+    // a later "gone" verdict fires reopen-or-exit. If the helper statted before
+    // opening (the bug), a fresh path would yield ENOENT → undefined baseline →
+    // probe stuck at "unknown" → NEVER exits on a wipe.
+    let opened = false;
+    let wiped = false;
+    let tick: (() => void) | undefined;
+    const exitCodes: number[] = [];
+    armServeDbWatchdog(realDbPath, {
+      openDb: () => {
+        if (wiped) throw new Error("ENOENT: state dir wiped");
+        opened = true;
+        return openHubDb(realDbPath);
+      },
+      statInode: () => {
+        if (wiped) return undefined; // path gone
+        // Proves ordering: if the helper statted before opening, this throws and
+        // the helper's catch leaves initialInode undefined (watchdog disabled).
+        if (!opened) throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
+        return { dev: 1, ino: 42 };
+      },
+      setIntervalFn: (cb) => {
+        tick = cb;
+        return 0;
+      },
+      clearIntervalFn: () => {},
+      exit: (code) => exitCodes.push(code),
+    });
+
+    // Simulate the wipe, then drive one watchdog tick.
+    wiped = true;
+    expect(tick).toBeInstanceOf(Function);
+    tick?.();
+
+    // The probe saw "gone" against a real baseline → reopen threw (dir gone) →
+    // exit(1). A non-zero exitCodes proves `initialInode` was a defined baseline,
+    // which proves the db was opened before the inode snapshot.
+    expect(exitCodes).toEqual([1]);
+  });
+});

package/src/__tests__/wizard.test.ts

@@ -9,7 +9,9 @@
  *      OK envelope.
  *   3. POST /admin/setup/vault (application/json) → 200 OK envelope
  *      with `op_id`.
- *   4. GET /api/modules/operations/<id> until terminal status.
+ *   4. GET /admin/setup?op=<id> until the `operation` envelope field
+ *      reaches a terminal status (hub#616 — session-authed poll surface,
+ *      NOT the Bearer-gated /api/modules/operations/:id the SPA uses).
  *   5. POST /admin/setup/expose (application/json) → 200 OK.
  *
  * The stub fetch in this file is a mini-router that mimics the
@@ -32,6 +34,15 @@ interface FakeHubState {
   importParams?: { remoteUrl: string; pat?: string; mode: string };
   exposeMode?: string;
   posted: Array<{ path: string; body: unknown }>;
+  /** hub#616: path+query of every op-poll GET, to assert the wizard polls the session surface. */
+  polled: string[];
+  /**
+   * hub#616: number of poll ticks the vault op reports `"running"` before
+   * flipping to `"succeeded"`. 0 (default) = succeeds immediately on the first
+   * poll. >0 exercises the multi-tick poll loop (the import-mode long-running
+   * provisioning path).
+   */
+  vaultProvisionTicks?: number;
   /** hub#576: when set, the fake GET /admin/setup reports requireBootstrapToken=true. */
   requireBootstrapToken?: boolean;
   /** hub#576: when set, the fake GET also returns it (loopback-probe behavior). */
@@ -52,6 +63,7 @@ function makeFakeHub(initialState?: Partial<FakeHubState>): {
     hasVault: false,
     hasExposeMode: false,
     posted: [],
+    polled: [],
     ...initialState,
   };
   // Synthesize a stable CSRF token + session token for the stub. The
@@ -69,6 +81,9 @@ function makeFakeHub(initialState?: Partial<FakeHubState>): {
       error?: string;
     }
   >();
+  // hub#616: per-op countdown of remaining `"running"` poll ticks before the
+  // op flips to `"succeeded"` (see FakeHubState.vaultProvisionTicks).
+  const opTicksRemaining = new Map<string, number>();
 
   const fetchImpl = async (
     input: string | URL | Request,
@@ -81,12 +96,28 @@ function makeFakeHub(initialState?: Partial<FakeHubState>): {
     const body = init?.body;
     const bodyJson: unknown = body ? JSON.parse(String(body)) : null;
 
-    // GET /admin/setup
-    if (path === "/admin/setup" && method === "GET") {
+    // GET /admin/setup (incl. the `?op=<id>` poll surface — hub#616)
+    if (url.pathname === "/admin/setup" && method === "GET") {
       let step: "welcome" | "vault" | "expose" | "done" = "welcome";
       if (state.hasAdmin && state.hasVault && state.hasExposeMode) step = "done";
       else if (state.hasAdmin && state.hasVault) step = "expose";
       else if (state.hasAdmin) step = "vault";
+      // hub#616: the CLI wizard polls vault provisioning via this session-authed
+      // GET with `?op=<id>`; the op snapshot rides in the `operation` field.
+      const opId = url.searchParams.get("op");
+      if (opId) state.polled.push(path);
+      const op = opId ? ops.get(opId) : undefined;
+      // hub#616: drive a multi-tick op (running → succeeded) so the poll loop
+      // is exercised across more than one fetch when configured.
+      if (op && op.status === "running") {
+        const remaining = opTicksRemaining.get(op.id) ?? 0;
+        if (remaining <= 0) {
+          op.status = "succeeded";
+          state.hasVault = true;
+        } else {
+          opTicksRemaining.set(op.id, remaining - 1);
+        }
+      }
       const respBody = JSON.stringify({
         step,
         hasAdmin: state.hasAdmin,
@@ -96,6 +127,7 @@ function makeFakeHub(initialState?: Partial<FakeHubState>): {
         csrfToken: csrf,
         // hub#576: a loopback probe carries the actual token value.
         ...(state.bootstrapToken ? { bootstrapToken: state.bootstrapToken } : {}),
+        ...(op ? { operation: op } : {}),
       });
       return new Response(respBody, {
         status: 200,
@@ -155,30 +187,36 @@ function makeFakeHub(initialState?: Partial<FakeHubState>): {
           ...(b.pat ? { pat: b.pat } : {}),
         };
       }
-      // Create an op + drive it to succeeded immediately for the test.
+      // Create an op. By default it succeeds immediately on the first poll;
+      // when `vaultProvisionTicks` is set it reports `"running"` for that many
+      // poll ticks first (hub#616 — exercises the multi-tick poll loop).
       const opId = `op-test-${++opCount}`;
-      ops.set(opId, { id: opId, status: "succeeded", log: ["bun add -g", "spawned"] });
-      state.hasVault = true;
+      const ticks = state.vaultProvisionTicks ?? 0;
+      if (ticks > 0) {
+        ops.set(opId, { id: opId, status: "running", log: ["bun add -g"] });
+        opTicksRemaining.set(opId, ticks);
+      } else {
+        ops.set(opId, { id: opId, status: "succeeded", log: ["bun add -g", "spawned"] });
+        state.hasVault = true;
+      }
       return new Response(JSON.stringify({ op_id: opId, step: "vault", mode: b.mode }), {
         status: 200,
         headers: { "content-type": "application/json; charset=utf-8" },
       });
     }
 
-    // GET /api/modules/operations/<id>
-    if (path.startsWith("/api/modules/operations/") && method === "GET") {
-      const id = path.slice("/api/modules/operations/".length);
-      const op = ops.get(id);
-      if (!op) {
-        return new Response(JSON.stringify({ error: "not found" }), {
-          status: 404,
-          headers: { "content-type": "application/json; charset=utf-8" },
-        });
-      }
-      return new Response(JSON.stringify(op), {
-        status: 200,
-        headers: { "content-type": "application/json; charset=utf-8" },
-      });
+    // GET /api/modules/operations/<id> — the Bearer-gated SPA/install-CLI poll
+    // surface. hub#616: the CLI wizard must NOT use it (it holds only a session
+    // cookie, not a host-admin Bearer). Mirror the real 401 so any regression
+    // back to this path fails the wizard poll loudly instead of silently.
+    if (url.pathname.startsWith("/api/modules/operations/") && method === "GET") {
+      return new Response(
+        JSON.stringify({
+          error: "unauthenticated",
+          error_description: "Authorization: Bearer required",
+        }),
+        { status: 401, headers: { "content-type": "application/json; charset=utf-8" } },
+      );
     }
 
     // POST /admin/setup/expose
@@ -287,6 +325,37 @@ describe("runCliWizard", () => {
     expect(vaultBody.mode).toBe("create");
     expect(vaultBody.vault_name).toBe("default");
     expect(state.exposeMode).toBe("localhost");
+    // hub#616: the vault-provisioning op is polled over the session-authed
+    // wizard surface, NOT the Bearer-gated /api/modules/operations/:id (which
+    // the fake 401s, mirroring the real gate). At least one poll must land on
+    // /admin/setup?op=, and every poll must use that path.
+    expect(state.polled.length).toBeGreaterThan(0);
+    for (const p of state.polled) expect(p.startsWith("/admin/setup?op=")).toBe(true);
+  });
+
+  test("multi-tick vault op (running → succeeded) polls the session surface across ticks — hub#616", async () => {
+    // Models the import-mode long-running provisioning path: the op reports
+    // `running` for two poll ticks before flipping to `succeeded`.
+    const { state, fetchImpl } = makeFakeHub({ vaultProvisionTicks: 2 });
+    const logs: string[] = [];
+    const code = await runCliWizard({
+      hubUrl: "http://127.0.0.1:1939",
+      log: (l) => logs.push(l),
+      fetchImpl,
+      sleep: async () => {},
+      accountUsername: "admin",
+      accountPassword: "longpassword",
+      vaultMode: "create",
+      vaultName: "default",
+      exposeMode: "localhost",
+    });
+    expect(code).toBe(0);
+    // The loop ran more than once (2 running ticks + the terminal succeeded
+    // poll), and every tick used the session-authed surface — never the
+    // Bearer-gated /api/modules/operations/:id.
+    expect(state.polled.length).toBeGreaterThanOrEqual(3);
+    for (const p of state.polled) expect(p.startsWith("/admin/setup?op=")).toBe(true);
+    expect(state.exposeMode).toBe("localhost");
   });
 
   test("loopback-probe bootstrap token is sent transparently (no prompt) — hub#576", async () => {

package/src/commands/serve.ts

@@ -34,7 +34,7 @@ import { generateBootstrapToken } from "../bootstrap-token.ts";
 // path isolation.
 import { CONFIG_DIR, SERVICES_MANIFEST_PATH } from "../config.ts";
 import { readExposeState } from "../expose-state.ts";
-import { createDbHolder } from "../hub-db-liveness.ts";
+import { createDbHolder, defaultStatInode, startDbPathLivenessTimer } from "../hub-db-liveness.ts";
 import { hubDbPath, openHubDb } from "../hub-db.ts";
 import { hubFetch } from "../hub-server.ts";
 import { writeHubFile } from "../hub.ts";
@@ -304,6 +304,80 @@ export function formatBootstrapTokenBanner(token: string, hubUrl?: string): stri
   ].join("\n");
 }
 
+/**
+ * Injectable seams for {@link armServeDbWatchdog} (test-only). Generic on the
+ * timer handle `H` so the scheduler seams never name `setInterval` in type
+ * position — mirrors `DbLivenessTimerDeps<H>` in hub-db-liveness.ts, which
+ * keeps the public interface portable to a types-less tsc environment.
+ */
+export interface ServeDbWatchdogDeps<H = unknown> {
+  log?: (line: string) => void;
+  /** Open a db handle (default {@link openHubDb}). Tests inject a fake that creates a fixture. */
+  openDb?: (path: string) => ReturnType<typeof openHubDb>;
+  /** Path stat for the inode snapshot + proactive probe (default {@link defaultStatInode}). */
+  statInode?: typeof defaultStatInode;
+  /** Injectable scheduler threaded to the liveness timer (default `setInterval`). */
+  setIntervalFn?: (cb: () => void, ms: number) => H;
+  /** Injectable clear threaded to the liveness timer (default `clearInterval`). */
+  clearIntervalFn?: (handle: H) => void;
+  /** Process-exit fn threaded into the holder's reopen-or-exit (default `process.exit`). */
+  exit?: (code: number) => void;
+}
+
+/**
+ * Build the self-heal DB holder (#594) + start the proactive ghost-fd watchdog
+ * (#610) for the `parachute serve` path, returning both so the caller wires
+ * `getDb`/`onDbError`/`probeDbPath` and stops the timer on shutdown.
+ *
+ * Extracted + exported so the wiring is unit-testable WITHOUT binding a real
+ * port (#619): a serve()-level test would have to `Bun.serve` and risk the
+ * hub#535 launchd-bootout hazard, so this pure helper carries the load-bearing
+ * invariants instead — (1) the db is OPENED before the inode is snapshotted, so
+ * a fresh-install first boot gets a defined baseline (an ENOENT snapshot would
+ * silently disable the proactive probe for the whole process lifetime), and
+ * (2) the liveness timer is actually started. Both were absent on this path
+ * before #619 — the watchdog was wired only into `createHubServer`.
+ */
+export function armServeDbWatchdog<H = unknown>(
+  dbPath: string,
+  deps: ServeDbWatchdogDeps<H> = {},
+): {
+  dbHolder: ReturnType<typeof createDbHolder>;
+  livenessTimer: ReturnType<typeof startDbPathLivenessTimer>;
+} {
+  const openDb = deps.openDb ?? openHubDb;
+  const statInode = deps.statInode ?? defaultStatInode;
+  // Open FIRST — `openHubDb` mkdir's + creates the file when absent, so the
+  // stat below sees a real inode on a fresh-install first boot. Reversing this
+  // would leave `initialInode` undefined (ENOENT) and the probe at "unknown"
+  // for the process lifetime. Mirrors `createHubServer`'s ordering.
+  const db = openDb(dbPath);
+  let initialInode: ReturnType<typeof defaultStatInode> | undefined;
+  try {
+    initialInode = statInode(dbPath);
+  } catch {
+    initialInode = undefined;
+  }
+  const dbHolder = createDbHolder(db, {
+    reopen: () => openDb(dbPath),
+    dbPath,
+    statInode,
+    initialInode,
+    ...(deps.log !== undefined ? { log: deps.log } : {}),
+    ...(deps.exit !== undefined ? { exit: deps.exit } : {}),
+  });
+  // The active `parachute serve` path (systemd / launchd / container ExecStart)
+  // MUST start the watchdog here, not only in `createHubServer` — else a
+  // `rm -rf ~/.parachute` under a running unit leaves a ghost fd that keeps
+  // SELECT 1 succeeding with no thrown error, the reactive path never fires,
+  // and the hub never self-recovers (#619).
+  const livenessTimer = startDbPathLivenessTimer<H>(dbHolder, {
+    ...(deps.setIntervalFn !== undefined ? { setIntervalFn: deps.setIntervalFn } : {}),
+    ...(deps.clearIntervalFn !== undefined ? { clearIntervalFn: deps.clearIntervalFn } : {}),
+  });
+  return { dbHolder, livenessTimer };
+}
+
 /**
  * Run the hub fetch loop in the foreground. Resolves when `Bun.serve` is
  * bound; the returned `stop()` shuts the server down for tests.
@@ -355,15 +429,8 @@ export async function serve(opts: ServeOpts = {}): Promise<{
   writeHubFile(hubHtmlPath);
 
   const dbPath = hubDbPath();
-  // Self-heal-or-die DB holder (#594). The handle lives behind a mutable
-  // holder so a request that hits the persistent-corruption class (disk I/O
-  // error / malformed image — e.g. the state dir deleted under a running hub)
-  // can reopen the handle once, or exit(1) for the platform manager to restart
-  // us with a fresh one. `getDb` reads the current handle from the holder.
-  const dbHolder = createDbHolder(openHubDb(dbPath), {
-    reopen: () => openHubDb(dbPath),
-    log,
-  });
+  // Self-heal-or-die DB holder (#594) + proactive ghost-fd watchdog (#610/#619).
+  const { dbHolder, livenessTimer } = armServeDbWatchdog(dbPath, { log });
   const adminBootstrap = await seedInitialAdminIfNeeded(dbHolder.get(), env, log);
 
   if (adminBootstrap === "needs-setup") {
@@ -401,6 +468,9 @@ export async function serve(opts: ServeOpts = {}): Promise<{
       fetch: hubFetch(WELL_KNOWN_DIR, {
         getDb: () => dbHolder.get(),
         onDbError: (err) => dbHolder.healOrExit(err),
+        // #610: /health's db check probes the path so monitoring + the #591
+        // adoption probe see a wipe instead of the ghost-fd lie.
+        probeDbPath: () => dbHolder.probePath(),
         issuer,
         loopbackPort: port,
         supervisor,
@@ -486,6 +556,7 @@ export async function serve(opts: ServeOpts = {}): Promise<{
       for (const state of supervisor.list()) {
         await supervisor.stop(state.short);
       }
+      livenessTimer.stop();
       await server.stop();
       dbHolder.get().close();
     },

package/src/commands/wizard.ts

@@ -347,9 +347,15 @@ async function pollOperation(
   const start = Date.now();
   let lastLogIndex = 0;
   for (;;) {
+    // hub#616: poll over the session-authed wizard surface (`/admin/setup?op=`),
+    // mirroring the browser wizard's re-GET — NOT the Bearer-gated
+    // `/api/modules/operations/:id` the SPA + install CLI use. Mid-setup the
+    // wizard holds only a session cookie; the op endpoint demands a host-admin
+    // Bearer it doesn't have, so a direct poll 401s and the vault step dies.
+    // The op snapshot rides back in the envelope's `operation` field.
     const res = await setupFetch(
       hubUrl,
-      `/api/modules/operations/${encodeURIComponent(opId)}`,
+      `/admin/setup?op=${encodeURIComponent(opId)}`,
       jar,
       fetchImpl,
     );
@@ -358,10 +364,11 @@ async function pollOperation(
         `op-poll failed (${res.status}) for ${shortLabel} op ${opId}: ${res.bodyText.slice(0, 200)}`,
       );
     }
-    const body = res.json as Partial<OperationSnapshot> | undefined;
+    const envelope = res.json as { operation?: Partial<OperationSnapshot> } | undefined;
+    const body = envelope?.operation;
     if (!body || typeof body !== "object" || typeof body.id !== "string") {
       throw new Error(
-        `op-poll returned unexpected body for ${shortLabel} op ${opId}: ${res.bodyText.slice(0, 200)}`,
+        `op-poll returned no operation snapshot for ${shortLabel} op ${opId}: ${res.bodyText.slice(0, 200)}`,
       );
     }
     // Print any new log lines since the last tick so the operator sees

package/src/setup-wizard.ts

@@ -1640,6 +1640,12 @@ export function handleSetupGet(req: Request, deps: SetupWizardDeps): Response {
       requireBootstrapToken: boolean;
       csrfToken: string;
       bootstrapToken?: string;
+      operation?: {
+        id: string;
+        status: "pending" | "running" | "succeeded" | "failed";
+        log: readonly string[];
+        error?: string;
+      };
     } = {
       step: state.step,
       hasAdmin: state.hasAdmin,
@@ -1648,6 +1654,25 @@ export function handleSetupGet(req: Request, deps: SetupWizardDeps): Response {
       requireBootstrapToken: requireToken,
       csrfToken: csrf.token,
     };
+    // hub#616: the CLI wizard polls vault-provisioning over THIS session-authed
+    // surface (mirroring the browser wizard's `/admin/setup?op=<id>` re-GET),
+    // not the Bearer-gated `/api/modules/operations/:id` the SPA + install CLI
+    // use. The wizard holds only a session cookie mid-setup; the op endpoint
+    // requires a host-admin Bearer it doesn't have, so a direct poll 401s and
+    // the vault step dies. Threading the op snapshot into the envelope keeps the
+    // poll on the auth the wizard already carries.
+    const opId = url.searchParams.get("op");
+    if (opId) {
+      const op = deps.registry?.get(opId);
+      if (op) {
+        envelope.operation = {
+          id: op.id,
+          status: op.status,
+          log: op.log,
+          ...(op.error !== undefined ? { error: op.error } : {}),
+        };
+      }
+    }
     // hub#576: hand the actual token to a LOOPBACK caller only. The on-box
     // operator (`parachute init` → CLI wizard, or a curl from their own shell)
     // already proves box access by reaching loopback — same trust level as