@openparachute/hub 0.6.4 → 0.6.5-rc.2

package/src/commands/expose-cloudflare.ts

@@ -1,5 +1,5 @@
 import { spawnSync } from "node:child_process";
-import { mkdirSync, openSync } from "node:fs";
+import { existsSync, mkdirSync, openSync } from "node:fs";
 import { dirname } from "node:path";
 import {
   DEFAULT_TUNNEL_NAME,
@@ -37,8 +37,10 @@ import {
   type Tunnel,
   createTunnel,
   credentialsPath,
+  deleteTunnel,
   findTunnelByName,
   routeDns,
+  tunnelConnectionCount,
 } from "../cloudflare/tunnel.ts";
 import { CONFIG_DIR, SERVICES_MANIFEST_PATH } from "../config.ts";
 import {
@@ -267,6 +269,48 @@ export function looksLikeCloudflare(addresses: readonly string[]): boolean {
   return false;
 }
 
+/**
+ * Poll for the spawned connector establishing a live edge connection, bounded
+ * by `timeoutMs` (#593). Resolves true the first time `cloudflared tunnel
+ * info` reports ≥1 connector connection; false when the budget elapses with
+ * none. The pid existing is NOT proof of connection — the field repro had a
+ * live pid crash-looping on a missing creds file, every request returning
+ * Cloudflare error 1033. This is the loud verification that turns that silent
+ * false-success into an actionable failure.
+ *
+ * Injectable so tests drive both branches deterministically without a real
+ * cloudflared. Production uses `defaultVerifyConnection` (a bounded
+ * `tunnelConnectionCount` poll).
+ */
+export type VerifyConnectionFn = (args: {
+  runner: Runner;
+  tunnelName: string;
+  timeoutMs: number;
+  pollMs: number;
+  sleep: (ms: number) => Promise<void>;
+}) => Promise<boolean>;
+
+export const defaultVerifyConnection: VerifyConnectionFn = async ({
+  runner,
+  tunnelName,
+  timeoutMs,
+  pollMs,
+  sleep,
+}) => {
+  const deadline = Date.now() + timeoutMs;
+  // Probe immediately, then poll until a connector registers or the budget
+  // elapses. `tunnelConnectionCount` swallows its own CLI/parse errors → 0, so
+  // a not-yet-ready connector just costs another poll. Worst case is roughly
+  // ceil(timeoutMs / pollMs) iterations (each = one `cloudflared tunnel info`
+  // call + one sleep) before the deadline check returns false — with the
+  // production defaults (25_000 / 1_000) that's ~25 probes over ~25s.
+  for (;;) {
+    if ((await tunnelConnectionCount(runner, tunnelName)) > 0) return true;
+    if (Date.now() >= deadline) return false;
+    await sleep(pollMs);
+  }
+};
+
 export interface ExposeCloudflareOpts {
   runner?: Runner;
   spawner?: CloudflaredSpawner;
@@ -307,6 +351,23 @@ export interface ExposeCloudflareOpts {
    * Tests inject a stub; production uses `defaultResolveHost` (Bun DNS).
    */
   resolveHost?: ResolveHostFn;
+  /**
+   * Verify the spawned connector actually established an edge connection
+   * before claiming "✓ Cloudflare tunnel up" (#593). Production polls
+   * `cloudflared tunnel info` for a live connector (bounded). Tests inject a
+   * stub to drive the success / timeout branches without a real cloudflared.
+   * Returns true once at least one connection is live, false on timeout.
+   * Default policy mirrors `connectorPids`/`resolveHost`: when a test injects a
+   * stub `spawner` (and no explicit seam), default to an inert "connected"
+   * stub so existing stub-spawner suites don't have to model the probe.
+   */
+  verifyConnection?: VerifyConnectionFn;
+  /** Connection-verify budget in ms (default 25_000). */
+  verifyTimeoutMs?: number;
+  /** Poll interval for the connection-verify probe in ms (default 1_000). */
+  verifyPollMs?: number;
+  /** Sleep between connection-verify polls. Tests pin to a no-op. */
+  sleep?: (ms: number) => Promise<void>;
   log?: (line: string) => void;
   manifestPath?: string;
   statePath?: string;
@@ -402,6 +463,10 @@ interface Resolved {
   }) => InstallResult;
   removeService: (args: { tunnelName: string }) => RemoveResult;
   resolveHost: ResolveHostFn;
+  verifyConnection: VerifyConnectionFn;
+  verifyTimeoutMs: number;
+  verifyPollMs: number;
+  sleep: (ms: number) => Promise<void>;
   log: (line: string) => void;
   manifestPath: string;
   statePath: string;
@@ -488,6 +553,17 @@ function resolve(opts: ExposeCloudflareOpts, tunnelNameDefault: string): Resolve
     resolveHost:
       opts.resolveHost ??
       (opts.spawner === undefined ? defaultResolveHost : async () => ["104.16.0.1"]),
+    // Connection-verify seam (#593). Same defaulting policy as
+    // `connectorPids`/`resolveHost`: when a test injects a stub `spawner` (and
+    // no explicit seam), default to an inert "connected" stub so existing
+    // stub-spawner suites don't have to model the `tunnel info` probe.
+    // Production (no spawner override) gets the real bounded poll.
+    verifyConnection:
+      opts.verifyConnection ??
+      (opts.spawner === undefined ? defaultVerifyConnection : async () => true),
+    verifyTimeoutMs: opts.verifyTimeoutMs ?? 25_000,
+    verifyPollMs: opts.verifyPollMs ?? 1_000,
+    sleep: opts.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))),
     log: opts.log ?? ((line) => console.log(line)),
     manifestPath: opts.manifestPath ?? SERVICES_MANIFEST_PATH,
     statePath: opts.statePath ?? CLOUDFLARED_STATE_PATH,
@@ -694,7 +770,50 @@ export async function exposeCloudflareUp(
       "  Each machine gets its own dedicated tunnel — you don't need to run `cloudflared tunnel create` separately; expose does it.",
     );
   } else {
-    r.log(`✓ Reusing existing tunnel "${r.tunnelName}" (${tunnel.id})`);
+    // Reuse-path credentials verification + self-heal (#593). `findTunnelByName`
+    // only proves the tunnel exists ACCOUNT-side. The connector needs the LOCAL
+    // credentials file (`~/.cloudflared/<uuid>.json`, written at `tunnel create`
+    // time) to authenticate — and that file gets lost on clean-slate flows
+    // (`rm -rf ~/.parachute` and friends) while the account-side tunnel
+    // survives. The field repro: tunnel reused, "✓ tunnel up" printed, connector
+    // crash-looping on "credentials file … doesn't exist", every request → 1033.
+    //
+    // If the creds file is missing we recreate the tunnel: delete the
+    // account-side tunnel by name (`--force`, so a stale registered connector
+    // doesn't block it), then `tunnel create` re-writes a fresh creds file. The
+    // new tunnel gets a new UUID; `routeDns` below uses `--overwrite-dns`, so the
+    // hostname's CNAME is repointed at the new UUID even though it pointed at the
+    // old one. The field case confirmed `tunnel delete` + re-run heals cleanly.
+    const existingCreds = credentialsPath(tunnel.id, r.cloudflaredHome);
+    if (existsSync(existingCreds)) {
+      r.log(`✓ Reusing existing tunnel "${r.tunnelName}" (${tunnel.id})`);
+    } else {
+      r.log(
+        `⚠ Tunnel "${r.tunnelName}" (${tunnel.id}) exists in Cloudflare, but its local credentials`,
+      );
+      r.log(`  file is missing (${existingCreds}) — the connector can't authenticate from this`);
+      r.log("  machine. Recreating the tunnel so a fresh credentials file is written…");
+      try {
+        await deleteTunnel(r.runner, r.tunnelName);
+      } catch (err) {
+        if (err instanceof CloudflaredError) {
+          r.log("");
+          r.log(`✗ Couldn't delete the stale tunnel automatically: ${err.message}`);
+          r.log("");
+          r.log("Recover manually, then re-run this command:");
+          r.log(`    cloudflared tunnel delete --force ${r.tunnelName}`);
+          r.log(`    parachute expose public --cloudflare --domain ${hostname}`);
+          return 1;
+        }
+        throw err;
+      }
+      try {
+        tunnel = await createTunnel(r.runner, r.tunnelName);
+      } catch (err) {
+        return reportCloudflaredError(err, r.log);
+      }
+      r.log(`✓ Recreated tunnel ${tunnel.id} (fresh credentials written).`);
+    }
   }
 
   r.log(`Routing DNS: ${hostname} → tunnel ${tunnel.id}…`);
@@ -955,6 +1074,42 @@ export async function exposeCloudflareUp(
     }
   }
 
+  // Post-start connection verification (#593). The connector pid existing is
+  // NOT proof it connected — the field repro had a live pid crash-looping on a
+  // missing creds file, with every public request returning Cloudflare error
+  // 1033 (tunnel registered, no connector) while the CLI printed "✓ tunnel up".
+  // Poll `cloudflared tunnel info` for a live edge connection, bounded. On
+  // timeout, fail LOUDLY with the connector log path + the crash-loop signature
+  // to grep for, instead of claiming success.
+  r.log("");
+  r.log("Verifying the connector established a tunnel connection…");
+  const connected = await r.verifyConnection({
+    runner: r.runner,
+    tunnelName: r.tunnelName,
+    timeoutMs: r.verifyTimeoutMs,
+    pollMs: r.verifyPollMs,
+    sleep: r.sleep,
+  });
+  if (!connected) {
+    r.log("");
+    r.log(
+      `✗ The cloudflared connector (pid ${pid}) started but never registered a tunnel connection`,
+    );
+    r.log(`  within ${Math.round(r.verifyTimeoutMs / 1000)}s. Public requests to ${hostname} will`);
+    r.log("  return Cloudflare error 1033 (tunnel registered, no connector) until this resolves.");
+    r.log("");
+    r.log("Check the connector log for the crash-loop cause:");
+    r.log(`    tail -n 50 ${r.logPath}`);
+    r.log('  A repeating "credentials file … doesn\'t exist" line means the local credentials are');
+    r.log(
+      "  gone — re-run this command (it auto-recreates the tunnel). Other repeating errors point",
+    );
+    r.log("  at the specific failure. Confirm the connector once it's healthy with:");
+    r.log(`    cloudflared tunnel info ${r.tunnelName}`);
+    return 1;
+  }
+  r.log("✓ Connector connected.");
+
   const baseUrl = `https://${hostname}`;
   let vaultUrl: string | undefined;
   if (vaultEntry) {

package/src/commands/serve.ts

@@ -34,6 +34,7 @@ import { generateBootstrapToken } from "../bootstrap-token.ts";
 // path isolation.
 import { CONFIG_DIR, SERVICES_MANIFEST_PATH } from "../config.ts";
 import { readExposeState } from "../expose-state.ts";
+import { createDbHolder } from "../hub-db-liveness.ts";
 import { hubDbPath, openHubDb } from "../hub-db.ts";
 import { hubFetch } from "../hub-server.ts";
 import { writeHubFile } from "../hub.ts";
@@ -345,8 +346,16 @@ export async function serve(opts: ServeOpts = {}): Promise<{
   if (!existsSync(hubHtmlPath)) writeHubFile(hubHtmlPath);
 
   const dbPath = hubDbPath();
-  const db = openHubDb(dbPath);
-  const adminBootstrap = await seedInitialAdminIfNeeded(db, env, log);
+  // Self-heal-or-die DB holder (#594). The handle lives behind a mutable
+  // holder so a request that hits the persistent-corruption class (disk I/O
+  // error / malformed image — e.g. the state dir deleted under a running hub)
+  // can reopen the handle once, or exit(1) for the platform manager to restart
+  // us with a fresh one. `getDb` reads the current handle from the holder.
+  const dbHolder = createDbHolder(openHubDb(dbPath), {
+    reopen: () => openHubDb(dbPath),
+    log,
+  });
+  const adminBootstrap = await seedInitialAdminIfNeeded(dbHolder.get(), env, log);
 
   if (adminBootstrap === "needs-setup") {
     log(
@@ -381,7 +390,8 @@ export async function serve(opts: ServeOpts = {}): Promise<{
       // CMD), so the fix has to land here too. Closes hub#399.
       idleTimeout: 255,
       fetch: hubFetch(WELL_KNOWN_DIR, {
-        getDb: () => db,
+        getDb: () => dbHolder.get(),
+        onDbError: (err) => dbHolder.healOrExit(err),
         issuer,
         loopbackPort: port,
         supervisor,
@@ -468,7 +478,7 @@ export async function serve(opts: ServeOpts = {}): Promise<{
         await supervisor.stop(state.short);
       }
       await server.stop();
-      db.close();
+      dbHolder.get().close();
     },
   };
 }

package/src/hub-db-liveness.ts

@@ -0,0 +1,211 @@
+import type { Database } from "bun:sqlite";
+
+/**
+ * SQLite-handle liveness + self-heal policy (#594).
+ *
+ * Field repro: an operator deleted `~/.parachute` while the hub unit was
+ * running. The process kept an fd to the now-unlinked `hub.db` inode — cached
+ * reads half-worked, every write / WAL op threw `SQLiteError: disk I/O error`.
+ * Result: `/health` stayed 200 (it never touched the DB), every DB-touching
+ * route 500'd indefinitely, and operator-facing CLI checks lied (served from
+ * the dead handle's cached pages). An hour of clean 500s behind a green
+ * /health is the worst possible failure shape — a crash-restart would have
+ * self-healed in seconds (the platform manager re-`openHubDb`s a fresh handle).
+ *
+ * The policy here: on a request that hits the persistent-corruption error
+ * class, attempt ONE reopen of the handle; if reopen fails OR the error
+ * recurs immediately, log loudly and `process.exit(1)` so the platform
+ * manager (launchd / systemd / container runtime) restarts with a fresh
+ * handle. We are careful to scope "fatal" to the persistent class — a
+ * transient `SQLITE_BUSY` (a momentary write lock) must NOT kill the hub.
+ */
+
+/**
+ * How a thrown DB error should be treated.
+ *   - `fatal`     → persistent corruption / dead handle (disk I/O error,
+ *                   database disk image is malformed, NOTADB, CORRUPT, IOERR).
+ *                   Triggers the reopen-once-or-exit machinery.
+ *   - `transient` → a momentary lock (SQLITE_BUSY / SQLITE_LOCKED). Never
+ *                   fatal; the caller surfaces it as an ordinary error and
+ *                   the next request likely succeeds.
+ *   - `other`     → not a recognized SQLite-handle failure (e.g. a constraint
+ *                   violation, a programming error). Not the liveness concern;
+ *                   the caller handles it as a normal error.
+ */
+export type DbErrorClass = "fatal" | "transient" | "other";
+
+/**
+ * Pull a lowercase "<code> <message>" string out of an unknown thrown value
+ * for substring matching. `bun:sqlite` throws `SQLiteError` with a `code`
+ * (e.g. `SQLITE_IOERR`, `SQLITE_BUSY`) and a `message` (e.g. "disk I/O
+ * error"). We match on both so a runtime that surfaces one but not the other
+ * still classifies correctly.
+ */
+function errorSignature(err: unknown): string {
+  if (err && typeof err === "object") {
+    const e = err as { code?: unknown; message?: unknown; name?: unknown };
+    const code = typeof e.code === "string" ? e.code : "";
+    const message = typeof e.message === "string" ? e.message : "";
+    const name = typeof e.name === "string" ? e.name : "";
+    return `${code} ${name} ${message}`.toLowerCase();
+  }
+  return String(err).toLowerCase();
+}
+
+/**
+ * Classify a thrown DB error. Order matters: a transient BUSY/LOCKED is
+ * checked FIRST so it's never mistaken for the fatal class, even if a future
+ * message happened to share a substring.
+ */
+export function classifyDbError(err: unknown): DbErrorClass {
+  const sig = errorSignature(err);
+  if (sig.length === 0) return "other";
+
+  // Transient locks — explicitly NON-fatal. SQLITE_BUSY is a momentary write
+  // lock under WAL contention; killing the hub on it would turn ordinary
+  // concurrency into a restart loop. SQLITE_LOCKED is the same class.
+  if (sig.includes("sqlite_busy") || sig.includes("sqlite_locked")) return "transient";
+  if (/\bdatabase is locked\b/.test(sig) || /\bdatabase table is locked\b/.test(sig)) {
+    return "transient";
+  }
+  // A handful of SQLITE_IOERR *sub-codes* are contention, not corruption:
+  // SQLITE_IOERR_BLOCKED (a legacy busy variant) and SQLITE_IOERR_LOCK (a
+  // lock-acquisition failure). The generic `sqlite_ioerr` substring match
+  // below would otherwise sweep these into the fatal class and exit the hub on
+  // transient I/O contention. Check them FIRST so they classify as transient.
+  if (sig.includes("sqlite_ioerr_blocked") || sig.includes("sqlite_ioerr_lock")) {
+    return "transient";
+  }
+
+  // Persistent-corruption / dead-handle class → fatal (reopen-once-or-exit).
+  // `disk I/O error` is the exact field message (state dir deleted under a
+  // running hub); the malformed-image + corrupt + notadb codes are the
+  // related on-disk-corruption shapes the issue calls out.
+  //
+  // `sqlite_ioerr` matches the GENERIC `SQLITE_IOERR` code, which is what Bun
+  // surfaces for the dead-handle case (the unlinked-inode field repro reports
+  // exactly `code: "SQLITE_IOERR", message: "disk I/O error"`, not a
+  // sub-code). The two transient IOERR sub-codes are already filtered out
+  // above, so reaching this `includes` means either the generic code or a
+  // corruption sub-code — both fatal. (`disk i/o error` is also matched
+  // directly so a runtime that surfaces the message but not the code still
+  // classifies.)
+  if (
+    sig.includes("disk i/o error") ||
+    sig.includes("sqlite_ioerr") ||
+    sig.includes("database disk image is malformed") ||
+    sig.includes("sqlite_corrupt") ||
+    sig.includes("sqlite_notadb") ||
+    sig.includes("file is not a database")
+  ) {
+    return "fatal";
+  }
+
+  return "other";
+}
+
+/**
+ * Cheap DB liveness probe for `/health` (#594). Runs `SELECT 1`. Returns
+ * `"ok"` on success, or `"error: <class>"` where class is the
+ * {@link classifyDbError} verdict, so a monitor can tell "hub up but DB dead"
+ * apart from "hub up, DB fine". NEVER throws — a probe that threw would make
+ * /health itself 500, defeating the point (/health must stay fast + reliable).
+ */
+export function probeDbLiveness(db: Database): "ok" | string {
+  try {
+    db.query("SELECT 1").get();
+    return "ok";
+  } catch (err) {
+    return `error: ${classifyDbError(err)}`;
+  }
+}
+
+/**
+ * A mutable holder for the hub's `Database` handle so a request handler that
+ * hits the fatal error class can swap in a freshly-reopened handle without
+ * re-threading the closure-captured `db` through every call site. `getDb()`
+ * in hub-server reads `holder.get()`; the self-heal path calls
+ * `holder.healOrExit(err)`.
+ */
+export interface DbHolder {
+  /** The current live handle. */
+  get(): Database;
+  /**
+   * React to a thrown DB error per the liveness policy:
+   *   - `transient`/`other` → return `"ignored"` (caller surfaces a normal error).
+   *   - `fatal`, reopen succeeds + a `SELECT 1` passes on the new handle →
+   *     swap the handle in, return `"healed"` (caller retries / surfaces a
+   *     transient error the next request clears).
+   *   - `fatal`, reopen fails OR the new handle still fails `SELECT 1` →
+   *     log loudly + `exit(1)`. Returns `"exited"` only in tests (the injected
+   *     exit fn doesn't actually exit the process).
+   *
+   * Reopen-once semantics: a single fatal error triggers one reopen attempt.
+   * If the *reopened* handle is also dead (e.g. the underlying dir is still
+   * gone), we exit rather than loop — the platform manager owns the restart.
+   */
+  healOrExit(err: unknown): "ignored" | "healed" | "exited";
+}
+
+export interface DbHolderDeps {
+  /** Open a fresh handle (production: `() => openHubDb(dbPath)`). */
+  reopen: () => Database;
+  /** Loud log sink (default `console.error`). */
+  log?: (line: string) => void;
+  /** Process-exit fn (default `process.exit`; tests inject a spy). */
+  exit?: (code: number) => void;
+  /** Close a (presumed-dead) handle best-effort before swapping (default `db.close()`). */
+  closeOld?: (db: Database) => void;
+}
+
+/**
+ * Build a {@link DbHolder} over an initial handle. Production wires
+ * `reopen: () => openHubDb(dbPath)` and the default exit/log; tests inject a
+ * fake reopen + a non-exiting `exit` spy so the fatal branch is exercised
+ * without killing the test process.
+ */
+export function createDbHolder(initial: Database, deps: DbHolderDeps): DbHolder {
+  let current = initial;
+  const log = deps.log ?? ((line) => console.error(line));
+  const exit = deps.exit ?? ((code) => process.exit(code));
+  const closeOld =
+    deps.closeOld ??
+    ((db) => {
+      try {
+        db.close();
+      } catch {
+        // Best-effort — a dead handle may throw on close; we're replacing it.
+      }
+    });
+
+  return {
+    get: () => current,
+    healOrExit(err: unknown) {
+      const klass = classifyDbError(err);
+      if (klass !== "fatal") return "ignored";
+
+      const detail = err instanceof Error ? err.message : String(err);
+      log(`parachute hub: persistent SQLite failure (${detail}). Attempting one DB handle reopen…`);
+
+      let reopened: Database;
+      try {
+        reopened = deps.reopen();
+        // Confirm the fresh handle is actually live before trusting it.
+        reopened.query("SELECT 1").get();
+      } catch (reopenErr) {
+        const rd = reopenErr instanceof Error ? reopenErr.message : String(reopenErr);
+        log(
+          `parachute hub: DB reopen failed (${rd}); exiting so the platform manager restarts the hub with a fresh handle.`,
+        );
+        exit(1);
+        return "exited";
+      }
+
+      // Reopen succeeded + verified. Swap it in; the old handle is dead.
+      closeOld(current);
+      current = reopened;
+      log("parachute hub: DB handle reopened successfully; continuing.");
+      return "healed";
+    },
+  };
+}