@openparachute/hub 0.6.4 → 0.6.5-rc.1

package/src/__tests__/hub-server.test.ts

@@ -5,6 +5,7 @@ import { join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { buildCsrfCookie, generateCsrfToken } from "../csrf.ts";
 import { HUB_SVC, hubPortPath } from "../hub-control.ts";
+import { createDbHolder } from "../hub-db-liveness.ts";
 import { hubDbPath, openHubDb } from "../hub-db.ts";
 import {
   findServiceUpstream,
@@ -1240,14 +1241,10 @@ describe("hubFetch routing", () => {
   // open shouldn't cascade into a restart loop. The body advertises the
   // running version so a deploy verifier can confirm the rolled-out
   // image is the one it expected.
-  test("/health returns 200 JSON without invoking the db", async () => {
+  test("/health returns 200 JSON; unconfigured db field when no getDb", async () => {
     const h = makeHarness();
     try {
-      const res = await hubFetch(h.dir, {
-        getDb: () => {
-          throw new Error("getDb must not be called by /health");
-        },
-      })(req("/health"));
+      const res = await hubFetch(h.dir)(req("/health"));
       expect(res.status).toBe(200);
       expect(res.headers.get("content-type")).toContain("application/json");
       expect(res.headers.get("cache-control")).toBe("no-store");
@@ -1255,11 +1252,153 @@ describe("hubFetch routing", () => {
       expect(body.status).toBe("ok");
       expect(body.service).toBe("parachute-hub");
       expect(typeof body.version).toBe("string");
+      expect(body.db).toBe("unconfigured");
     } finally {
       h.cleanup();
     }
   });
 
+  test("/health db field is ok on a live db (#594)", async () => {
+    const h = makeHarness();
+    const db = openHubDb(hubDbPath(h.dir));
+    try {
+      const res = await hubFetch(h.dir, { getDb: () => db })(req("/health"));
+      expect(res.status).toBe(200);
+      const body = (await res.json()) as Record<string, unknown>;
+      expect(body.status).toBe("ok");
+      expect(body.db).toBe("ok");
+    } finally {
+      db.close();
+      h.cleanup();
+    }
+  });
+
+  test("/health db field reports error: <class> on a dead handle, still 200 (#594)", async () => {
+    const h = makeHarness();
+    const db = openHubDb(hubDbPath(h.dir));
+    db.close(); // dead handle — every SELECT throws
+    try {
+      const res = await hubFetch(h.dir, { getDb: () => db })(req("/health"));
+      // Always 200 while the process is up — the HTTP status is process
+      // liveness, the db field is the readiness signal.
+      expect(res.status).toBe(200);
+      const body = (await res.json()) as Record<string, unknown>;
+      expect(body.status).toBe("ok");
+      expect(typeof body.db).toBe("string");
+      expect((body.db as string).startsWith("error:")).toBe(true);
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("a fatal DB throw escaping a handler returns a structured body + invokes onDbError (#594)", async () => {
+    const h = makeHarness();
+    try {
+      const fatal = Object.assign(new Error("disk I/O error"), {
+        name: "SQLiteError",
+        code: "SQLITE_IOERR",
+      });
+      let onDbErrorCalls = 0;
+      // A non-/health, DB-touching route (`/`) whose getDb throws the fatal
+      // class. The top-level self-heal wrapper catches it, calls onDbError,
+      // and returns a structured db_unavailable body (not a bare 500).
+      const res = await hubFetch(h.dir, {
+        manifestPath: h.manifestPath,
+        getDb: () => {
+          throw fatal;
+        },
+        onDbError: () => {
+          onDbErrorCalls += 1;
+          return "healed";
+        },
+      })(req("/", { headers: { accept: "text/html" } }));
+      expect(onDbErrorCalls).toBe(1);
+      expect(res.status).toBe(503);
+      const body = (await res.json()) as Record<string, unknown>;
+      expect(body.error).toBe("db_unavailable");
+      expect(typeof body.error_description).toBe("string");
+      expect(body.error_description as string).toContain("reopened");
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("a non-DB throw still propagates (not swallowed by the self-heal wrapper) (#594)", async () => {
+    const h = makeHarness();
+    try {
+      let onDbErrorCalls = 0;
+      const handler = hubFetch(h.dir, {
+        manifestPath: h.manifestPath,
+        getDb: () => {
+          throw new Error("some unrelated programming error");
+        },
+        onDbError: () => {
+          onDbErrorCalls += 1;
+          return "ignored";
+        },
+      });
+      await expect(handler(req("/", { headers: { accept: "text/html" } }))).rejects.toThrow(
+        "some unrelated programming error",
+      );
+      expect(onDbErrorCalls).toBe(0);
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("a transient SQLITE_BUSY escaping a handler → 503, does NOT exit the hub (#594)", async () => {
+    const h = makeHarness();
+    const db = openHubDb(hubDbPath(h.dir));
+    try {
+      // Wire a REAL DbHolder so this pins the end-to-end transient path: the
+      // wrapper catches the throw, routes it to the holder's healOrExit, and
+      // the holder must classify SQLITE_BUSY as transient → "ignored" → NO
+      // reopen, NO exit. A spy `exit` that fails the test if ever called is the
+      // regression guard ("a momentary lock never kills the hub").
+      let exited = false;
+      let reopened = false;
+      const holder = createDbHolder(db, {
+        reopen: () => {
+          reopened = true;
+          return db;
+        },
+        exit: () => {
+          exited = true;
+        },
+        log: () => {},
+      });
+      const busy = Object.assign(new Error("database is locked"), {
+        name: "SQLiteError",
+        code: "SQLITE_BUSY",
+      });
+      // First getDb() (the wizard-redirect userCount read) throws BUSY; the
+      // wrapper's catch routes it through the holder.
+      let firstCall = true;
+      const res = await hubFetch(h.dir, {
+        manifestPath: h.manifestPath,
+        getDb: () => {
+          if (firstCall) {
+            firstCall = false;
+            throw busy;
+          }
+          return holder.get();
+        },
+        onDbError: (err) => holder.healOrExit(err),
+      })(req("/", { headers: { accept: "text/html" } }));
+      expect(res.status).toBe(503);
+      const body = (await res.json()) as Record<string, unknown>;
+      expect(body.error).toBe("db_unavailable");
+      // Transient class is named in the structured body, not "reopened".
+      expect(body.error_description as string).toContain("transient");
+      // The crux: the hub did NOT exit and did NOT reopen on a transient lock.
+      expect(exited).toBe(false);
+      expect(reopened).toBe(false);
+    } finally {
+      db.close();
+      h.cleanup();
+    }
+  });
+
   // First-boot setup wizard (hub#259, expanding hub#258's static
   // placeholder). When no admin exists, GET /admin/setup renders the
   // wizard's account-step form. Once admin + vault both exist, it 301s

package/src/__tests__/hub-unit.test.ts

@@ -44,7 +44,7 @@ function fakeDeps(
      * `null` (hub not answering) or `{ ok, version }`. Drives
      * `probeHealthVersion` across the version-check + post-restart re-probe.
      */
-    healthVersionSeq?: ({ ok: boolean; version?: string } | null)[];
+    healthVersionSeq?: ({ ok: boolean; version?: string; db?: string } | null)[];
     listeningSeq?: boolean[];
     installedUnit?: boolean;
   } = {},
@@ -756,4 +756,113 @@ describe("ensureHubVersionMatches — version-check-and-restart at adoption (#59
     expect(res.outcome).toBe("restart-failed");
     expect(res.messages.join("\n")).toContain("Unit parachute-hub.service not found.");
   });
+
+  // #594: a hub whose VERSION matches but whose /health reports a db fault
+  // (dead handle — state dir deleted under it) must be treated as needing a
+  // restart, through the same restart-once machinery.
+  test("version matches but /health reports db fault → restart-once → restarted when db heals", async () => {
+    const f = fakeDeps({
+      platform: "darwin",
+      getuid: () => 501,
+      installedUnit: true,
+      // first probe: right version but dead DB handle; after the restart the
+      // re-probe sees a live DB.
+      healthVersionSeq: [
+        { ok: true, version: INSTALLED, db: "error: fatal" },
+        { ok: true, version: INSTALLED, db: "ok" },
+      ],
+    });
+    const res = await ensureHubVersionMatches({
+      installedVersion: INSTALLED,
+      port: 1939,
+      deps: f.deps,
+      readyPollMs: 0,
+    });
+    expect(res.outcome).toBe("restarted");
+    const restarts = f.calls.filter((c) => c.includes("kickstart"));
+    expect(restarts).toHaveLength(1);
+  });
+
+  // #594: a SUSTAINED transient fault visible in /health (e.g. a write lock
+  // that never clears) is still an "error:" verdict, so the adoption probe
+  // treats it as needing a restart — same as the fatal case. Pins that
+  // `healthReportsDbFault` keys on the "error:" prefix, not the fatal class.
+  test("version matches but /health reports db error: transient → restart-once (#594)", async () => {
+    const f = fakeDeps({
+      platform: "darwin",
+      getuid: () => 501,
+      installedUnit: true,
+      // first probe: right version, sustained transient DB fault; after the
+      // restart the re-probe sees a live DB.
+      healthVersionSeq: [
+        { ok: true, version: INSTALLED, db: "error: transient" },
+        { ok: true, version: INSTALLED, db: "ok" },
+      ],
+    });
+    const res = await ensureHubVersionMatches({
+      installedVersion: INSTALLED,
+      port: 1939,
+      deps: f.deps,
+      readyPollMs: 0,
+    });
+    expect(res.outcome).toBe("restarted");
+    const restarts = f.calls.filter((c) => c.includes("kickstart"));
+    expect(restarts).toHaveLength(1);
+  });
+
+  test("version + db both ok → match, NO restart (#594 doesn't fire on a healthy hub)", async () => {
+    const f = fakeDeps({
+      platform: "darwin",
+      getuid: () => 501,
+      installedUnit: true,
+      healthVersionSeq: [{ ok: true, version: INSTALLED, db: "ok" }],
+    });
+    const res = await ensureHubVersionMatches({
+      installedVersion: INSTALLED,
+      port: 1939,
+      deps: f.deps,
+      readyPollMs: 0,
+    });
+    expect(res.outcome).toBe("match");
+    expect(f.calls).toEqual([]);
+  });
+
+  test("db fault persists after the restart → still-mismatched with a db-specific message (#594)", async () => {
+    const f = fakeDeps({
+      platform: "darwin",
+      getuid: () => 501,
+      installedUnit: true,
+      // Every probe reports the dead handle (state dir still gone). Restart
+      // once, then settle — no loop.
+      healthVersionSeq: [{ ok: true, version: INSTALLED, db: "error: fatal" }],
+    });
+    const res = await ensureHubVersionMatches({
+      installedVersion: INSTALLED,
+      port: 1939,
+      deps: f.deps,
+      readyTimeoutMs: 0,
+      readyPollMs: 0,
+    });
+    expect(res.outcome).toBe("still-mismatched");
+    const restarts = f.calls.filter((c) => c.includes("kickstart"));
+    expect(restarts).toHaveLength(1);
+    expect(res.messages.join("\n")).toContain("database still reports a fault");
+  });
+
+  test("a hub with NO db field (pre-#594) on a version match → match, not treated as a fault", async () => {
+    const f = fakeDeps({
+      platform: "darwin",
+      getuid: () => 501,
+      installedUnit: true,
+      healthVersionSeq: [{ ok: true, version: INSTALLED /* no db field */ }],
+    });
+    const res = await ensureHubVersionMatches({
+      installedVersion: INSTALLED,
+      port: 1939,
+      deps: f.deps,
+      readyPollMs: 0,
+    });
+    expect(res.outcome).toBe("match");
+    expect(f.calls).toEqual([]);
+  });
 });

package/src/cloudflare/tunnel.ts

@@ -133,3 +133,73 @@ export async function routeDns(
 export function credentialsPath(uuid: string, cloudflaredHome: string): string {
   return join(cloudflaredHome, `${uuid}.json`);
 }
+
+/**
+ * `cloudflared tunnel delete <name>` removes the account-side tunnel. Used by
+ * the reuse-path self-heal (#593): when an existing tunnel's local credentials
+ * file is missing, the tunnel is unusable from this machine — we delete the
+ * account-side tunnel and recreate it so `tunnel create` re-writes a fresh
+ * `~/.cloudflared/<uuid>.json`.
+ *
+ * `--force` makes the delete non-interactive and tears down any lingering
+ * connector record cloudflared still has registered for the tunnel — without
+ * it, `tunnel delete` refuses ("tunnel has active connections") when a stale
+ * connector is registered account-side, which is exactly the crash-loop state
+ * #593 self-heals. Deleting a tunnel with no live local connector is safe: the
+ * field repro showed `tunnel delete` + re-run worked cleanly.
+ */
+export async function deleteTunnel(runner: Runner, name: string): Promise<void> {
+  const cmd = ["cloudflared", "tunnel", "delete", "--force", name];
+  const result = await runner(cmd);
+  if (result.code !== 0) {
+    throw new CloudflaredError(
+      `cloudflared tunnel delete failed: ${combineErrStreams(result)}`,
+      cmd,
+      result,
+    );
+  }
+}
+
+/**
+ * Count the active connector connections cloudflared reports for a tunnel via
+ * `cloudflared tunnel info --output json <name>`. Used by the post-start
+ * connection verification (#593): a spawned connector pid existing ≠ the
+ * connector actually registered an edge connection (the error-1033 field
+ * repro — pid alive, connector crash-looping on a missing creds file, every
+ * request 1033).
+ *
+ * The JSON shape is `{ conns: [ { ... }, … ] }` (or a top-level `connections`
+ * array on some cloudflared versions). We count entries defensively across
+ * both shapes and treat any parse/CLI failure as `0` (not-yet-connected) — the
+ * caller polls, so a transient miss just costs one more poll. Returns the
+ * connector count; `> 0` means at least one edge connection is live.
+ */
+export async function tunnelConnectionCount(runner: Runner, name: string): Promise<number> {
+  const cmd = ["cloudflared", "tunnel", "info", "--output", "json", name];
+  let result: CommandResult;
+  try {
+    result = await runner(cmd);
+  } catch {
+    return 0;
+  }
+  if (result.code !== 0) return 0;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(result.stdout);
+  } catch {
+    return 0;
+  }
+  if (!parsed || typeof parsed !== "object") return 0;
+  const obj = parsed as Record<string, unknown>;
+  // `cloudflared tunnel info --output json` reports per-connector entries under
+  // `conns` on current versions; older shapes used a flat `connections` array.
+  // Count whichever is present.
+  const conns = obj.conns ?? obj.connections;
+  if (Array.isArray(conns)) {
+    // Each entry may itself carry a nested `conns` array (per-colo connector
+    // detail). Count an entry as a live connection when it exists; that's the
+    // signal we need ("the connector registered at least one edge connection").
+    return conns.length;
+  }
+  return 0;
+}

package/src/commands/expose-cloudflare.ts

@@ -1,5 +1,5 @@
 import { spawnSync } from "node:child_process";
-import { mkdirSync, openSync } from "node:fs";
+import { existsSync, mkdirSync, openSync } from "node:fs";
 import { dirname } from "node:path";
 import {
   DEFAULT_TUNNEL_NAME,
@@ -37,8 +37,10 @@ import {
   type Tunnel,
   createTunnel,
   credentialsPath,
+  deleteTunnel,
   findTunnelByName,
   routeDns,
+  tunnelConnectionCount,
 } from "../cloudflare/tunnel.ts";
 import { CONFIG_DIR, SERVICES_MANIFEST_PATH } from "../config.ts";
 import {
@@ -267,6 +269,48 @@ export function looksLikeCloudflare(addresses: readonly string[]): boolean {
   return false;
 }
 
+/**
+ * Poll for the spawned connector establishing a live edge connection, bounded
+ * by `timeoutMs` (#593). Resolves true the first time `cloudflared tunnel
+ * info` reports ≥1 connector connection; false when the budget elapses with
+ * none. The pid existing is NOT proof of connection — the field repro had a
+ * live pid crash-looping on a missing creds file, every request returning
+ * Cloudflare error 1033. This is the loud verification that turns that silent
+ * false-success into an actionable failure.
+ *
+ * Injectable so tests drive both branches deterministically without a real
+ * cloudflared. Production uses `defaultVerifyConnection` (a bounded
+ * `tunnelConnectionCount` poll).
+ */
+export type VerifyConnectionFn = (args: {
+  runner: Runner;
+  tunnelName: string;
+  timeoutMs: number;
+  pollMs: number;
+  sleep: (ms: number) => Promise<void>;
+}) => Promise<boolean>;
+
+export const defaultVerifyConnection: VerifyConnectionFn = async ({
+  runner,
+  tunnelName,
+  timeoutMs,
+  pollMs,
+  sleep,
+}) => {
+  const deadline = Date.now() + timeoutMs;
+  // Probe immediately, then poll until a connector registers or the budget
+  // elapses. `tunnelConnectionCount` swallows its own CLI/parse errors → 0, so
+  // a not-yet-ready connector just costs another poll. Worst case is roughly
+  // ceil(timeoutMs / pollMs) iterations (each = one `cloudflared tunnel info`
+  // call + one sleep) before the deadline check returns false — with the
+  // production defaults (25_000 / 1_000) that's ~25 probes over ~25s.
+  for (;;) {
+    if ((await tunnelConnectionCount(runner, tunnelName)) > 0) return true;
+    if (Date.now() >= deadline) return false;
+    await sleep(pollMs);
+  }
+};
+
 export interface ExposeCloudflareOpts {
   runner?: Runner;
   spawner?: CloudflaredSpawner;
@@ -307,6 +351,23 @@ export interface ExposeCloudflareOpts {
    * Tests inject a stub; production uses `defaultResolveHost` (Bun DNS).
    */
   resolveHost?: ResolveHostFn;
+  /**
+   * Verify the spawned connector actually established an edge connection
+   * before claiming "✓ Cloudflare tunnel up" (#593). Production polls
+   * `cloudflared tunnel info` for a live connector (bounded). Tests inject a
+   * stub to drive the success / timeout branches without a real cloudflared.
+   * Returns true once at least one connection is live, false on timeout.
+   * Default policy mirrors `connectorPids`/`resolveHost`: when a test injects a
+   * stub `spawner` (and no explicit seam), default to an inert "connected"
+   * stub so existing stub-spawner suites don't have to model the probe.
+   */
+  verifyConnection?: VerifyConnectionFn;
+  /** Connection-verify budget in ms (default 25_000). */
+  verifyTimeoutMs?: number;
+  /** Poll interval for the connection-verify probe in ms (default 1_000). */
+  verifyPollMs?: number;
+  /** Sleep between connection-verify polls. Tests pin to a no-op. */
+  sleep?: (ms: number) => Promise<void>;
   log?: (line: string) => void;
   manifestPath?: string;
   statePath?: string;
@@ -402,6 +463,10 @@ interface Resolved {
   }) => InstallResult;
   removeService: (args: { tunnelName: string }) => RemoveResult;
   resolveHost: ResolveHostFn;
+  verifyConnection: VerifyConnectionFn;
+  verifyTimeoutMs: number;
+  verifyPollMs: number;
+  sleep: (ms: number) => Promise<void>;
   log: (line: string) => void;
   manifestPath: string;
   statePath: string;
@@ -488,6 +553,17 @@ function resolve(opts: ExposeCloudflareOpts, tunnelNameDefault: string): Resolve
     resolveHost:
       opts.resolveHost ??
       (opts.spawner === undefined ? defaultResolveHost : async () => ["104.16.0.1"]),
+    // Connection-verify seam (#593). Same defaulting policy as
+    // `connectorPids`/`resolveHost`: when a test injects a stub `spawner` (and
+    // no explicit seam), default to an inert "connected" stub so existing
+    // stub-spawner suites don't have to model the `tunnel info` probe.
+    // Production (no spawner override) gets the real bounded poll.
+    verifyConnection:
+      opts.verifyConnection ??
+      (opts.spawner === undefined ? defaultVerifyConnection : async () => true),
+    verifyTimeoutMs: opts.verifyTimeoutMs ?? 25_000,
+    verifyPollMs: opts.verifyPollMs ?? 1_000,
+    sleep: opts.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))),
     log: opts.log ?? ((line) => console.log(line)),
     manifestPath: opts.manifestPath ?? SERVICES_MANIFEST_PATH,
     statePath: opts.statePath ?? CLOUDFLARED_STATE_PATH,
@@ -694,7 +770,50 @@ export async function exposeCloudflareUp(
       "  Each machine gets its own dedicated tunnel — you don't need to run `cloudflared tunnel create` separately; expose does it.",
     );
   } else {
-    r.log(`✓ Reusing existing tunnel "${r.tunnelName}" (${tunnel.id})`);
+    // Reuse-path credentials verification + self-heal (#593). `findTunnelByName`
+    // only proves the tunnel exists ACCOUNT-side. The connector needs the LOCAL
+    // credentials file (`~/.cloudflared/<uuid>.json`, written at `tunnel create`
+    // time) to authenticate — and that file gets lost on clean-slate flows
+    // (`rm -rf ~/.parachute` and friends) while the account-side tunnel
+    // survives. The field repro: tunnel reused, "✓ tunnel up" printed, connector
+    // crash-looping on "credentials file … doesn't exist", every request → 1033.
+    //
+    // If the creds file is missing we recreate the tunnel: delete the
+    // account-side tunnel by name (`--force`, so a stale registered connector
+    // doesn't block it), then `tunnel create` re-writes a fresh creds file. The
+    // new tunnel gets a new UUID; `routeDns` below uses `--overwrite-dns`, so the
+    // hostname's CNAME is repointed at the new UUID even though it pointed at the
+    // old one. The field case confirmed `tunnel delete` + re-run heals cleanly.
+    const existingCreds = credentialsPath(tunnel.id, r.cloudflaredHome);
+    if (existsSync(existingCreds)) {
+      r.log(`✓ Reusing existing tunnel "${r.tunnelName}" (${tunnel.id})`);
+    } else {
+      r.log(
+        `⚠ Tunnel "${r.tunnelName}" (${tunnel.id}) exists in Cloudflare, but its local credentials`,
+      );
+      r.log(`  file is missing (${existingCreds}) — the connector can't authenticate from this`);
+      r.log("  machine. Recreating the tunnel so a fresh credentials file is written…");
+      try {
+        await deleteTunnel(r.runner, r.tunnelName);
+      } catch (err) {
+        if (err instanceof CloudflaredError) {
+          r.log("");
+          r.log(`✗ Couldn't delete the stale tunnel automatically: ${err.message}`);
+          r.log("");
+          r.log("Recover manually, then re-run this command:");
+          r.log(`    cloudflared tunnel delete --force ${r.tunnelName}`);
+          r.log(`    parachute expose public --cloudflare --domain ${hostname}`);
+          return 1;
+        }
+        throw err;
+      }
+      try {
+        tunnel = await createTunnel(r.runner, r.tunnelName);
+      } catch (err) {
+        return reportCloudflaredError(err, r.log);
+      }
+      r.log(`✓ Recreated tunnel ${tunnel.id} (fresh credentials written).`);
+    }
   }
 
   r.log(`Routing DNS: ${hostname} → tunnel ${tunnel.id}…`);
@@ -955,6 +1074,42 @@ export async function exposeCloudflareUp(
     }
   }
 
+  // Post-start connection verification (#593). The connector pid existing is
+  // NOT proof it connected — the field repro had a live pid crash-looping on a
+  // missing creds file, with every public request returning Cloudflare error
+  // 1033 (tunnel registered, no connector) while the CLI printed "✓ tunnel up".
+  // Poll `cloudflared tunnel info` for a live edge connection, bounded. On
+  // timeout, fail LOUDLY with the connector log path + the crash-loop signature
+  // to grep for, instead of claiming success.
+  r.log("");
+  r.log("Verifying the connector established a tunnel connection…");
+  const connected = await r.verifyConnection({
+    runner: r.runner,
+    tunnelName: r.tunnelName,
+    timeoutMs: r.verifyTimeoutMs,
+    pollMs: r.verifyPollMs,
+    sleep: r.sleep,
+  });
+  if (!connected) {
+    r.log("");
+    r.log(
+      `✗ The cloudflared connector (pid ${pid}) started but never registered a tunnel connection`,
+    );
+    r.log(`  within ${Math.round(r.verifyTimeoutMs / 1000)}s. Public requests to ${hostname} will`);
+    r.log("  return Cloudflare error 1033 (tunnel registered, no connector) until this resolves.");
+    r.log("");
+    r.log("Check the connector log for the crash-loop cause:");
+    r.log(`    tail -n 50 ${r.logPath}`);
+    r.log('  A repeating "credentials file … doesn\'t exist" line means the local credentials are');
+    r.log(
+      "  gone — re-run this command (it auto-recreates the tunnel). Other repeating errors point",
+    );
+    r.log("  at the specific failure. Confirm the connector once it's healthy with:");
+    r.log(`    cloudflared tunnel info ${r.tunnelName}`);
+    return 1;
+  }
+  r.log("✓ Connector connected.");
+
   const baseUrl = `https://${hostname}`;
   let vaultUrl: string | undefined;
   if (vaultEntry) {

package/src/commands/serve.ts

@@ -34,6 +34,7 @@ import { generateBootstrapToken } from "../bootstrap-token.ts";
 // path isolation.
 import { CONFIG_DIR, SERVICES_MANIFEST_PATH } from "../config.ts";
 import { readExposeState } from "../expose-state.ts";
+import { createDbHolder } from "../hub-db-liveness.ts";
 import { hubDbPath, openHubDb } from "../hub-db.ts";
 import { hubFetch } from "../hub-server.ts";
 import { writeHubFile } from "../hub.ts";
@@ -345,8 +346,16 @@ export async function serve(opts: ServeOpts = {}): Promise<{
   if (!existsSync(hubHtmlPath)) writeHubFile(hubHtmlPath);
 
   const dbPath = hubDbPath();
-  const db = openHubDb(dbPath);
-  const adminBootstrap = await seedInitialAdminIfNeeded(db, env, log);
+  // Self-heal-or-die DB holder (#594). The handle lives behind a mutable
+  // holder so a request that hits the persistent-corruption class (disk I/O
+  // error / malformed image — e.g. the state dir deleted under a running hub)
+  // can reopen the handle once, or exit(1) for the platform manager to restart
+  // us with a fresh one. `getDb` reads the current handle from the holder.
+  const dbHolder = createDbHolder(openHubDb(dbPath), {
+    reopen: () => openHubDb(dbPath),
+    log,
+  });
+  const adminBootstrap = await seedInitialAdminIfNeeded(dbHolder.get(), env, log);
 
   if (adminBootstrap === "needs-setup") {
     log(
@@ -381,7 +390,8 @@ export async function serve(opts: ServeOpts = {}): Promise<{
       // CMD), so the fix has to land here too. Closes hub#399.
       idleTimeout: 255,
       fetch: hubFetch(WELL_KNOWN_DIR, {
-        getDb: () => db,
+        getDb: () => dbHolder.get(),
+        onDbError: (err) => dbHolder.healOrExit(err),
         issuer,
         loopbackPort: port,
         supervisor,
@@ -468,7 +478,7 @@ export async function serve(opts: ServeOpts = {}): Promise<{
         await supervisor.stop(state.short);
       }
       await server.stop();
-      db.close();
+      dbHolder.get().close();
     },
   };
 }