@openparachute/hub 0.5.14-rc.2 → 0.5.14-rc.21

package/src/__tests__/lifecycle.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import {
@@ -11,9 +11,18 @@ import {
   start,
   stop,
 } from "../commands/lifecycle.ts";
+import { readEnvFileValues } from "../env-file.ts";
 import { writeHubPort } from "../hub-control.ts";
+import { hubDbPath, openHubDb } from "../hub-db.ts";
+import { validateAccessToken } from "../jwt-sign.ts";
+import {
+  OPERATOR_TOKEN_SCOPE_SET_CLAIM,
+  issueOperatorToken,
+  readOperatorTokenFile,
+} from "../operator-token.ts";
 import { ensureLogPath, logPath, readPid, writePid } from "../process-state.ts";
-import { upsertService } from "../services-manifest.ts";
+import { readManifest, upsertService } from "../services-manifest.ts";
+import { rotateSigningKey } from "../signing-keys.ts";
 
 interface Harness {
   configDir: string;
@@ -188,6 +197,87 @@ describe("parachute start", () => {
     }
   });
 
+  test("missing startCmd binary → friendly missing-dependency message + no spawn", async () => {
+    const h = makeHarness();
+    try {
+      seedVault(h.manifestPath);
+      const spawner = makeSpawner([4242]);
+      const logs: string[] = [];
+      const code = await start("vault", {
+        configDir: h.configDir,
+        manifestPath: h.manifestPath,
+        spawner,
+        // Force the preflight's missing-binary branch: parachute-vault not on PATH.
+        which: () => null,
+        log: (l) => logs.push(l),
+      });
+      expect(code).toBe(1);
+      // Preflight fired before the spawn — the stub spawner is never called.
+      expect(spawner.calls).toHaveLength(0);
+      const out = logs.join("\n");
+      expect(out).toMatch(/vault failed to start/);
+      // The friendly install block names the binary + its install path.
+      expect(out).toContain("parachute-vault is required to run the Vault module Hub supervises");
+      expect(out).toContain("parachute install vault");
+      expect(readPid("vault", h.configDir)).toBeUndefined();
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("missing startCmd binary persists lastStartError so a later status surfaces it", async () => {
+    const h = makeHarness();
+    try {
+      seedVault(h.manifestPath);
+      await start("vault", {
+        configDir: h.configDir,
+        manifestPath: h.manifestPath,
+        spawner: makeSpawner([4242]),
+        which: () => null,
+        log: () => {},
+      });
+      const entry = readManifest(h.manifestPath).services.find((s) => s.name === "parachute-vault");
+      expect(entry?.lastStartError?.error_type).toBe("missing_dependency");
+      expect(entry?.lastStartError?.binary).toBe("parachute-vault");
+      expect(entry?.lastStartError?.at).toBeDefined();
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("a successful start clears a previously-recorded lastStartError", async () => {
+    const h = makeHarness();
+    try {
+      seedVault(h.manifestPath);
+      // First start fails (binary missing) → records the error.
+      await start("vault", {
+        configDir: h.configDir,
+        manifestPath: h.manifestPath,
+        spawner: makeSpawner([1]),
+        which: () => null,
+        log: () => {},
+      });
+      expect(
+        readManifest(h.manifestPath).services.find((s) => s.name === "parachute-vault")
+          ?.lastStartError,
+      ).toBeDefined();
+      // Second start succeeds (binary present via the permissive default which
+      // — stub spawner path) → clears the recorded error.
+      await start("vault", {
+        configDir: h.configDir,
+        manifestPath: h.manifestPath,
+        spawner: makeSpawner([4242]),
+        log: () => {},
+      });
+      expect(
+        readManifest(h.manifestPath).services.find((s) => s.name === "parachute-vault")
+          ?.lastStartError,
+      ).toBeUndefined();
+    } finally {
+      h.cleanup();
+    }
+  });
+
   test("notes start command includes configured port and notes-serve shim path", async () => {
     const h = makeHarness();
     try {
@@ -348,6 +438,82 @@ describe("parachute start", () => {
         PORT: "1940",
         PARACHUTE_HUB_ORIGIN: "https://parachute.taildf9ce2.ts.net",
       });
+      // OAuth issuer-mismatch fix: the spawn-env injection above is ephemeral
+      // (lost on the next launchd / systemd boot). `start vault` ALSO persists
+      // the public origin into vault/.env so the out-of-band daemon validates
+      // hub-minted JWTs' `iss` against it. Without this, every reconnect after
+      // a reboot / crash-restart 401s.
+      expect(readEnvFileValues(join(h.configDir, "vault", ".env")).PARACHUTE_HUB_ORIGIN).toBe(
+        "https://parachute.taildf9ce2.ts.net",
+      );
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("self-heals a stale-loopback vault/.env from a cloudflare expose-state on restart", async () => {
+    // Existing-broken-deploy shape: a Cloudflare deploy whose vault/.env had a
+    // loopback PARACHUTE_HUB_ORIGIN baked in (or was unset and a prior run
+    // wrote loopback). expose-state.json carries the real public origin. A
+    // plain `parachute start vault` must rewrite vault/.env to the public
+    // origin so the daemon stops 401ing hub tokens — the self-heal half of the
+    // Cloudflare 401 fix.
+    const h = makeHarness();
+    try {
+      seedVault(h.manifestPath);
+      writeFileSync(
+        join(h.configDir, "expose-state.json"),
+        JSON.stringify({
+          version: 1,
+          layer: "public",
+          mode: "subdomain",
+          canonicalFqdn: "gitcoin-parachute.unforced.dev",
+          port: 1939,
+          funnel: false,
+          entries: [{ kind: "proxy", mount: "/", target: "http://localhost:1939", service: "hub" }],
+          hubOrigin: "https://gitcoin-parachute.unforced.dev",
+        }),
+      );
+      // Pre-seed vault/.env with a stale loopback value (the broken state).
+      mkdirSync(join(h.configDir, "vault"), { recursive: true });
+      writeFileSync(
+        join(h.configDir, "vault", ".env"),
+        "PARACHUTE_HUB_ORIGIN=http://127.0.0.1:1939\n",
+      );
+      const spawner = makeSpawner([4242]);
+      const code = await start("vault", {
+        configDir: h.configDir,
+        manifestPath: h.manifestPath,
+        spawner,
+        log: () => {},
+      });
+      expect(code).toBe(0);
+      expect(readEnvFileValues(join(h.configDir, "vault", ".env")).PARACHUTE_HUB_ORIGIN).toBe(
+        "https://gitcoin-parachute.unforced.dev",
+      );
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("does NOT persist a loopback origin into vault/.env (would shadow a later exposure)", async () => {
+    const h = makeHarness();
+    try {
+      seedVault(h.manifestPath);
+      writeHubPort(1939, h.configDir);
+      const spawner = makeSpawner([4242]);
+      const code = await start("vault", {
+        configDir: h.configDir,
+        manifestPath: h.manifestPath,
+        spawner,
+        log: () => {},
+      });
+      expect(code).toBe(0);
+      // Loopback is fine to inject into the ephemeral spawn env (local dev),
+      // but persisting it would brick the daemon path once exposure comes up:
+      // the baked loopback would shadow the real origin. So vault/.env stays
+      // absent of the key on a loopback-only start.
+      expect(existsSync(join(h.configDir, "vault", ".env"))).toBe(false);
     } finally {
       h.cleanup();
     }
@@ -751,6 +917,159 @@ describe("parachute start", () => {
       h.cleanup();
     }
   });
+
+  // hub#487 — readiness gating beyond the bare liveness settle. Aaron hit this
+  // on a fresh EC2 box: `parachute start vault` printed "✓ vault started" while
+  // the process died ~instantly on EADDRINUSE (an orphan held 1940), and
+  // `parachute status` then showed it inactive.
+
+  /**
+   * A stub spawner that also seeds the service's log file with `content`, so
+   * the readiness-failure path's log-tail + EADDRINUSE detection can read a
+   * realistic boot error. Mirrors how the real spawner appends stdout/stderr
+   * to the logfile.
+   */
+  function makeSpawnerWithLog(pid: number, content: string): SpawnerStub {
+    const calls: SpawnerStub["calls"] = [];
+    return {
+      calls,
+      spawn(cmd, logFile, opts) {
+        calls.push({ cmd: [...cmd], logFile, env: opts?.env, cwd: opts?.cwd });
+        // The start path calls ensureLogPath() before spawn, so logFile's
+        // parent dir already exists — just write the simulated boot output.
+        writeFileSync(logFile, content);
+        return pid;
+      },
+    };
+  }
+
+  test("hub#487: EADDRINUSE in the log → port-in-use message + log tail, not ✓", async () => {
+    const h = makeHarness();
+    try {
+      seedVault(h.manifestPath);
+      const spawner = makeSpawnerWithLog(
+        4242,
+        "booting vault…\nerror: listen EADDRINUSE: address already in use 0.0.0.0:1940\n",
+      );
+      const lines: string[] = [];
+      const code = await start("vault", {
+        configDir: h.configDir,
+        manifestPath: h.manifestPath,
+        spawner,
+        alive: () => false, // process died right after the EADDRINUSE throw
+        sleep: async () => {},
+        startSettleMs: 1,
+        log: (l) => lines.push(l),
+      });
+      expect(code).toBe(1);
+      expect(readPid("vault", h.configDir)).toBeUndefined();
+      const out = lines.join("\n");
+      expect(out).toMatch(/port 1940 is already in use/);
+      expect(out).toMatch(/lsof -ti:1940/);
+      // The real boot error is surfaced inline so the operator doesn't have to
+      // go tail the log themselves.
+      expect(out).toMatch(/EADDRINUSE/);
+      expect(out).not.toMatch(/✓ vault started/);
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("hub#487: process survives settle but never binds its port → failure with log tail", async () => {
+    const h = makeHarness();
+    try {
+      seedVault(h.manifestPath);
+      const spawner = makeSpawnerWithLog(4242, "vault crashed mid-boot\n");
+      const lines: string[] = [];
+      let aliveCalls = 0;
+      const code = await start("vault", {
+        configDir: h.configDir,
+        manifestPath: h.manifestPath,
+        spawner,
+        // Alive through the settle + first readiness poll, then dies — the
+        // slow-EADDRINUSE / crash-after-boot shape.
+        alive: () => {
+          aliveCalls++;
+          return aliveCalls <= 1;
+        },
+        sleep: async () => {},
+        startSettleMs: 1,
+        startReadyMs: 50,
+        startReadyPollMs: 1,
+        portListening: async () => false, // never binds
+        log: (l) => lines.push(l),
+      });
+      expect(code).toBe(1);
+      expect(readPid("vault", h.configDir)).toBeUndefined();
+      const out = lines.join("\n");
+      expect(out).toMatch(/✗ vault failed to start/);
+      expect(out).toMatch(/exited during startup/);
+      expect(out).not.toMatch(/✓ vault started/);
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("hub#487: alive but port silent past the window → non-fatal warning, exit 0", async () => {
+    const h = makeHarness();
+    try {
+      seedVault(h.manifestPath);
+      const spawner = makeSpawner([4242]);
+      const lines: string[] = [];
+      const code = await start("vault", {
+        configDir: h.configDir,
+        manifestPath: h.manifestPath,
+        spawner,
+        alive: () => true, // stays up the whole time
+        sleep: async () => {},
+        startSettleMs: 1,
+        startReadyMs: 10,
+        startReadyPollMs: 1,
+        portListening: async () => false, // slow boot — not listening yet
+        log: (l) => lines.push(l),
+      });
+      // A slow-but-alive daemon isn't a hard failure — we warn rather than fail.
+      expect(code).toBe(0);
+      expect(readPid("vault", h.configDir)).toBe(4242);
+      const out = lines.join("\n");
+      expect(out).toMatch(/port 1940 isn't accepting connections yet/);
+      expect(out).not.toMatch(/✓ vault started/);
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("hub#487: alive + port listening → success", async () => {
+    const h = makeHarness();
+    try {
+      seedVault(h.manifestPath);
+      const spawner = makeSpawner([4242]);
+      const lines: string[] = [];
+      let probeCalls = 0;
+      const code = await start("vault", {
+        configDir: h.configDir,
+        manifestPath: h.manifestPath,
+        spawner,
+        alive: () => true,
+        sleep: async () => {},
+        startSettleMs: 1,
+        startReadyMs: 50,
+        startReadyPollMs: 1,
+        // Not listening on the first poll, bound on the second — exercises the
+        // poll loop rather than an instant true.
+        portListening: async () => {
+          probeCalls++;
+          return probeCalls >= 2;
+        },
+        log: (l) => lines.push(l),
+      });
+      expect(code).toBe(0);
+      expect(readPid("vault", h.configDir)).toBe(4242);
+      expect(lines.join("\n")).toMatch(/✓ vault started \(pid 4242\)/);
+    } finally {
+      h.cleanup();
+    }
+  });
 });
 
 describe("parachute stop", () => {
@@ -1295,6 +1614,149 @@ describe("parachute start|stop|restart hub", () => {
     }
   });
 
+  // hub#481 — `start hub` self-heals a stale operator-token issuer. Tests use
+  // the injectable `hub.selfHealOperatorToken` seam to assert the call happens
+  // (and to make it throw without failing start); a separate test drives the
+  // REAL self-heal against an on-disk operator token + hub.db.
+  test("start hub: invokes operator-token self-heal with the resolved issuer + configDir", async () => {
+    const h = makeHarness();
+    try {
+      const log: string[] = [];
+      const calls: Array<{ issuer: string; configDir: string }> = [];
+      const code = await start("hub", {
+        configDir: h.configDir,
+        manifestPath: h.manifestPath,
+        hubOrigin: "https://hub.example.com",
+        hub: {
+          ensureRunning: async () => ({ pid: 4711, port: 1939, started: true }),
+          selfHealOperatorToken: async (args) => {
+            calls.push({ issuer: args.issuer, configDir: args.configDir });
+            return {
+              kind: "rotated",
+              path: "/x/operator.token",
+              scopeSet: "admin",
+              expiresAt: "z",
+            };
+          },
+        },
+        log: (l) => log.push(l),
+      });
+      expect(code).toBe(0);
+      expect(calls).toEqual([{ issuer: "https://hub.example.com", configDir: h.configDir }]);
+      // Rotation emits an operator-facing line.
+      expect(log.join("\n")).toMatch(
+        /refreshed operator\.token issuer → https:\/\/hub\.example\.com/,
+      );
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("start hub: skips operator-token self-heal when no hub origin is resolvable", async () => {
+    const h = makeHarness();
+    try {
+      let called = false;
+      // No hubOrigin override, no expose-state, no hub.port file → resolveHubOrigin
+      // yields undefined, so the self-heal seam must NOT be called.
+      const code = await start("hub", {
+        configDir: h.configDir,
+        manifestPath: h.manifestPath,
+        hub: {
+          ensureRunning: async () => ({ pid: 4711, port: 1939, started: true }),
+          selfHealOperatorToken: async () => {
+            called = true;
+            return { kind: "absent" };
+          },
+        },
+        log: () => {},
+      });
+      expect(code).toBe(0);
+      expect(called).toBe(false);
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("start hub: a thrown error inside operator-token self-heal does NOT fail start", async () => {
+    const h = makeHarness();
+    try {
+      const log: string[] = [];
+      const code = await start("hub", {
+        configDir: h.configDir,
+        manifestPath: h.manifestPath,
+        hubOrigin: "https://hub.example.com",
+        hub: {
+          ensureRunning: async () => ({ pid: 4711, port: 1939, started: true }),
+          selfHealOperatorToken: async () => {
+            throw new Error("hub.db is locked");
+          },
+        },
+        log: (l) => log.push(l),
+      });
+      expect(code).toBe(0);
+      // Degrades to a brief note, not a hard failure.
+      expect(log.join("\n")).toMatch(
+        /operator\.token issuer self-heal skipped \(hub\.db is locked\)/,
+      );
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("start hub: real self-heal re-mints a stale-iss operator token on disk", async () => {
+    const h = makeHarness();
+    try {
+      // Seed signing keys + a stale-iss operator token in the harness configDir's
+      // hub.db / operator.token, then drive the production self-heal seam.
+      const db = openHubDb(hubDbPath(h.configDir));
+      try {
+        rotateSigningKey(db);
+        await issueOperatorToken(db, "user-abc", {
+          dir: h.configDir,
+          issuer: "http://127.0.0.1:1939",
+          scopeSet: "start",
+        });
+      } finally {
+        db.close();
+      }
+
+      const log: string[] = [];
+      const code = await start("hub", {
+        configDir: h.configDir,
+        manifestPath: h.manifestPath,
+        hubOrigin: "https://gitcoin-parachute.unforced.dev",
+        // No selfHealOperatorToken override → exercises defaultSelfHealOperatorToken
+        // (opens hub.db at <configDir>/hub.db).
+        hub: {
+          ensureRunning: async () => ({ pid: 4711, port: 1939, started: true }),
+        },
+        log: (l) => log.push(l),
+      });
+      expect(code).toBe(0);
+      expect(log.join("\n")).toMatch(
+        /refreshed operator\.token issuer → https:\/\/gitcoin-parachute\.unforced\.dev/,
+      );
+
+      // The on-disk token now validates under the new issuer, scope-set preserved.
+      const verifyDb = openHubDb(hubDbPath(h.configDir));
+      try {
+        const onDisk = await readOperatorTokenFile(h.configDir);
+        expect(onDisk).not.toBeNull();
+        const validated = await validateAccessToken(
+          verifyDb,
+          onDisk as string,
+          "https://gitcoin-parachute.unforced.dev",
+        );
+        expect(validated.payload.iss).toBe("https://gitcoin-parachute.unforced.dev");
+        expect(validated.payload[OPERATOR_TOKEN_SCOPE_SET_CLAIM]).toBe("start");
+      } finally {
+        verifyDb.close();
+      }
+    } finally {
+      h.cleanup();
+    }
+  });
+
   test("stop hub: dispatches to stopHub, true → '✓ hub stopped'", async () => {
     const h = makeHarness();
     try {