@openparachute/hub 0.7.6-rc.1 → 0.7.6-rc.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openparachute/hub",
-  "version": "0.7.6-rc.1",
+  "version": "0.7.6-rc.3",
   "description": "parachute — the local hub for the Parachute ecosystem (discovery, ports, lifecycle, soon OAuth).",
   "license": "AGPL-3.0",
   "publishConfig": {

package/src/__tests__/doctor.test.ts

@@ -753,3 +753,134 @@ describe("doctor --fix — canonical-port repair (confirm-gated, idempotent, non
     }
   });
 });
+
+describe("doctor — loopback-hijack check (hub#737)", () => {
+  test("no hub-instance.json → PASS (benign; the Hub check owns 'down') — #717", async () => {
+    const h = makeHarness();
+    try {
+      seedCurrentManifest(h.manifestPath);
+      seedOperatorToken(h.configDir);
+      // No instance file seeded, no seams overridden — defaults read the empty
+      // sandbox and short-circuit before any real network/lsof.
+      const { code, checks } = await runDoctor(h, healthyDeps());
+      expect(byName(checks, "loopback-hijack")?.status).toBe("pass");
+      expect(code).toBe(0);
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("loopback nonce matches ours + single listener → PASS", async () => {
+    const h = makeHarness();
+    try {
+      seedCurrentManifest(h.manifestPath);
+      seedOperatorToken(h.configDir);
+      const { code, checks } = await runDoctor(
+        h,
+        healthyDeps({
+          readInstanceRecord: () => ({ instance: "n1", pid: 1, port: 1939, startedAt: "" }),
+          probeLoopbackInstance: async () => ({ reachable: true, status: 200, instance: "n1" }),
+          countHubListeners: () => 1,
+        }),
+      );
+      const c = byName(checks, "loopback-hijack");
+      expect(c?.status).toBe("pass");
+      expect(c?.detail).toContain("instance nonce");
+      expect(code).toBe(0);
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("loopback nonce MISMATCH → FAIL with lsof/orb remediation + incident ref", async () => {
+    const h = makeHarness();
+    try {
+      seedCurrentManifest(h.manifestPath);
+      seedOperatorToken(h.configDir);
+      const { code, checks } = await runDoctor(
+        h,
+        healthyDeps({
+          readInstanceRecord: () => ({ instance: "ours", pid: 1, port: 1939, startedAt: "" }),
+          probeLoopbackInstance: async () => ({
+            reachable: true,
+            status: 200,
+            instance: "rogue-hub",
+          }),
+          countHubListeners: () => 2,
+        }),
+      );
+      const c = byName(checks, "loopback-hijack");
+      expect(c?.status).toBe("fail");
+      expect(c?.detail).toContain("rogue-hub");
+      expect(c?.detail).toContain("2 listeners");
+      expect(c?.detail).toContain("hub#737");
+      expect(c?.fix).toContain("lsof -nP -iTCP:1939 -sTCP:LISTEN");
+      expect(c?.fix).toContain("orb list");
+      expect(code).toBe(1);
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("foreign process answering with NO nonce → FAIL (the OrbStack container-hub shape)", async () => {
+    const h = makeHarness();
+    try {
+      seedCurrentManifest(h.manifestPath);
+      seedOperatorToken(h.configDir);
+      const { checks } = await runDoctor(
+        h,
+        healthyDeps({
+          readInstanceRecord: () => ({ instance: "ours", pid: 1, port: 1939, startedAt: "" }),
+          probeLoopbackInstance: async () => ({ reachable: true, status: 200 }),
+          countHubListeners: () => undefined, // lsof indeterminate — still FAILs on the nonce alone
+        }),
+      );
+      const c = byName(checks, "loopback-hijack");
+      expect(c?.status).toBe("fail");
+      expect(c?.detail).toContain("foreign process");
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("nonce matches but a SECOND listener exists → WARN (latent shadow, not FAIL)", async () => {
+    const h = makeHarness();
+    try {
+      seedCurrentManifest(h.manifestPath);
+      seedOperatorToken(h.configDir);
+      const { code, checks } = await runDoctor(
+        h,
+        healthyDeps({
+          readInstanceRecord: () => ({ instance: "n1", pid: 1, port: 1939, startedAt: "" }),
+          probeLoopbackInstance: async () => ({ reachable: true, status: 200, instance: "n1" }),
+          countHubListeners: () => 2,
+        }),
+      );
+      const c = byName(checks, "loopback-hijack");
+      expect(c?.status).toBe("warn");
+      expect(c?.detail).toContain("2 listeners");
+      // WARN never fails the exit code.
+      expect(code).toBe(0);
+    } finally {
+      h.cleanup();
+    }
+  });
+
+  test("record present but loopback unreachable → PASS (defers to the Hub check)", async () => {
+    const h = makeHarness();
+    try {
+      seedCurrentManifest(h.manifestPath);
+      seedOperatorToken(h.configDir);
+      const { checks } = await runDoctor(
+        h,
+        healthyDeps({
+          readInstanceRecord: () => ({ instance: "n1", pid: 1, port: 1939, startedAt: "" }),
+          probeLoopbackInstance: async () => ({ reachable: false }),
+        }),
+      );
+      expect(byName(checks, "loopback-hijack")?.status).toBe("pass");
+    } finally {
+      h.cleanup();
+    }
+  });
+});

package/src/__tests__/hub-instance.test.ts

@@ -0,0 +1,297 @@
+import { describe, expect, test } from "bun:test";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  type HubInstanceRecord,
+  type LoopbackProbe,
+  type SelfProbeState,
+  armHubSelfProbe,
+  classifyLoopback,
+  clearHubInstanceFile,
+  generateInstanceNonce,
+  hijackAlertMessage,
+  hubInstancePath,
+  probeLoopbackInstance,
+  readHubInstanceFile,
+  writeHubInstanceFile,
+} from "../hub-instance.ts";
+
+/**
+ * hub#737 loopback-hijack detection. The nonce file is the linchpin — an
+ * external process learns THIS hub's true identity from disk, then compares it
+ * to what a (possibly hijacked) loopback /health returns. Every side effect
+ * (fs, network) is exercised against a tmp dir / injected fetch.
+ */
+
+function makeDir(): { configDir: string; cleanup: () => void } {
+  const dir = mkdtempSync(join(tmpdir(), "parachute-instance-test-"));
+  return { configDir: dir, cleanup: () => rmSync(dir, { recursive: true, force: true }) };
+}
+
+function record(over: Partial<HubInstanceRecord> = {}): HubInstanceRecord {
+  return {
+    instance: "nonce-1",
+    pid: 4242,
+    port: 1939,
+    startedAt: "2026-07-02T00:00:00.000Z",
+    ...over,
+  };
+}
+
+describe("nonce + instance file", () => {
+  test("generateInstanceNonce yields distinct UUID-shaped values", () => {
+    const a = generateInstanceNonce();
+    const b = generateInstanceNonce();
+    expect(a).not.toBe(b);
+    expect(a).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/);
+  });
+
+  test("write → read round-trips the record (incl. selfProbe)", () => {
+    const { configDir, cleanup } = makeDir();
+    try {
+      const sp: SelfProbeState = {
+        status: "ok",
+        checkedAt: "2026-07-02T00:01:00.000Z",
+      };
+      expect(writeHubInstanceFile(record({ selfProbe: sp }), { configDir })).toBe(true);
+      const back = readHubInstanceFile(configDir);
+      expect(back?.instance).toBe("nonce-1");
+      expect(back?.port).toBe(1939);
+      expect(back?.pid).toBe(4242);
+      expect(back?.selfProbe?.status).toBe("ok");
+      // Written 0644 (world-readable diagnostic aid, not a secret).
+      const raw = readFileSync(hubInstancePath(configDir), "utf8");
+      expect(raw.endsWith("\n")).toBe(true);
+    } finally {
+      cleanup();
+    }
+  });
+
+  test("read returns null on absent / malformed / instance-less files", () => {
+    const { configDir, cleanup } = makeDir();
+    try {
+      expect(readHubInstanceFile(configDir)).toBeNull(); // absent
+      writeFileSync(hubInstancePath(configDir), "not json{");
+      expect(readHubInstanceFile(configDir)).toBeNull(); // malformed
+      writeFileSync(hubInstancePath(configDir), JSON.stringify({ port: 1939 }));
+      expect(readHubInstanceFile(configDir)).toBeNull(); // no instance field
+    } finally {
+      cleanup();
+    }
+  });
+
+  test("clearHubInstanceFile removes the file (graceful-shutdown hygiene) + no-ops when absent", () => {
+    const { configDir, cleanup } = makeDir();
+    try {
+      writeHubInstanceFile(record(), { configDir });
+      expect(readHubInstanceFile(configDir)).not.toBeNull();
+      clearHubInstanceFile(configDir);
+      expect(readHubInstanceFile(configDir)).toBeNull();
+      // Idempotent — clearing an already-absent file must not throw.
+      expect(() => clearHubInstanceFile(configDir)).not.toThrow();
+    } finally {
+      cleanup();
+    }
+  });
+
+  test("write failure is swallowed (returns false, logs) — never throws", () => {
+    const logs: string[] = [];
+    // An un-writable path (a file where a dir must be) forces the mkdir/write to
+    // fail; the helper must degrade, not throw.
+    const { configDir, cleanup } = makeDir();
+    try {
+      const filePath = join(configDir, "blocker");
+      writeFileSync(filePath, "x");
+      const ok = writeHubInstanceFile(record(), {
+        configDir: join(filePath, "nested"),
+        log: (l) => logs.push(l),
+      });
+      expect(ok).toBe(false);
+      expect(logs.length).toBe(1);
+    } finally {
+      cleanup();
+    }
+  });
+});
+
+describe("classifyLoopback", () => {
+  test("unreachable when the probe didn't answer", () => {
+    expect(classifyLoopback("n1", { reachable: false })).toBe("unreachable");
+  });
+  test("ok when the returned instance matches ours", () => {
+    expect(classifyLoopback("n1", { reachable: true, status: 200, instance: "n1" })).toBe("ok");
+  });
+  test("hijacked when a different instance answers", () => {
+    expect(classifyLoopback("n1", { reachable: true, status: 200, instance: "n2" })).toBe(
+      "hijacked",
+    );
+  });
+  test("hijacked when a reachable process carries NO instance (old/foreign hub)", () => {
+    expect(classifyLoopback("n1", { reachable: true, status: 200 })).toBe("hijacked");
+  });
+});
+
+describe("probeLoopbackInstance", () => {
+  test("extracts instance + isHub from a well-formed /health body", async () => {
+    const fetchFn = (async () =>
+      new Response(JSON.stringify({ service: "parachute-hub", instance: "xyz" }), {
+        status: 200,
+      })) as unknown as typeof fetch;
+    const out = await probeLoopbackInstance(1939, { fetchFn });
+    expect(out).toEqual({ reachable: true, status: 200, instance: "xyz", isHub: true });
+  });
+
+  test("reachable-but-junk body → reachable with no instance (foreign server shape)", async () => {
+    const fetchFn = (async () =>
+      new Response("<html>not a hub</html>", { status: 200 })) as unknown as typeof fetch;
+    const out = await probeLoopbackInstance(1939, { fetchFn });
+    expect(out.reachable).toBe(true);
+    expect(out.instance).toBeUndefined();
+    expect(out.isHub).toBeUndefined();
+  });
+
+  test("network error → not reachable, never throws", async () => {
+    const fetchFn = (async () => {
+      throw new Error("ECONNREFUSED");
+    }) as unknown as typeof fetch;
+    const out = await probeLoopbackInstance(1939, { fetchFn });
+    expect(out).toEqual({ reachable: false });
+  });
+});
+
+describe("armHubSelfProbe", () => {
+  /** A probe stub that returns a fixed LoopbackProbe. */
+  function fixedProbe(p: LoopbackProbe): (port: number) => Promise<LoopbackProbe> {
+    return async () => p;
+  }
+
+  test("ok verdict: writes state, stays quiet (no loud log)", async () => {
+    const logs: string[] = [];
+    const states: SelfProbeState[] = [];
+    const probe = armHubSelfProbe(
+      { port: 1939, nonce: "n1", record: record() },
+      {
+        probe: fixedProbe({ reachable: true, status: 200, instance: "n1" }),
+        writeState: (s) => states.push(s),
+        log: (l) => logs.push(l),
+        setIntervalFn: () => 0,
+      },
+    );
+    const verdict = await probe.probeOnce();
+    expect(verdict).toBe("ok");
+    expect(logs).toEqual([]);
+    expect(states.at(-1)?.status).toBe("ok");
+  });
+
+  test("hijacked verdict: LOUD alert on EVERY probe + observedInstance persisted", async () => {
+    const logs: string[] = [];
+    const states: SelfProbeState[] = [];
+    const probe = armHubSelfProbe(
+      { port: 1939, nonce: "n1", record: record() },
+      {
+        probe: fixedProbe({ reachable: true, status: 200, instance: "rogue-9" }),
+        writeState: (s) => states.push(s),
+        log: (l) => logs.push(l),
+        setIntervalFn: () => 0,
+      },
+    );
+    expect(await probe.probeOnce()).toBe("hijacked");
+    expect(await probe.probeOnce()).toBe("hijacked");
+    // Repeated verbatim — a hijack is a standing emergency, not a one-shot.
+    expect(logs.length).toBe(2);
+    expect(logs[0]).toContain("LOOPBACK HIJACK");
+    expect(logs[0]).toContain("lsof -nP -iTCP:1939");
+    expect(states.at(-1)?.observedInstance).toBe("rogue-9");
+    expect(states.at(-1)?.status).toBe("hijacked");
+  });
+
+  test("unreachable verdict: logs ONCE per state change, not per tick", async () => {
+    const logs: string[] = [];
+    const probe = armHubSelfProbe(
+      { port: 1939, nonce: "n1", record: record() },
+      {
+        probe: fixedProbe({ reachable: false }),
+        writeState: () => {},
+        log: (l) => logs.push(l),
+        setIntervalFn: () => 0,
+      },
+    );
+    await probe.probeOnce();
+    await probe.probeOnce();
+    expect(logs.length).toBe(1); // second consecutive unreachable is silent
+    expect(logs[0]).toContain("did not answer");
+  });
+
+  test("recovery: hijacked → ok logs a single 'cleared' line", async () => {
+    const logs: string[] = [];
+    let current: LoopbackProbe = { reachable: true, status: 200, instance: "rogue" };
+    const probe = armHubSelfProbe(
+      { port: 1939, nonce: "n1", record: record() },
+      {
+        probe: async () => current,
+        writeState: () => {},
+        log: (l) => logs.push(l),
+        setIntervalFn: () => 0,
+      },
+    );
+    expect(await probe.probeOnce()).toBe("hijacked");
+    current = { reachable: true, status: 200, instance: "n1" };
+    expect(await probe.probeOnce()).toBe("ok");
+    expect(logs.length).toBe(2);
+    expect(logs[1]).toContain("Hijack cleared");
+  });
+
+  test("stop() clears the interval handle", () => {
+    let cleared: unknown;
+    const probe = armHubSelfProbe(
+      { port: 1939, nonce: "n1", record: record() },
+      {
+        probe: fixedProbe({ reachable: true, instance: "n1" }),
+        writeState: () => {},
+        setIntervalFn: () => "the-handle",
+        clearIntervalFn: (h) => {
+          cleared = h;
+        },
+      },
+    );
+    probe.stop();
+    expect(cleared).toBe("the-handle");
+  });
+
+  test("default writeState patches selfProbe into the real instance file", async () => {
+    const { configDir, cleanup } = makeDir();
+    try {
+      const rec = record();
+      writeHubInstanceFile(rec, { configDir });
+      const probe = armHubSelfProbe(
+        { port: 1939, nonce: "n1", record: rec, configDir },
+        {
+          probe: fixedProbe({ reachable: true, status: 200, instance: "someone-else" }),
+          log: () => {},
+          setIntervalFn: () => 0,
+        },
+      );
+      await probe.probeOnce();
+      const back = readHubInstanceFile(configDir);
+      expect(back?.selfProbe?.status).toBe("hijacked");
+      expect(back?.instance).toBe("nonce-1"); // base record preserved
+    } finally {
+      cleanup();
+    }
+  });
+});
+
+describe("hijackAlertMessage", () => {
+  test("names the observed instance + the diagnosis commands + incident ref", () => {
+    const msg = hijackAlertMessage(1939, "rogue-42");
+    expect(msg).toContain("instance=rogue-42");
+    expect(msg).toContain("lsof -nP -iTCP:1939 -sTCP:LISTEN");
+    expect(msg).toContain("orb list");
+    expect(msg).toContain("hub#737");
+  });
+  test("degrades gracefully when the foreign process carries no nonce", () => {
+    const msg = hijackAlertMessage(1939);
+    expect(msg).toContain("no hub instance nonce");
+  });
+});

package/src/__tests__/hub-server.test.ts

@@ -15,6 +15,7 @@ import {
   layerOf,
   parseArgs,
   resolveClientIp,
+  stripHopByHopHeaders,
 } from "../hub-server.ts";
 import { setNotesRedirectDisabled } from "../hub-settings.ts";
 import { clearNotesRedirectLogState } from "../notes-redirect.ts";
@@ -139,6 +140,27 @@ describe("hubFetch routing", () => {
     }
   });
 
+  test("/health echoes the instance nonce when set, omits it when absent (hub#737)", async () => {
+    const h = makeHarness();
+    try {
+      // With a nonce threaded in, /health carries it as `instance`.
+      const withNonce = (await (
+        await hubFetch(h.dir, { instanceNonce: "nonce-abc-123" })(req("/health"))
+      ).json()) as { service: string; instance?: string };
+      expect(withNonce.service).toBe("parachute-hub");
+      expect(withNonce.instance).toBe("nonce-abc-123");
+
+      // Without one (DB-less / dev entrypoint), the field is omitted — additive,
+      // so no strict /health consumer breaks.
+      const withoutNonce = (await (await hubFetch(h.dir, {})(req("/health"))).json()) as {
+        instance?: string;
+      };
+      expect(withoutNonce.instance).toBeUndefined();
+    } finally {
+      h.cleanup();
+    }
+  });
+
   test("/health surfaces db:error:path-gone when the proactive probe sees a wiped path (#610)", async () => {
     // The ghost-fd lie: SELECT 1 still succeeds against the unlinked inode, so
     // probeDbLiveness alone would report ok. probeDbPath stat()s the PATH and
@@ -5963,6 +5985,153 @@ describe("substrate trust headers — X-Parachute-Layer / X-Parachute-Client-IP
   });
 });
 
+describe("stripHopByHopHeaders (hub#738)", () => {
+  test("drops the standard hop-by-hop set", () => {
+    const h = new Headers({
+      connection: "keep-alive",
+      "keep-alive": "timeout=5",
+      "proxy-authorization": "Basic xxx",
+      te: "trailers",
+      "transfer-encoding": "chunked",
+      upgrade: "h2c",
+      authorization: "Bearer keep-me",
+      "content-type": "application/json",
+    });
+    stripHopByHopHeaders(h);
+    expect(h.has("connection")).toBe(false);
+    expect(h.has("keep-alive")).toBe(false);
+    expect(h.has("proxy-authorization")).toBe(false);
+    expect(h.has("te")).toBe(false);
+    expect(h.has("transfer-encoding")).toBe(false);
+    expect(h.has("upgrade")).toBe(false);
+    // End-to-end headers survive.
+    expect(h.get("authorization")).toBe("Bearer keep-me");
+    expect(h.get("content-type")).toBe("application/json");
+  });
+
+  test("drops headers NAMED in the Connection field-value (RFC 9110 §7.6.1)", () => {
+    const h = new Headers({
+      connection: "close, X-Custom-Hop",
+      "x-custom-hop": "should-go",
+      "x-real-header": "should-stay",
+    });
+    stripHopByHopHeaders(h);
+    expect(h.has("connection")).toBe(false);
+    expect(h.has("x-custom-hop")).toBe(false);
+    expect(h.get("x-real-header")).toBe("should-stay");
+  });
+});
+
+describe("proxy upstream keep-alive (hub#738)", () => {
+  const fakeServer = (address: string) => ({ requestIP: () => ({ address }) });
+
+  function writeUpstreamService(h: Harness, port: number): void {
+    writeManifest(
+      {
+        services: [
+          {
+            name: "keepalive-svc",
+            port,
+            paths: ["/keepalive-svc"],
+            health: "/keepalive-svc/health",
+            version: "0.1.0",
+          },
+        ],
+      },
+      h.manifestPath,
+    );
+  }
+
+  test("200 proxied requests reuse a handful of upstream sockets even when the client sends Connection: close", async () => {
+    // Regression for hub#738: the proxy forwarded the client's hop-by-hop
+    // `Connection: close` verbatim, so Bun opened a FRESH ephemeral socket per
+    // proxied request instead of reusing its per-origin pool → 16k+ TIME_WAIT
+    // → host-wide port exhaustion (the 2026-07-02 incident, amplifying an agent
+    // reconciler 401 loop). We count the distinct client (ephemeral) ports the
+    // upstream sees: a working pool reuses one (occasionally a couple); the bug
+    // produced ~N distinct ports.
+    const seenPorts = new Set<number>();
+    const upstream = Bun.serve({
+      port: 0,
+      hostname: "127.0.0.1",
+      fetch: (r, server) => {
+        const ip = server.requestIP(r);
+        if (ip) seenPorts.add(ip.port);
+        return new Response("unauthorized", { status: 401 }); // the incident's 401 loop
+      },
+    });
+    const h = makeHarness();
+    try {
+      writeUpstreamService(h, upstream.port as number);
+      const fetcher = hubFetch(h.dir, { manifestPath: h.manifestPath });
+      const N = 200;
+      for (let i = 0; i < N; i++) {
+        const res = await fetcher(
+          req("/keepalive-svc/x", { headers: { connection: "close" } }),
+          fakeServer("127.0.0.1"),
+        );
+        await res.arrayBuffer(); // fully drain so the socket returns to the pool
+      }
+      expect(seenPorts.size).toBeLessThanOrEqual(8);
+    } finally {
+      upstream.stop(true);
+      h.cleanup();
+    }
+  });
+
+  test("a client hang-up mid-stream aborts the upstream fetch (socket released, not left streaming to nobody)", async () => {
+    // Without forwarding req.signal, an aborted client left the upstream
+    // streaming its whole body to a gone client and holding the socket. We
+    // forward the signal so the upstream is torn down (cancel() fires or the
+    // enqueue loop errors) instead of running to completion.
+    let tornDown = false;
+    const upstream = Bun.serve({
+      port: 0,
+      hostname: "127.0.0.1",
+      fetch: () =>
+        new Response(
+          new ReadableStream({
+            async start(controller) {
+              try {
+                for (let i = 0; i < 40; i++) {
+                  controller.enqueue(new TextEncoder().encode(`chunk ${i}\n`));
+                  await Bun.sleep(10);
+                }
+                controller.close();
+              } catch {
+                tornDown = true; // enqueue threw after the peer dropped
+              }
+            },
+            cancel() {
+              tornDown = true; // Bun cancelled the source on disconnect
+            },
+          }),
+          { headers: { "content-type": "text/plain" } },
+        ),
+    });
+    const h = makeHarness();
+    try {
+      writeUpstreamService(h, upstream.port as number);
+      const fetcher = hubFetch(h.dir, { manifestPath: h.manifestPath });
+      const ac = new AbortController();
+      const res = await fetcher(
+        req("/keepalive-svc/stream", { signal: ac.signal }),
+        fakeServer("127.0.0.1"),
+      );
+      const reader = res.body!.getReader();
+      await reader.read(); // consume one chunk, then hang up
+      ac.abort();
+      // Full stream is 40 * 10ms = 400ms; wait past it. With the signal
+      // forwarded the upstream tears down early; without, it runs to close.
+      await Bun.sleep(700);
+      expect(tornDown).toBe(true);
+    } finally {
+      upstream.stop(true);
+      h.cleanup();
+    }
+  });
+});
+
 describe("resolveClientIp (H2)", () => {
   test("CF-Connecting-IP wins over X-Forwarded-For and peer", () => {
     const r = req("/", {