@sleep2agi/commhub-server 0.9.0-preview.3 → 0.9.0-preview.5

package/src/probe.test.ts

@@ -0,0 +1,80 @@
+import { describe, expect, test, beforeEach } from "bun:test";
+import {
+  putPendingProbeSecret,
+  takePendingProbeSecret,
+  peekPendingProbeSecret,
+  evictExpiredProbeSecrets,
+  stopBackgroundProbeTimersForTest,
+  type PendingProbeSecret,
+} from "./probe.js";
+
+// RFC-028 P1 fold-in #4 — takePendingProbeSecret 4-case unit. Mirrors
+// RFC-026 §4.4 takePendingEnvBlob unit pattern. This is the C2 cross-
+// daemon guard for probes: a daemon-X that intercepts a probe_id minted
+// for daemon-Y must NOT be able to consume it. Wrong-caller branch must
+// NOT evict (so the right daemon can still consume in TTL).
+
+function mkSecret(over: Partial<PendingProbeSecret> = {}): Omit<PendingProbeSecret, "expires_at"> {
+  return {
+    probe_id: over.probe_id ?? "pr_test_1",
+    daemon_node_id: over.daemon_node_id ?? "node_daemon_alpha",
+    provider_id: over.provider_id ?? "prov_1",
+    vendor: over.vendor ?? "anthropic",
+    base_url: over.base_url ?? "https://api.anthropic.com",
+    model_name: over.model_name ?? "claude-opus-test",
+    api_key: over.api_key ?? "sk-secret-not-leaked",
+    network_id: over.network_id ?? "net_1",
+  };
+}
+
+describe("RFC-028 P1 — takePendingProbeSecret 4-case", () => {
+  beforeEach(() => stopBackgroundProbeTimersForTest());
+
+  test("case 1: happy path — right daemon, within TTL → returns + evicts", () => {
+    putPendingProbeSecret(mkSecret({ probe_id: "pr_happy" }));
+    expect(peekPendingProbeSecret("pr_happy")).not.toBeNull();
+    const out = takePendingProbeSecret("pr_happy", "node_daemon_alpha");
+    expect(out).not.toBeNull();
+    expect(out!.api_key).toBe("sk-secret-not-leaked");
+    // Evicted on success → second take returns null
+    expect(takePendingProbeSecret("pr_happy", "node_daemon_alpha")).toBeNull();
+    expect(peekPendingProbeSecret("pr_happy")).toBeNull();
+  });
+
+  test("case 2: wrong daemon → returns null AND does NOT evict (right daemon can still consume)", () => {
+    putPendingProbeSecret(mkSecret({ probe_id: "pr_wrongcaller", daemon_node_id: "node_daemon_alpha" }));
+    // attacker daemon tries to consume
+    const wrong = takePendingProbeSecret("pr_wrongcaller", "node_daemon_attacker");
+    expect(wrong).toBeNull();
+    // entry MUST still be there — peek confirms (this is the C2 invariant:
+    // a wrong-caller probe cannot DoS the legitimate one by triggering eviction)
+    const peeked = peekPendingProbeSecret("pr_wrongcaller");
+    expect(peeked).not.toBeNull();
+    expect(peeked!.daemon_node_id).toBe("node_daemon_alpha");
+    // right daemon STILL gets it
+    const right = takePendingProbeSecret("pr_wrongcaller", "node_daemon_alpha");
+    expect(right).not.toBeNull();
+    expect(right!.api_key).toBe("sk-secret-not-leaked");
+  });
+
+  test("case 3: not found → returns null (no side-effect)", () => {
+    const out = takePendingProbeSecret("pr_nonexistent", "node_daemon_alpha");
+    expect(out).toBeNull();
+    // also confirm peek is null
+    expect(peekPendingProbeSecret("pr_nonexistent")).toBeNull();
+  });
+
+  test("case 4: expired → returns null + evicts (so memory doesn't accumulate)", async () => {
+    // Use evictExpiredProbeSecrets's now-arg surrogate: put a fresh entry then
+    // ask take with a time-shifted view by mutating the map. Since the public
+    // API doesn't expose expires_at directly, we use the sweeper to verify
+    // expired entries get cleaned, then assert take returns null.
+    putPendingProbeSecret(mkSecret({ probe_id: "pr_expired" }));
+    // Sweep with a forced "now" 120s in the future → past TTL_MS (60s)
+    const n = evictExpiredProbeSecrets(Date.now() + 120_000);
+    expect(n).toBeGreaterThanOrEqual(1);
+    // Now any take must return null + the entry is gone
+    expect(takePendingProbeSecret("pr_expired", "node_daemon_alpha")).toBeNull();
+    expect(peekPendingProbeSecret("pr_expired")).toBeNull();
+  });
+});

package/src/probe.ts

@@ -0,0 +1,200 @@
+// RFC-028 P1 §2.5 — hub-side probe state (pendingProbeSecrets Map +
+// orphan revoke sweeper + finalize helper). MCP tools register in
+// tools.ts (calls into here).
+//
+// Mirrors RFC-026 §4.4 mint-stream-evict pattern:
+//  - hub creates probe row (metadata only) + stashes decrypted secret
+//    + child fetcher params in in-memory Map (TTL 60s)
+//  - daemon get_probe_request takes (one-shot consume + evict)
+//  - daemon ack_probe_request → hub deriveErrorLabel + final row write
+//  - sweeper revokes orphan rows + tokens (P2 not needed since probe
+//    doesn't mint long-lived tokens — short-window only)
+
+import { db, uuidv4 } from "./db.js";
+import { vaultGet, VaultError } from "./vault.js";
+import {
+  validateBaseUrl, deriveErrorLabel, rejectIfSecretLeaked,
+  ProbeAckPayloadSchema, type ProbeAckPayload,
+  ProbeValidationError,
+} from "./probe-validate.js";
+
+// ── Probe ephemeral state Map (F1 mint-stream-evict, RFC-028 §2.5) ──
+
+export interface PendingProbeSecret {
+  probe_id: string;
+  daemon_node_id: string;          // C2 cross-daemon guard
+  provider_id: string;
+  vendor: string;
+  base_url: string;
+  model_name: string;
+  api_key: string;                  // plaintext, ephemeral, never persisted
+  network_id: string;
+  expires_at: number;
+}
+
+const TTL_MS = 60_000;
+const SWEEPER_INTERVAL_MS = 30_000;
+const REAPER_TTL_MS = 60_000;
+
+const pendingProbeSecrets = new Map<string, PendingProbeSecret>();
+let _gcTimer: ReturnType<typeof setInterval> | null = null;
+let _sweeperTimer: ReturnType<typeof setInterval> | null = null;
+
+export function putPendingProbeSecret(b: Omit<PendingProbeSecret, "expires_at">): void {
+  pendingProbeSecrets.set(b.probe_id, { ...b, expires_at: Date.now() + TTL_MS });
+}
+
+/** Caller-bound one-shot consume. Returns null if not found, wrong
+ *  daemon, or expired; entry deleted on success. */
+export function takePendingProbeSecret(probe_id: string, callerDaemonNodeId: string): PendingProbeSecret | null {
+  const b = pendingProbeSecrets.get(probe_id);
+  if (!b) return null;
+  if (b.daemon_node_id !== callerDaemonNodeId) return null;   // wrong-caller probe: DON'T evict
+  if (Date.now() > b.expires_at) { pendingProbeSecrets.delete(probe_id); return null; }
+  pendingProbeSecrets.delete(probe_id);
+  return b;
+}
+
+export function peekPendingProbeSecret(probe_id: string): PendingProbeSecret | null {
+  return pendingProbeSecrets.get(probe_id) || null;
+}
+
+export function evictExpiredProbeSecrets(now = Date.now()): number {
+  let n = 0;
+  for (const [k, v] of pendingProbeSecrets) {
+    if (now > v.expires_at) { pendingProbeSecrets.delete(k); n++; }
+  }
+  return n;
+}
+
+export function startPendingProbeGcTimer(): void {
+  if (_gcTimer) return;
+  _gcTimer = setInterval(() => {
+    const n = evictExpiredProbeSecrets();
+    if (n > 0) console.log(`[probe] GC: evicted ${n} expired probe secret(s)`);
+  }, 5_000);
+  (_gcTimer as any).unref?.();
+}
+
+export function stopBackgroundProbeTimersForTest(): void {
+  if (_gcTimer) { clearInterval(_gcTimer); _gcTimer = null; }
+  if (_sweeperTimer) { clearInterval(_sweeperTimer); _sweeperTimer = null; }
+  pendingProbeSecrets.clear();
+}
+
+// ── Sweeper: terminal-transition probe_results rows that never got ack ──
+// Mirrors RFC-026 §4.4.8 — probe rows older than REAPER_TTL_MS that
+// are still status='pending' get marked timeout + error_label set.
+
+export function runOrphanProbeSweepOnce(now = Date.now()): { swept: number } {
+  const cutoffMs = now - REAPER_TTL_MS;
+  const stale = db.all<{ probe_id: string }>(
+    `SELECT probe_id FROM probe_results WHERE status = 'pending' AND probed_at < ?1`,
+    cutoffMs,
+  );
+  if (stale.length === 0) return { swept: 0 };
+
+  let swept = 0;
+  try {
+    db.exec("BEGIN");
+    for (const row of stale) {
+      db.run(
+        `UPDATE probe_results SET status = ?1, error_label = ?2 WHERE probe_id = ?3 AND status = 'pending'`,
+        ["timeout", "连通性测试超时 (sweeper: 60s 无 daemon ack)", row.probe_id],
+      );
+      pendingProbeSecrets.delete(row.probe_id);
+      swept++;
+    }
+    db.exec("COMMIT");
+  } catch (e: any) {
+    try { db.exec("ROLLBACK"); } catch { /* ok */ }
+    console.warn(`[probe] sweeper failed: ${e?.message || e}`);
+    return { swept: 0 };
+  }
+  if (swept > 0) console.log(`[commhub] ✓ probe sweeper: swept ${swept} stale probe(s)`);
+  return { swept };
+}
+
+export function startProbeSweeperTimer(): void {
+  if (_sweeperTimer) return;
+  try { runOrphanProbeSweepOnce(); }
+  catch (e: any) { console.warn(`[probe] boot sweeper failed: ${e?.message || e}`); }
+  _sweeperTimer = setInterval(() => {
+    try { runOrphanProbeSweepOnce(); }
+    catch (e: any) { console.warn(`[probe] periodic sweeper failed: ${e?.message || e}`); }
+  }, SWEEPER_INTERVAL_MS);
+  (_sweeperTimer as any).unref?.();
+}
+
+// ── Helpers ─────────────────────────────────────────────────────────
+
+export function newProbeId(): string {
+  return `pr_${uuidv4()}`;
+}
+
+/** Hub-side: receive daemon ack, validate, redact, derive label, write
+ *  final row. Returns the persisted row OR throws ProbeValidationError. */
+export function finalizeProbeAck(
+  rawAck: unknown,
+  caller: { network_id: string; daemon_node_id: string },
+): { ok: boolean; error_label: string | null; status: string } {
+  // 1. Schema parse — rejects extra fields via .strict()
+  const ack = ProbeAckPayloadSchema.parse(rawAck);
+  // 2. Pull the row (existence + caller binding)
+  const row = db.get<{ probe_id: string; daemon_node_id: string; provider_id: string; status: string }>(
+    `SELECT probe_id, daemon_node_id, provider_id, status FROM probe_results WHERE probe_id = ?1`,
+    ack.probe_id,
+  );
+  if (!row) throw new ProbeValidationError("probe_not_found", { probe_id: ack.probe_id });
+  if (row.daemon_node_id !== caller.daemon_node_id) {
+    throw new ProbeValidationError("not_your_probe", { probe_id: ack.probe_id });
+  }
+  if (row.status !== "pending") {
+    throw new ProbeValidationError("probe_not_pending", { current_status: row.status });
+  }
+  // 3. Belt-and-suspenders secret-leak guard on full ack JSON
+  //    (catches daemon impl bug stuffing secret into probe_id etc.)
+  try {
+    const providerRow = db.get<{ secret_key_ref: string }>(
+      `SELECT secret_key_ref FROM providers WHERE provider_id = ?1 AND network_id = ?2`,
+      row.provider_id, caller.network_id,
+    );
+    if (providerRow) {
+      const secretVal = vaultGet(caller.network_id, providerRow.secret_key_ref);
+      if (secretVal) {
+        rejectIfSecretLeaked(JSON.stringify(ack), [secretVal]);
+      }
+    }
+  } catch (e) {
+    if (e instanceof ProbeValidationError && e.code === "ack_secret_leak") {
+      // audit-log this serious event (impl pattern below in tools.ts)
+      console.warn(`[probe] ack_secret_leak: daemon=${caller.daemon_node_id} probe=${ack.probe_id} reason=${JSON.stringify(e.detail)}`);
+      // mark row as failed for safety
+      db.run(
+        `UPDATE probe_results SET status = 'tls_error', error_label = ?1 WHERE probe_id = ?2 AND status = 'pending'`,
+        ["内部安全异常: daemon ack 含 secret 值, 已拒并 audit", ack.probe_id],
+      );
+      throw e;
+    }
+    // VaultError (missing key etc.) — non-fatal, skip leak guard but log
+    if (e instanceof VaultError) {
+      console.warn(`[probe] leak-guard skipped (vault read failed): ${e.message}`);
+    } else {
+      throw e;
+    }
+  }
+
+  // 4. Derive error_label from enum + raw_status_code
+  const label = deriveErrorLabel(ack);
+  // 5. Final write
+  db.run(
+    `UPDATE probe_results
+       SET status = ?1, latency_ms = ?2, raw_status_code = ?3, error_label = ?4
+     WHERE probe_id = ?5 AND status = 'pending'`,
+    [ack.status, ack.latency_ms, ack.raw_status_code ?? null, label, ack.probe_id],
+  );
+  return { ok: true, status: ack.status, error_label: label };
+}
+
+// Re-export validate funcs for tools.ts convenience
+export { validateBaseUrl, ProbeValidationError };

package/src/shared/probe-host-allowlist-drift.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, test } from "bun:test";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import {
+  VENDOR_HOST_ALLOWLIST as HUB_ALLOW,
+  FORBIDDEN_IPV4_RE as HUB_V4,
+  FORBIDDEN_IPV6_RE as HUB_V6,
+} from "./probe-host-allowlist.js";
+
+// RFC-028 P1 fold-in #1 — hub-and-daemon probe-host-allowlist drift
+// guard. Mirrors the G9 reserved-env drift test (RFC-026 §4.4.7).
+// Two enforcement modes:
+//   1. byte-identical source files
+//   2. runtime set-equality of the imported constants (vendor regex
+//      source strings + private-IP regex source strings)
+// If a future contributor edits one side without the other, this test
+// breaks the merge — the daemon-side compromised-hub defense depends
+// on the two lists being identical.
+
+const HUB_PATH = join(import.meta.dir, "probe-host-allowlist.ts");
+const DAEMON_PATH = join(
+  import.meta.dir,
+  "..", "..", "..", "agent-node", "src", "shared", "probe-host-allowlist.ts",
+);
+
+describe("RFC-028 P1 — hub/daemon probe-host-allowlist drift guard", () => {
+  test("byte-identical source files", () => {
+    const hub = readFileSync(HUB_PATH, "utf-8");
+    const daemon = readFileSync(DAEMON_PATH, "utf-8");
+    expect(daemon).toBe(hub);
+  });
+
+  test("VENDOR_HOST_ALLOWLIST regex sources equal at runtime", async () => {
+    const daemonMod = await import(DAEMON_PATH);
+    const daemonAllow = daemonMod.VENDOR_HOST_ALLOWLIST as Record<string, ReadonlyArray<RegExp>>;
+    // vendor set equality
+    expect(new Set(Object.keys(daemonAllow))).toEqual(new Set(Object.keys(HUB_ALLOW)));
+    // per-vendor regex source array equality (order matters — both should
+    // be authored in identical order to keep diffs minimal)
+    for (const vendor of Object.keys(HUB_ALLOW)) {
+      const hubSources = HUB_ALLOW[vendor].map(r => r.source);
+      const daemonSources = daemonAllow[vendor].map(r => r.source);
+      expect(daemonSources).toEqual(hubSources);
+    }
+  });
+
+  test("FORBIDDEN_IPV4_RE + FORBIDDEN_IPV6_RE sources equal at runtime", async () => {
+    const daemonMod = await import(DAEMON_PATH);
+    const daemonV4 = (daemonMod.FORBIDDEN_IPV4_RE as ReadonlyArray<RegExp>).map(r => r.source);
+    const daemonV6 = (daemonMod.FORBIDDEN_IPV6_RE as ReadonlyArray<RegExp>).map(r => r.source);
+    expect(daemonV4).toEqual(HUB_V4.map(r => r.source));
+    expect(daemonV6).toEqual(HUB_V6.map(r => r.source));
+  });
+});

package/src/shared/probe-host-allowlist.ts

@@ -0,0 +1,128 @@
+// RFC-028 P1 §4.4 — shared probe SSRF guards.
+// Host allowlist + private IP block + validateBaseUrl.
+//
+// MUST be byte-identical between hub (server/src/shared/) and daemon
+// (agent-node/src/shared/). Enforced by probe-host-allowlist-drift.test.ts
+// (same pattern as reserved-env.ts G9 drift guard, RFC-026 §4.4.7).
+//
+// Why mirror not symlink: hub and daemon are separate npm packages
+// installed independently on different machines. A drift-checked source
+// copy survives `npm pack` + global install; a symlink does not.
+//
+// Threat model the daemon re-check defends against:
+//   - Compromised hub records a malicious base_url in providers/spec
+//   - Hub bypassed validateBaseUrl (regression / different code path)
+//   - On-wire injection between hub→daemon SSE
+// Daemon trusts process.execPath + this file's content, NOT what
+// `get_probe_request` claims. If hub-side check is identical, daemon
+// re-check is a no-op; if hub-side ever loosens, daemon stays strict.
+
+export class ProbeValidationError extends Error {
+  constructor(public code: string, public detail?: Record<string, unknown>) {
+    super(`${code}${detail ? ` ${JSON.stringify(detail)}` : ""}`);
+    this.name = "ProbeValidationError";
+  }
+}
+
+// ── §4.4.1 per-vendor host allowlist ──────────────────────────────
+// `anthropic` vendor accepts Anthropic + curated 3rd-party Anthropic-
+// compatible endpoints that Vincent / N站马 ship live providers
+// against (DeepSeek + MiniMax both expose `/v1/messages` with the
+// Anthropic protocol surface). Whitelist semantics preserved — NOT
+// allow-any. New entries land here via PR with security review
+// (network-admin-configurable allowlist is backlog: needs strong
+// IP-range / shape guard before admins can edit).
+//
+// Per 通信龙 (RFC-028 P1.5+): the host whitelist is defense layer 1;
+// layer 2 is the DNS-resolved IP block (FORBIDDEN_IPV{4,6}_RE below),
+// applied AFTER allowlist passes. Any new host gets BOTH layers
+// automatically — host in allowlist + DNS resolves to private/metadata
+// IP still surfaces `probe_resolve_unsafe_ip`.
+//
+// P2 will add vendors (openai/zai/openrouter/qwen) as separate keys.
+export const VENDOR_HOST_ALLOWLIST: Record<string, ReadonlyArray<RegExp>> = {
+  anthropic:  [
+    /^api\.anthropic\.com$/,
+    /^api\.deepseek\.com$/,     // Anthropic-compatible /v1/messages
+    /^api\.minimax\.chat$/,     // MiniMax legacy domain
+    /^api\.minimax\.io$/,       // MiniMax new domain (Vincent live use)
+  ],
+  // P2:
+  // openai:     [/^api\.openai\.com$/],
+  // zai:        [/^api\.z\.ai$/, /^open\.bigmodel\.cn$/],
+  // openrouter: [/^openrouter\.ai$/],
+  // qwen:       [/^dashscope(-intl)?\.aliyuncs\.com$/],
+};
+export const SUPPORTED_VENDORS = Object.keys(VENDOR_HOST_ALLOWLIST);
+
+// ── §4.4.2 private/reserved IP block (anti SSRF) ──────────────────
+export const FORBIDDEN_IPV4_RE: ReadonlyArray<RegExp> = [
+  /^10\./,
+  /^127\./,
+  /^172\.(1[6-9]|2[0-9]|3[0-1])\./,
+  /^192\.168\./,
+  /^169\.254\./,           // link-local + 169.254.169.254 cloud metadata
+  /^100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\./,  // CGNAT
+  /^0\./,
+  /^22[4-9]\./, /^23[0-9]\./,  // multicast
+  /^24[0-9]\./, /^25[0-5]\./,  // experimental
+];
+export const FORBIDDEN_IPV6_RE: ReadonlyArray<RegExp> = [
+  /^::1$/,
+  /^::$/,
+  /^fe80:/i, /^fc00:/i, /^fd00:/i,
+];
+
+export function isLoopbackHost(host: string): boolean {
+  return host === "localhost" || host === "127.0.0.1" || host === "::1";
+}
+
+export function isForbiddenIp(ip: string): boolean {
+  // IPv4-mapped IPv6: ::ffff:10.0.0.1 — recurse on inner v4
+  if (ip.toLowerCase().startsWith("::ffff:")) {
+    return isForbiddenIp(ip.slice(7));
+  }
+  // Simple v4/v6 distinction by presence of '.' / ':'
+  if (ip.includes(":") && !ip.includes(".")) {
+    return FORBIDDEN_IPV6_RE.some(re => re.test(ip));
+  }
+  return FORBIDDEN_IPV4_RE.some(re => re.test(ip));
+}
+
+// ── §4.4.3 validateBaseUrl — boot/pre-fetch URL+host+vendor check ─
+/** Pure, no I/O. Called by hub (upsert_provider write path) AND
+ *  daemon (handleProbeDoorbell re-check after get_probe_request).
+ *  allowLoopback: dev-only opt-in honoring http://localhost / 127.0.0.1.
+ *  Production daemon MUST NOT set allowLoopback=true. */
+export function validateBaseUrl(
+  vendor: string,
+  baseUrl: string,
+  opts: { allowLoopback?: boolean } = {},
+): void {
+  let u: URL;
+  try { u = new URL(baseUrl); }
+  catch {
+    throw new ProbeValidationError("probe_base_url_invalid", {
+      reason: "not a valid URL", baseUrl: baseUrl.slice(0, 100),
+    });
+  }
+
+  if (u.protocol !== "https:") {
+    if (!opts.allowLoopback || !isLoopbackHost(u.hostname) || u.protocol !== "http:") {
+      throw new ProbeValidationError("probe_base_url_invalid", {
+        reason: "must be https (or http+loopback with dev opt-in)",
+      });
+    }
+  }
+  const allowed = VENDOR_HOST_ALLOWLIST[vendor];
+  if (!allowed) {
+    throw new ProbeValidationError("vendor_not_supported", {
+      vendor, supported: SUPPORTED_VENDORS,
+    });
+  }
+  if (!allowed.some(re => re.test(u.hostname))) {
+    throw new ProbeValidationError("probe_target_forbidden", {
+      vendor, host: u.hostname, allowed: allowed.map(r => r.source),
+    });
+  }
+}