@sleep2agi/commhub-server 0.9.0-preview.14 → 0.9.0-preview.140

package/src/env-transport-gate-http.test.ts

@@ -0,0 +1,146 @@
+// #1510 — the secret-transport gate, shipped as log-only first.
+//
+// The load-bearing property is symmetric: with the gate in log mode a write to a
+// `plain` node MUST still succeed (nobody's workflow breaks on upgrade) but MUST
+// leave a hit behind, and with `enforce` the same write MUST be refused. A
+// half-implemented gate that logs but also refuses would break every plain-HTTP
+// node on the next deploy; one that refuses but never logs leaves nothing to
+// decide with.
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+const PRIVATE_DB_DIR = mkdtempSync(join(tmpdir(), "anet-transport-gate-"));
+process.env.COMMHUB_DB ||= join(PRIVATE_DB_DIR, "hub.db");
+
+let net = "", ownerId = "", ownerToken = "";
+let server: ReturnType<typeof Bun.serve>;
+let base = "";
+const PLAIN = "n_gate_plain";
+const TLS = "n_gate_tls";
+
+const post = async (path: string, token: string, body: unknown) => {
+  const r = await fetch(`${base}${path}`, {
+    method: "POST",
+    headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+  });
+  return { status: r.status, body: await r.json() as any };
+};
+const get = async (path: string, token: string) => {
+  const r = await fetch(`${base}${path}`, { headers: { Authorization: `Bearer ${token}` } });
+  return { status: r.status, body: await r.json() as any };
+};
+
+beforeAll(async () => {
+  const { db } = await import("./db.js");
+  const { register } = await import("./auth.js");
+  const stamp = Date.now();
+  const owner = register(`gate_owner_${stamp}`, "GateOwner-Strong-1!");
+  expect(owner.ok).toBe(true);
+  ownerToken = owner.token!;
+  net = owner.network_id!;
+  ownerId = db.get<{ user_id: string }>("SELECT user_id FROM users WHERE username = ?1", `gate_owner_${stamp}`)!.user_id;
+
+  const seed = (nodeId: string, transport: string) => db.run(
+    `INSERT INTO nodes (node_id, node_name, alias, network_id, lifecycle_state, config_snapshot)
+     VALUES (?1, ?1, ?1, ?2, 'active', ?3)`,
+    [nodeId, net, JSON.stringify({ node_secrets_capable: true })],
+  );
+  seed(PLAIN, "plain");
+  seed(TLS, "https");
+  // The transport is reported on the session row (that is where report_status
+  // writes it).
+  const session = (nodeId: string, transport: string) => db.run(
+    `INSERT INTO sessions (resume_id, alias, status, node_id, network_id, env_transport, updated_at, last_seen_at)
+     VALUES (?1, ?1, 'idle', ?1, ?2, ?3, datetime('now'), datetime('now'))`,
+    [nodeId, net, transport],
+  );
+  session(PLAIN, "plain");
+  session(TLS, "https");
+
+  const mod = await import("./server.js");
+  server = mod.bootServer({ port: 0, hostname: "127.0.0.1" });
+  base = `http://127.0.0.1:${server.port}`;
+});
+
+afterAll(() => {
+  try { server?.stop(true); } catch {}
+  try { rmSync(PRIVATE_DB_DIR, { recursive: true, force: true }); } catch {}
+});
+
+beforeEach(async () => {
+  // The lane allows one pending request per node; drain whatever the previous
+  // case left so the next one starts clean.
+  const { pullNodeSecrets, ackNodeSecrets } = await import("./scoped-secrets.js");
+  for (const nodeId of [PLAIN, TLS]) {
+    const daemon = { ok: true as const, networkId: net, daemonNodeId: nodeId, daemonAlias: nodeId };
+    const pulled = pullNodeSecrets(daemon) as any;
+    for (const r of (pulled?.requests ?? [])) ackNodeSecrets(daemon, r.request_id, null, true);
+  }
+  process.env.ANET_SECRET_TRANSPORT_GATE = "log";
+});
+
+describe("log mode (default)", () => {
+  test("a put to a plain node still succeeds AND leaves a hit", async () => {
+    const r = await post(`/api/nodes/${PLAIN}/secrets?network_id=${net}`, ownerToken, { action: "put", key: "A_KEY", value: "v", revision: 1 });
+    expect(r.status).toBe(200);
+    expect(r.body.ok).toBe(true); // not refused: that is the whole point of log mode
+
+    const log = await get(`/api/networks/${net}/env-transport-log`, ownerToken);
+    expect(log.status).toBe(200);
+    expect(log.body.total).toBe(1);
+    expect(log.body.rows[0]).toMatchObject({ node_id: PLAIN, route: "node_secret_put", hits: 1 });
+  });
+
+  test("a put to a TLS node leaves no hit", async () => {
+    const before = (await get(`/api/networks/${net}/env-transport-log`, ownerToken)).body.total;
+    const r = await post(`/api/nodes/${TLS}/secrets?network_id=${net}`, ownerToken, { action: "put", key: "B_KEY", value: "v", revision: 1 });
+    expect(r.status).toBe(200);
+    const after = (await get(`/api/networks/${net}/env-transport-log`, ownerToken)).body.total;
+    expect(after).toBe(before);
+  });
+
+  test("list is not observed (it carries no secret)", async () => {
+    const before = (await get(`/api/networks/${net}/env-transport-log`, ownerToken)).body.total;
+    await post(`/api/nodes/${PLAIN}/secrets?network_id=${net}`, ownerToken, { action: "list" });
+    const after = (await get(`/api/networks/${net}/env-transport-log`, ownerToken)).body.total;
+    expect(after).toBe(before);
+  });
+});
+
+describe("enforce mode", () => {
+  test("the same put is refused, and nothing is queued", async () => {
+    process.env.ANET_SECRET_TRANSPORT_GATE = "enforce";
+    const r = await post(`/api/nodes/${PLAIN}/secrets?network_id=${net}`, ownerToken, { action: "put", key: "C_KEY", value: "v", revision: 1 });
+    expect(r.status).toBe(400);
+    expect(r.body.error).toBe("secret_transport_plain");
+  });
+
+  test("a TLS node is unaffected", async () => {
+    process.env.ANET_SECRET_TRANSPORT_GATE = "enforce";
+    const r = await post(`/api/nodes/${TLS}/secrets?network_id=${net}`, ownerToken, { action: "put", key: "D_KEY", value: "v", revision: 1 });
+    expect(r.status).toBe(200);
+  });
+});
+
+describe("off mode", () => {
+  test("records nothing at all", async () => {
+    process.env.ANET_SECRET_TRANSPORT_GATE = "off";
+    const before = (await get(`/api/networks/${net}/env-transport-log`, ownerToken)).body.total;
+    const r = await post(`/api/nodes/${PLAIN}/secrets?network_id=${net}`, ownerToken, { action: "put", key: "E_KEY", value: "v", revision: 1 });
+    expect(r.status).toBe(200);
+    const after = (await get(`/api/networks/${net}/env-transport-log`, ownerToken)).body.total;
+    expect(after).toBe(before);
+  });
+});
+
+describe("the log endpoint is owner/admin only", () => {
+  test("an unrelated user gets 403", async () => {
+    const { register } = await import("./auth.js");
+    const other = register(`gate_other_${Date.now()}`, "GateOther-Strong-1!");
+    const r = await get(`/api/networks/${net}/env-transport-log`, other.token!);
+    expect(r.status).toBe(403);
+  });
+});

package/src/env-transport-gate.ts

@@ -0,0 +1,74 @@
+// #1510 (log-only) — record the secret writes that a future transport gate would
+// have refused, without refusing any of them.
+//
+// Why log-only first: refusing makes an existing workflow fail on upgrade, and
+// the nodes that talk plain HTTP today are exactly the ones we cannot see from
+// the Hub. This table answers "how many writes would have been blocked, on which
+// node" *before* anyone decides to turn the gate on — the same rollout shape as
+// #519's stale-open sweep and #525's human_only log mode.
+//
+// Hourly aggregation, like node_permission_log: a chatty node must not turn this
+// into one row per write.
+import { db } from "./db.js";
+import { getEnvTransportMode } from "./env-transport-policy.js";
+
+const HOURS_RETAINED = 30 * 24;
+const MAX_ROWS = 20_000;
+
+function hourOf(now = new Date()): string {
+  return now.toISOString().slice(0, 13);
+}
+
+/** Record one would-refuse hit. Never throws: logging must not break a write. */
+export function noteEnvTransportWouldRefuse(networkId: string, nodeId: string, route: string, sample?: string): void {
+  try {
+    const now = new Date();
+    if (Math.random() < 0.02) {
+      db.run("DELETE FROM env_transport_would_refuse WHERE hour < ?1",
+        [hourOf(new Date(now.getTime() - HOURS_RETAINED * 86_400_000))]);
+    }
+    const count = Number(db.get<{ n: number | string }>("SELECT COUNT(*) AS n FROM env_transport_would_refuse")?.n ?? 0);
+    const hour = hourOf(now);
+    const s = sample ? sample.slice(0, 120) : null;
+    if (count >= MAX_ROWS) {
+      db.run(
+        "UPDATE env_transport_would_refuse SET hits = hits + 1, sample = COALESCE(sample, ?5) WHERE network_id = ?1 AND node_id = ?2 AND route = ?3 AND hour = ?4",
+        [networkId, nodeId, route, hour, s],
+      );
+      return;
+    }
+    db.run(
+      `INSERT INTO env_transport_would_refuse (network_id, node_id, route, hour, hits, sample) VALUES (?1, ?2, ?3, ?4, 1, ?5)
+       ON CONFLICT(network_id, node_id, route, hour) DO UPDATE SET hits = env_transport_would_refuse.hits + 1`,
+      [networkId, nodeId, route, hour, s],
+    );
+  } catch (e) {
+    console.error(`[env-transport] would-refuse log failed: ${(e as Error).message}`);
+  }
+}
+
+/** Where a secret write is aimed: the node's own row, or a daemon's. Returns the
+ *  transport the node last reported, or null when it is unknown. */
+export function transportOf(networkId: string, nodeId: string): string | null {
+  const row = db.get<{ env_transport: string | null }>(
+    "SELECT env_transport FROM sessions WHERE network_id = ?1 AND (node_id = ?2 OR alias = ?2) ORDER BY updated_at DESC LIMIT 1",
+    [networkId, nodeId],
+  );
+  return row?.env_transport ?? null;
+}
+
+/** Called from the write lanes. Logs when the current policy would refuse. */
+export function observeSecretWrite(networkId: string, nodeId: string, route: string): void {
+  if (getEnvTransportMode() !== "log") return;
+  const transport = transportOf(networkId, nodeId);
+  if (transport !== "plain") return;
+  noteEnvTransportWouldRefuse(networkId, nodeId, route, `transport=${transport}`);
+}
+
+export function readEnvTransportLog(networkId: string, sinceHour: string) {
+  return db.all<{ node_id: string; route: string; hour: string; hits: number | string; sample: string | null }>(
+    `SELECT node_id, route, hour, hits, sample FROM env_transport_would_refuse
+      WHERE network_id = ?1 AND hour >= ?2 ORDER BY hour DESC, node_id, route`,
+    [networkId, sinceHour],
+  ).map(r => ({ ...r, hits: Number(r.hits) }));
+}

package/src/env-transport-policy.ts

@@ -0,0 +1,21 @@
+// #1510 — the switch for the secret-transport gate.
+//
+//   (unset) / "log" — record what a gate WOULD refuse; refuse nothing.
+//   "enforce"       — refuse secret writes to nodes that report `plain`.
+//   "off"           — do nothing at all (no recording either).
+//
+// Reading the env on every call (like the stale-open sweep) means an operator can
+// change the mode without a code change and tests can flip it per case.
+export type EnvTransportMode = "off" | "log" | "enforce";
+
+export function getEnvTransportMode(env: Record<string, string | undefined> = process.env): EnvTransportMode {
+  const raw = (env.ANET_SECRET_TRANSPORT_GATE ?? "").trim().toLowerCase();
+  if (raw === "off") return "off";
+  if (raw === "enforce") return "enforce";
+  return "log";
+}
+
+/** True when a write to a node reporting `plain` must be refused. */
+export function refusesPlainTransport(transport: string | null, env: Record<string, string | undefined> = process.env): boolean {
+  return getEnvTransportMode(env) === "enforce" && transport === "plain";
+}

package/src/ext-token-shared.test.ts

@@ -0,0 +1,60 @@
+import { describe, expect, test } from "bun:test";
+import {
+  buildStoragePath,
+  pathForExistingBlob,
+  validateIndexEntry,
+} from "./uploads.js";
+
+const fileId = "0123456789abcdef0123456789abcdef";
+const dateBucket = "2026-08-09";
+const uploadsRoot = "/srv/uploads";
+
+const malicious = [
+  "../x",
+  "/abs/x",
+  ".png/../../y",
+  ".p ng",
+  ".p\0ng",
+  ".\\..\\x",
+  "." + "a".repeat(17),
+  ".tar.gz",
+];
+
+const legal = ["", ".png", ".PNG", ".mp4", "." + "a".repeat(16)];
+
+describe("#527 shared extension-token invariant", () => {
+  test("new-upload path rejects every malformed token and accepts the shared legal set", () => {
+    for (const ext of malicious) {
+      expect(() => buildStoragePath(fileId, ext, { uploadsRoot })).toThrow(/ext .* is invalid/);
+    }
+    for (const ext of legal) {
+      expect(buildStoragePath(fileId, ext, { uploadsRoot }).ext).toBe(ext);
+    }
+  });
+
+  test("existing-blob path rejects every malformed token and accepts the shared legal set", () => {
+    for (const ext of malicious) {
+      expect(() => pathForExistingBlob(dateBucket, fileId, ext, { uploadsRoot })).toThrow(/ext .* is invalid/);
+    }
+    for (const ext of legal) {
+      expect(pathForExistingBlob(dateBucket, fileId, ext, { uploadsRoot }).ext).toBe(ext);
+    }
+  });
+
+  test("stored-index gate rejects every malformed token and accepts the shared legal set", () => {
+    for (const ext of malicious) {
+      expect(validateIndexEntry({ file_id: fileId, date_bucket: dateBucket, ext, size: 0 })).toBe(false);
+    }
+    for (const ext of legal) {
+      expect(validateIndexEntry({ file_id: fileId, date_bucket: dateBucket, ext, size: 0 })).toBe(true);
+    }
+  });
+
+  test("all three boundaries fail closed on non-string runtime values", () => {
+    for (const ext of [undefined, null, 7, {}, []]) {
+      expect(() => buildStoragePath(fileId, ext as never, { uploadsRoot })).toThrow(/ext .* is invalid/);
+      expect(() => pathForExistingBlob(dateBucket, fileId, ext as never, { uploadsRoot })).toThrow(/ext .* is invalid/);
+      expect(validateIndexEntry({ file_id: fileId, date_bucket: dateBucket, ext, size: 0 })).toBe(false);
+    }
+  });
+});

package/src/ext-validation-regression.test.ts

@@ -0,0 +1,131 @@
+// Task #25 — regression pins for ext validation in validateIndexEntry.
+//
+// 🔴 SCOPE: this suite exercises the stored-index boundary through
+// `validateIndexEntry`. Issue #527 removed the three independent copies
+// of the stored extension-token grammar: buildStoragePath,
+// pathForExistingBlob and validateIndexEntry now use one shared private
+// validator. `ext-token-shared.test.ts` separately pins all three call
+// sites. sanitizeExt remains intentionally independent because it parses
+// a client filename rather than validating an already-extracted token.
+//
+// Background (通信龙 d01bb8ce): the stored extension-token grammar
+// `/^\.[A-Za-z0-9]{1,16}$/` correctly rejects 11 known malicious ext
+// values (path-escape, traversal, absolute paths, doubles, NUL bytes,
+// backslash forms, over-length, whitespace). Verified via real-run on
+// origin/main by lead — 11/11 malicious values → false.
+//
+// But NO test locked that behavior. Anyone who widens the regex in the
+// future (accidentally or on purpose) would not hit a red test — the
+// live protection would degrade silently. This suite pins current
+// behavior in two directions:
+//
+//   1. SAFETY  (欠修 direction) — 11 malicious values → false, one row
+//      per value with the exact literal. If regex is widened to accept
+//      any of them, that row turns red.
+//   2. NON-REGRESSION (过修 direction) — a set of common legitimate
+//      extensions → true. If regex is tightened to reject any of them
+//      (e.g. someone locks down to a whitelist), that row turns red.
+//
+// Witnessed-red for the SAFETY rows: temporarily widen the regex to
+// `/./` (or comment it out) in validateIndexEntry, run this suite,
+// confirm all 11 malicious rows turn red, then revert. The mutation
+// itself is not committed; the assertion IS the mutation-red proof
+// for the reader. Recording the mutation output in the followup PR
+// body (per Constraint 3 discipline established in #503).
+//
+// Pure unit test — no server setup, no DB, no filesystem. The regex
+// lives in a pure function, so the guard is testable at that level.
+
+import { describe, expect, test } from "bun:test";
+import { validateIndexEntry } from "./uploads.js";
+
+// A structurally-valid base entry — file_id / date_bucket / size all
+// pass their own checks — so `ext` is the only field a test row varies.
+// This isolates the ext regex as the deciding factor.
+const validBase = {
+  file_id: "0123456789abcdef0123456789abcdef",
+  date_bucket: "2026-07-30",
+  size: 0,
+};
+
+describe("Task #25 (#503 followup) — ext validation regression pins", () => {
+  describe("SAFETY: the 11 malicious ext values from lead d01bb8ce all return false", () => {
+    // Each row uses the exact literal from lead's verification list so
+    // a widening regex catches on the same value the manual audit did.
+    const malicious: Array<{ label: string; ext: string }> = [
+      { label: "path-escape prefix",          ext: "../x" },
+      { label: "deep traversal to /etc",       ext: "../../etc/passwd" },
+      { label: "absolute path",               ext: "/abs/x" },
+      { label: "double-dot alone",            ext: ".." },
+      { label: "single-dot alone",            ext: "." },
+      { label: "chained after legit prefix",  ext: ".png/../../y" },
+      { label: "embedded whitespace",         ext: ".p ng" },
+      { label: "over-length (>16 chars)",     ext: "." + "a".repeat(17) },
+      { label: "embedded NUL byte",           ext: ".p\x00ng" },
+      { label: "backslash traversal",         ext: ".\\..\\x" },
+      { label: "double-backslash prefix",     ext: "..\\x" },
+    ];
+    for (const { label, ext } of malicious) {
+      test(`rejects: ${label} — ext=${JSON.stringify(ext)}`, () => {
+        expect(validateIndexEntry({ ...validBase, ext })).toBe(false);
+      });
+    }
+
+    test("precondition: 11 malicious cases enumerated (defends against silent test-shrink)", () => {
+      // If someone deletes rows from the malicious list, this fails
+      // instead of the deletion silently narrowing the guard. Number
+      // matches lead's verification list — bump only when lead adds a
+      // case and confirms it via real-run.
+      expect(malicious.length).toBe(11);
+    });
+  });
+
+  describe("NON-REGRESSION: common legitimate extensions all return true (防过修)", () => {
+    // These are the extensions the live upload path actually produces
+    // via sanitizeExt / mime-typed multipart uploads. If someone
+    // tightens the regex to a stricter whitelist, any of these turning
+    // red flags a break in a normal user path.
+    const legal = [".png", ".pdf", ".pptx", ".jpg", ".jpeg", ".txt", ".mp4", ".zip", ".json", ".md"];
+    for (const ext of legal) {
+      test(`accepts: ext=${JSON.stringify(ext)}`, () => {
+        expect(validateIndexEntry({ ...validBase, ext })).toBe(true);
+      });
+    }
+
+    test("empty ext is accepted (file with no extension is a legitimate upload)", () => {
+      expect(validateIndexEntry({ ...validBase, ext: "" })).toBe(true);
+    });
+
+    test("precondition: at least 10 legitimate ext values pinned", () => {
+      // Same discipline as the malicious side — reject silent shrink.
+      expect(legal.length).toBeGreaterThanOrEqual(10);
+    });
+  });
+
+  describe("regex shape assumptions (documented, not enforced by the function)", () => {
+    // These rows document facts the current regex enforces beyond the
+    // 11 malicious literals. They are close to the boundary and would
+    // be the first things to move if someone reworked the regex.
+    test("upper-case extensions accepted (regex is case-insensitive by [A-Za-z])", () => {
+      expect(validateIndexEntry({ ...validBase, ext: ".PNG" })).toBe(true);
+    });
+    test("mixed-case extensions accepted", () => {
+      expect(validateIndexEntry({ ...validBase, ext: ".PdF" })).toBe(true);
+    });
+    test("digits after dot accepted (e.g. .mp4, .mp3)", () => {
+      expect(validateIndexEntry({ ...validBase, ext: ".mp4" })).toBe(true);
+    });
+    test("exactly 16 chars after dot is accepted (upper bound of {1,16})", () => {
+      expect(validateIndexEntry({ ...validBase, ext: "." + "a".repeat(16) })).toBe(true);
+    });
+    test("exactly 17 chars after dot is rejected (immediately past upper bound)", () => {
+      expect(validateIndexEntry({ ...validBase, ext: "." + "a".repeat(17) })).toBe(false);
+    });
+    test("compound extensions like .tar.gz are rejected (contains inner dot)", () => {
+      // Documented: the current regex does not support compound
+      // extensions. If we ever want .tar.gz support, it needs its
+      // own explicit design + this row updated with lead sign-off.
+      expect(validateIndexEntry({ ...validBase, ext: ".tar.gz" })).toBe(false);
+    });
+  });
+});