@sleep2agi/commhub-server 0.9.0-preview.13 → 0.9.0-preview.130

package/src/node-health-dispatch-http.test.ts

@@ -0,0 +1,279 @@
+// #460(#448 子项)—— 降级节点拒收新任务。
+//
+// 钉住:
+//   1. 新鲜健康报告说某层坏了(app_server.ok=false / tui.ok=false / model_auth revoked|expired)
+//      → REST /api/task 409、MCP send_task / retry_task / reassign_task 返回 node_degraded,带层、原因、修法;
+//      定时任务的这一次 run 记 failed + error_code=node_degraded,不建任务行。
+//   2. 不知道 = 放行:没有报告(老 agent-node)、报告过期(> TTL)、各层都 ok、某层没报 —— 与改动前逐字相同的成功路径。
+//   3. 逃生口 force=true 只对用户令牌生效;节点令牌带 force 照样被拒(不让 agent 绕过对另一个 agent 的健康判断)。
+//   4. 老客户端请求形状(不带 force)在健康 / 未知节点上完全照旧 —— 兼容回放。
+//
+// 跑法:cd server && COMMHUB_DB=/tmp/x.db bun test src/node-health-dispatch-http.test.ts
+//       PG:COMMHUB_TEST_PG_URL=… COMMHUB_PG_EXPERIMENTAL=1(tests/test2123-hub-postgres-ladder 里注册)
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { mkdtempSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { createNetworkTokenForNode, register } from "./auth.js";
+import { db } from "./db.js";
+import { registerTools } from "./tools.js";
+import { clearNodeHealthStore, NODE_HEALTH_TTL_MS, recordNodeHealth } from "./node-health-store.js";
+import { degradedLayers } from "./node-health-guard.js";
+import { runDueScheduledTasks } from "./scheduled-tasks.js";
+
+const dir = mkdtempSync(join(tmpdir(), "anet-health-dispatch-"));
+const activeDbPath = process.env.COMMHUB_DB ?? (process.env.COMMHUB_TEST_PG_URL ? "postgres" : undefined);
+if (!activeDbPath) throw new Error("node-health-dispatch requires COMMHUB_DB (or COMMHUB_TEST_PG_URL) before module import");
+
+let server: any;
+let base = "";
+let ownerToken = "", ownerId = "", networkId = "";
+let peerNodeToken = "", peerNodeTokenId = "";
+const stamp = Date.now();
+const TARGET = `hd-target-${stamp}`;
+const TARGET_NODE = `n_hd_target_${stamp}`;
+const PEER = `hd-peer-${stamp}`;
+const PEER_NODE = `n_hd_peer_${stamp}`;
+let seq = 0;
+const text = (what: string) => `${what} #${++seq}`; // distinct content: the 5-minute dedup must not interfere
+
+async function api(token: string, path: string, init?: RequestInit) {
+  const res = await fetch(`${base}${path}`, {
+    ...init,
+    headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", ...(init?.headers || {}) },
+  });
+  return { status: res.status, body: await res.json() as any };
+}
+const postTask = (token: string, extra: Record<string, unknown> = {}) =>
+  api(token, "/api/task", { method: "POST", body: JSON.stringify({ alias: TARGET, task: text("rest"), network_id: networkId, ...extra }) });
+
+async function mcpAs(identity: { net: string | null; user: string; alias: string; isNetworkToken: boolean; tokenId: string }) {
+  const s = new McpServer({ name: "health-dispatch", version: "1" });
+  registerTools(s, undefined, identity.net, identity.user, identity.alias, identity.isNetworkToken, identity.tokenId);
+  const client = new Client({ name: "health-dispatch-client", version: "1" });
+  const [ct, st] = InMemoryTransport.createLinkedPair();
+  await s.connect(st);
+  await client.connect(ct);
+  const call = async (name: string, args: Record<string, unknown>) => {
+    const r: any = await client.callTool({ name, arguments: args });
+    const raw = r.content?.[0]?.text ?? "null";
+    try { return JSON.parse(raw); } catch { return { raw }; }
+  };
+  return { call, close: async () => { await client.close(); await s.close(); } };
+}
+
+const DOWN_APP = { bridge: "ok" as const, app_server: { ok: false, rtt_ms: null, last_error: "connect ECONNREFUSED 127.0.0.1:4500" }, model_auth: "ok" as const };
+const HEALTHY = { bridge: "ok" as const, app_server: { ok: true, rtt_ms: 3, last_error: null }, tui: { ok: true, reason: "running" }, model_auth: "ok" as const };
+const setHealth = (h: any, at?: number) => recordNodeHealth(networkId, TARGET, h, at);
+const taskCount = () => db.get<{ n: number }>("SELECT COUNT(*) AS n FROM tasks WHERE to_name = ?1 AND network_id = ?2", TARGET, networkId)!.n;
+
+beforeAll(async () => {
+  process.env.COMMHUB_UPLOADS_DIR = join(dir, "uploads");
+  process.env.HOST = "127.0.0.1";
+  const owner = register(`hd_owner_${stamp}`, "HealthDispatchOwner123!", undefined, "seed");
+  ownerToken = owner.token!; ownerId = owner.user!.user_id; networkId = owner.network_id!;
+  for (const [nodeId, alias] of [[TARGET_NODE, TARGET], [PEER_NODE, PEER]]) {
+    db.run(`INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, owner_user_id) VALUES (?1, ?2, ?2, 'codex-app-server', ?3, ?4)`, [nodeId, alias, networkId, ownerId]);
+    db.run(`INSERT INTO sessions (resume_id, alias, status, node_id, network_id) VALUES (?1, ?2, 'idle', ?3, ?4)`, [`r_${nodeId}`, alias, nodeId, networkId]);
+  }
+  const minted = createNetworkTokenForNode(ownerId, networkId, PEER, PEER_NODE);
+  peerNodeToken = minted.token!; peerNodeTokenId = minted.token_id!;
+  const mod: any = await import("./server.js");
+  server = mod.bootServer({ port: 0, hostname: "127.0.0.1" });
+  base = `http://127.0.0.1:${server.port}`;
+}, 30_000);
+afterAll(() => { server?.stop?.(true); clearNodeHealthStore(); });
+beforeEach(() => clearNodeHealthStore());
+
+describe("#460 which layers count as degraded", () => {
+  test("each failing layer is named; ok / missing layers are not", () => {
+    expect(degradedLayers(null)).toEqual([]);
+    expect(degradedLayers(HEALTHY)).toEqual([]);
+    expect(degradedLayers({ bridge: "ok", model_auth: "unknown" })).toEqual([]);
+    expect(degradedLayers(DOWN_APP).map(l => [l.layer, l.label])).toEqual([["app_server", "App Server 断开"]]);
+    expect(degradedLayers({ tui: { ok: false, reason: "sleep-placeholder" } }).map(l => [l.layer, l.reason, l.label])).toEqual([["tui", "sleep-placeholder", "TUI 被占位(sleep)"]]);
+    expect(degradedLayers({ model_auth: "revoked" }).map(l => l.label)).toEqual(["需要重新登录"]);
+    expect(degradedLayers({ model_auth: "expired" }).map(l => l.label)).toEqual(["登录已过期"]);
+    // 修法只指向本节点重新登录,不教人拷别人的凭据
+    expect(degradedLayers({ model_auth: "revoked" })[0].hint).toContain("不要拷别的节点的 auth.json");
+  });
+});
+
+describe("#460 REST /api/task", () => {
+  test("unknown health (old agent-node) → dispatched exactly as before", async () => {
+    const before = taskCount();
+    const r = await postTask(ownerToken);
+    expect(r.status).toBe(200);
+    expect(r.body.ok).toBe(true);
+    expect(taskCount()).toBe(before + 1);
+  });
+
+  test("healthy report → dispatched", async () => {
+    setHealth(HEALTHY);
+    expect((await postTask(ownerToken)).status).toBe(200);
+  });
+
+  test("App Server down → 409 node_degraded with layer, reason and hint; no task row", async () => {
+    setHealth(DOWN_APP);
+    const before = taskCount();
+    const r = await postTask(ownerToken);
+    expect(r.status).toBe(409);
+    expect(r.body.ok).toBe(false);
+    expect(r.body.error).toBe("node_degraded");
+    expect(r.body.alias).toBe(TARGET);
+    expect(r.body.layers.map((l: any) => l.layer)).toEqual(["app_server"]);
+    expect(r.body.layers[0].reason).toContain("ECONNREFUSED");
+    expect(r.body.message).toContain("App Server 断开");
+    expect(r.body.hint).toContain("force=true");
+    expect(r.body.force_allowed).toBe(true);
+    expect(typeof r.body.health_observed_ms_ago).toBe("number");
+    expect(taskCount()).toBe(before);
+  });
+
+  test("TUI gone and login revoked are refused too, all layers listed", async () => {
+    setHealth({ bridge: "ok", app_server: { ok: true, rtt_ms: 2, last_error: null }, tui: { ok: false, reason: "session-missing" }, model_auth: "revoked" });
+    const r = await postTask(ownerToken);
+    expect(r.status).toBe(409);
+    expect(r.body.layers.map((l: any) => l.layer)).toEqual(["tui", "model_auth"]);
+  });
+
+  test("expired login alone is refused", async () => {
+    setHealth({ bridge: "ok", model_auth: "expired" });
+    expect((await postTask(ownerToken)).body.error).toBe("node_degraded");
+  });
+
+  test("stale report (older than the TTL) = unknown → dispatched", async () => {
+    setHealth(DOWN_APP, Date.now() - NODE_HEALTH_TTL_MS - 1_000);
+    expect((await postTask(ownerToken)).status).toBe(200);
+  });
+
+  test("owner escape hatch: force=true with a user token dispatches", async () => {
+    setHealth(DOWN_APP);
+    const before = taskCount();
+    const r = await postTask(ownerToken, { force: true });
+    expect(r.status).toBe(200);
+    expect(taskCount()).toBe(before + 1);
+  });
+
+  test("a node token cannot force past another node's health", async () => {
+    setHealth(DOWN_APP);
+    const r = await api(peerNodeToken, "/api/task", { method: "POST", body: JSON.stringify({ alias: TARGET, task: text("node-force"), force: true }) });
+    expect(r.status).toBe(409);
+    expect(r.body.error).toBe("node_degraded");
+    expect(r.body.force_allowed).toBe(false);
+    expect(r.body.hint).not.toContain("force=true");
+  });
+
+  test("old-client request shapes are untouched on a healthy node (compat replay)", async () => {
+    setHealth(HEALTHY);
+    // the app's send (meta + client_request_id), a plain anet send, and a parent-linked dispatch
+    const shapes = [
+      { alias: TARGET, task: text("app"), network_id: networkId, priority: "normal", meta: { client_request_id: `cr_${stamp}_${seq}` } },
+      { alias: TARGET, task: text("anet"), network_id: networkId },
+      { alias: TARGET, task: text("ttl"), network_id: networkId, priority: "high", ttl_seconds: 600 },
+    ];
+    for (const shape of shapes) {
+      const r = await api(ownerToken, "/api/task", { method: "POST", body: JSON.stringify(shape) });
+      expect(r.status).toBe(200);
+      expect(r.body.ok).toBe(true);
+    }
+  });
+});
+
+describe("#460 MCP send_task / retry_task / reassign_task", () => {
+  const asOwner = () => ({ net: null, user: ownerId, alias: `hd_owner_${stamp}`, isNetworkToken: false, tokenId: "tok_hd_owner_login" });
+  const asPeerNode = () => ({ net: networkId, user: ownerId, alias: PEER, isNetworkToken: true, tokenId: peerNodeTokenId });
+
+  test("send_task from an agent to a degraded node → node_degraded (no queueing); force ignored for node tokens", async () => {
+    setHealth(DOWN_APP);
+    const peer = await mcpAs(asPeerNode());
+    try {
+      const before = taskCount();
+      const r = await peer.call("send_task", { alias: TARGET, task: text("mcp"), force: true });
+      expect(r.ok).toBe(false);
+      expect(r.error).toBe("node_degraded");
+      expect(r.force_allowed).toBe(false);
+      expect(taskCount()).toBe(before);
+    } finally { await peer.close(); }
+  });
+
+  test("send_task: unknown health → ok; owner force → ok", async () => {
+    const owner = await mcpAs(asOwner());
+    try {
+      expect((await owner.call("send_task", { alias: TARGET, task: text("mcp-unknown"), network_id: networkId })).ok).toBe(true);
+      setHealth({ bridge: "ok", model_auth: "revoked" });
+      expect((await owner.call("send_task", { alias: TARGET, task: text("mcp-refused"), network_id: networkId })).error).toBe("node_degraded");
+      expect((await owner.call("send_task", { alias: TARGET, task: text("mcp-forced"), network_id: networkId, force: true })).ok).toBe(true);
+    } finally { await owner.close(); }
+  });
+
+  test("retry_task and reassign_task onto a degraded node are refused", async () => {
+    const owner = await mcpAs(asOwner());
+    try {
+      const sent = await owner.call("send_task", { alias: PEER, task: text("to-peer"), network_id: networkId });
+      expect(sent.ok).toBe(true);
+      setHealth(DOWN_APP);
+      const re = await owner.call("reassign_task", { task_id: sent.task_id ?? sent.message_id, new_alias: TARGET, network_id: networkId });
+      expect(re.error).toBe("node_degraded");
+      const failed = await owner.call("send_task", { alias: TARGET, task: text("to-fail"), network_id: networkId, force: true });
+      const failedId = failed.task_id ?? failed.message_id;
+      db.run("UPDATE tasks SET status = 'failed' WHERE task_id = ?1", [failedId]);
+      const retry = await owner.call("retry_task", { task_id: failedId, network_id: networkId });
+      expect(retry.error).toBe("node_degraded");
+      clearNodeHealthStore();
+      expect((await owner.call("retry_task", { task_id: failedId, network_id: networkId })).ok).toBe(true);
+    } finally { await owner.close(); }
+  });
+});
+
+describe("#460 scheduled tasks", () => {
+  test("a due run on a degraded node fails with node_degraded and creates no task", async () => {
+    const created = await api(ownerToken, "/api/scheduled-tasks", {
+      method: "POST",
+      body: JSON.stringify({ network_id: networkId, name: `hd-schedule-${stamp}`, target_node_id: TARGET_NODE, task: text("scheduled"), timezone: "UTC", schedule: { type: "interval", every_seconds: 60 } }),
+    });
+    expect(created.status).toBe(201);
+    const scheduleId = created.body.schedule.schedule_id;
+    setHealth({ bridge: "ok", tui: { ok: false, reason: "pane-dead" }, model_auth: "ok" });
+    const before = taskCount();
+    db.run("UPDATE scheduled_tasks SET next_run_at = ?1 WHERE schedule_id = ?2", [new Date(Date.now() - 60_000).toISOString(), scheduleId]);
+    runDueScheduledTasks();
+    const run = db.get<{ status: string; error_code: string | null; error_message: string | null; task_id: string | null }>(
+      "SELECT status, error_code, error_message, task_id FROM scheduled_task_runs WHERE schedule_id = ?1 ORDER BY created_at DESC LIMIT 1", scheduleId);
+    expect(run?.status).toBe("failed");
+    expect(run?.error_code).toBe("node_degraded");
+    expect(run?.error_message).toContain("TUI 已退出");
+    expect(run?.task_id ?? null).toBeNull();
+    expect(taskCount()).toBe(before);
+  });
+});
+
+describe("#460 /api/status carries `degraded` only for degraded nodes", () => {
+  const row = async (light: boolean) => {
+    const r = await api(ownerToken, `/api/status?network_id=${encodeURIComponent(networkId)}${light ? "&light=1" : ""}`);
+    expect(r.status).toBe(200);
+    return r.body.sessions.find((x: any) => x.alias === TARGET);
+  };
+  test("unknown / healthy → no `degraded` key at all (light response bytes unchanged)", async () => {
+    expect("degraded" in (await row(true))).toBe(false);
+    setHealth(HEALTHY);
+    expect("degraded" in (await row(true))).toBe(false);
+    expect("degraded" in (await row(false))).toBe(false);
+  });
+  test("degraded → light and full rows list the failing layers with the app's labels", async () => {
+    setHealth({ bridge: "ok", app_server: { ok: false, rtt_ms: null, last_error: "refused" }, model_auth: "revoked" });
+    const want = [
+      { layer: "app_server", label: "App Server 断开", reason: "refused" },
+      { layer: "model_auth", label: "需要重新登录", reason: "revoked" },
+    ];
+    expect((await row(true)).degraded).toEqual(want);
+    expect((await row(false)).degraded).toEqual(want);
+  });
+  test("stale → no `degraded`", async () => {
+    setHealth(DOWN_APP, Date.now() - NODE_HEALTH_TTL_MS - 1_000);
+    expect("degraded" in (await row(true))).toBe(false);
+  });
+});

package/src/node-health-guard.ts

@@ -0,0 +1,105 @@
+// #460(#448 的子项)—— 降级节点拒收新任务。
+//
+// 现场:节点在 Hub 上 idle / in_flight=0,实际 app-server 端口没人听、TUI 变成 `sleep infinity`、
+// 或登录态 revoked —— 派过去的任务**静默排队**,谁都不知道它永远不会被执行。agent-node .94 起
+// 把这几层报上来(report_status.health,node-health-store.ts 只存内存、10 分钟 TTL);这里在**派发**
+// 那一刻读它:新鲜的健康报告说某一层坏了,就直接拒,带上是哪一层、怎么修。
+//
+// 🔴 只拒「确知坏了」的:没有报告(老 agent-node / claude-code 节点)、报告已过期(>TTL)、
+//    或某层没报(比如非共存节点没有 tui)= 不知道 = 放行。宁可像以前一样排队,
+//    也不能因为缺数据把一台正常的老节点挡在外面。
+// 🔴 只管**新任务派发**(send_task / REST /api/task / 定时任务 / retry / reassign / broadcast)。
+//    回复、send_message、ack 不经过这里 —— 降级节点发出的回复照常送达,发给它的普通消息也照常入库。
+// 逃生口:用户(不是节点令牌)可以带 force=true 强制派发 —— 比如明知 TUI 不在、但 bridge 能干活。
+//    节点令牌不能 force:不让 agent 绕过对另一个 agent 的健康判断。
+
+import { readNodeHealth, type NodeHealth } from "./node-health-store.js";
+
+export type DegradedLayer = {
+  layer: "app_server" | "tui" | "model_auth";
+  /** 机器可读的原因:app_server 的错误摘要 / tui 的 reason / model_auth 的状态。 */
+  reason: string;
+  /** 给人看的一句(中文,与 app 的「降级」徽标同一套措辞)。 */
+  label: string;
+  hint: string;
+};
+
+const TUI_LABEL: Record<string, string> = {
+  "session-missing": "TUI 会话不在",
+  "pane-dead": "TUI 已退出",
+  "sleep-placeholder": "TUI 被占位(sleep)",
+  "tmux-unavailable": "tmux 不可用",
+};
+
+/** 一份健康报告里确知坏掉的层;没报的层不算。 */
+export function degradedLayers(health: NodeHealth | null | undefined): DegradedLayer[] {
+  if (!health) return [];
+  const out: DegradedLayer[] = [];
+  if (health.app_server && health.app_server.ok === false) {
+    out.push({
+      layer: "app_server",
+      reason: (health.app_server.last_error || "unreachable").slice(0, 200),
+      label: "App Server 断开",
+      hint: "节点的 codex app-server 端口没有响应:在节点所在机器上重启该节点(anet node restart <别名>)",
+    });
+  }
+  if (health.tui && health.tui.ok === false) {
+    out.push({
+      layer: "tui",
+      reason: health.tui.reason,
+      label: TUI_LABEL[health.tui.reason] ?? "TUI 不可用",
+      hint: "共存 TUI 不在运行:在节点所在机器上重新拉起该节点的 TUI(anet node restart <别名>)",
+    });
+  }
+  if (health.model_auth === "revoked" || health.model_auth === "expired") {
+    out.push({
+      layer: "model_auth",
+      reason: health.model_auth,
+      label: health.model_auth === "revoked" ? "需要重新登录" : "登录已过期",
+      hint: "该节点自己的 CODEX_HOME 需要重新登录(在那台机器上:CODEX_HOME=<节点目录>/codex-home codex login);不要拷别的节点的 auth.json",
+    });
+  }
+  return out;
+}
+
+export type HealthGuardResult =
+  | { ok: true; forced?: true }
+  | {
+    ok: false;
+    error: "node_degraded";
+    alias: string;
+    layers: DegradedLayer[];
+    message: string;
+    hint: string;
+    health_observed_ms_ago: number;
+    /** 能不能用 force=true 绕过(只有用户令牌能)。 */
+    force_allowed: boolean;
+  };
+
+/**
+ * 派发前的健康门。networkId 用与 report_status 写入时同一个键(sessionNetId = 网络 id ?? "default")。
+ * force 只在 forceAllowed(用户令牌)时生效。
+ */
+export function assertNodeHealthy(
+  alias: string,
+  networkId: string | null | undefined,
+  opts: { force?: boolean; forceAllowed: boolean; now?: number },
+): HealthGuardResult {
+  if (!alias) return { ok: true };
+  const hit = readNodeHealth(networkId ?? "default", alias, opts.now);
+  if (!hit) return { ok: true };
+  const layers = degradedLayers(hit.health);
+  if (!layers.length) return { ok: true };
+  if (opts.force && opts.forceAllowed) return { ok: true, forced: true };
+  const labels = layers.map(l => l.label).join("、");
+  return {
+    ok: false,
+    error: "node_degraded",
+    alias,
+    layers,
+    message: `节点 ${alias} 处于降级状态(${labels}),任务不会被执行,已拒绝派发。`,
+    hint: layers[0].hint + (opts.forceAllowed ? "。确需强制派发可带 force=true。" : ""),
+    health_observed_ms_ago: hit.observed_ms_ago,
+    force_allowed: opts.forceAllowed,
+  };
+}

package/src/node-health-passthrough.test.ts

@@ -0,0 +1,115 @@
+// #448 — report_status.health passthrough (layered node health, report-only).
+//
+// Hub-side contract verified here:
+//   1. A well-formed `health` from the node's own token is kept (memory-only) and readable.
+//   2. Malformed `health` never rejects the report — the node stays online, health reads as null.
+//   3. Another node / a user login cannot speak for this node's health.
+//   4. Stale (> TTL) health reads as null, never as "healthy".
+//
+// 跑法：cd server && COMMHUB_DB=/tmp/node-health.db bun test src/node-health-passthrough.test.ts
+import { afterAll, beforeEach, describe, expect, test } from "bun:test";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { db } from "./db.js";
+import { registerTools } from "./tools.js";
+import { clearNodeHealthStore, NODE_HEALTH_TTL_MS, readNodeHealth, recordNodeHealth } from "./node-health-store.js";
+
+const NET = "net_health_t";
+const OWNER = "u_health_owner";
+const NODE_A = "health-node-a";
+const NODE_B = "health-node-b";
+const NODE_A_ID = "node_health_a";
+const NODE_B_ID = "node_health_b";
+const T_A = "tok_health_node_a";
+const T_B = "tok_health_node_b";
+
+function cleanup() {
+  for (const t of ["sessions", "nodes", "api_tokens", "network_members", "networks"]) {
+    try { db.run(`DELETE FROM ${t} WHERE network_id = ?1`, [NET]); } catch {}
+  }
+  try { db.run("DELETE FROM users WHERE user_id = ?1", [OWNER]); } catch {}
+  clearNodeHealthStore();
+}
+function seed() {
+  db.run(`INSERT INTO users (user_id, username, password_hash, role, created_at) VALUES (?1, ?2, 'x', 'user', datetime('now'))`, [OWNER, OWNER]);
+  db.run(`INSERT INTO networks (network_id, network_name, owner_id, created_at) VALUES (?1, ?2, ?3, datetime('now'))`, [NET, NET, OWNER]);
+  db.run(`INSERT INTO network_members (user_id, network_id, role, joined_at) VALUES (?1, ?2, 'owner', datetime('now'))`, [OWNER, NET]);
+  for (const [id, alias, tok] of [[NODE_A_ID, NODE_A, T_A], [NODE_B_ID, NODE_B, T_B]]) {
+    db.run(`INSERT INTO nodes (node_id, node_name, alias, network_id, owner_user_id, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, datetime('now'), datetime('now'))`, [id, alias, alias, NET, OWNER]);
+    db.run(`INSERT INTO api_tokens (token_id, user_id, network_id, scope, name, token_hash, expires_at, revoked_at, bound_node_id) VALUES (?1, ?2, ?3, 'network', ?4, ?5, NULL, NULL, ?6)`, [tok, OWNER, NET, `node:${alias}`, `hash_${tok}`, id]);
+  }
+}
+
+async function connectAs(alias: string, tokenId: string) {
+  const server = new McpServer({ name: "health-test", version: "1" });
+  registerTools(server, undefined, NET, OWNER, alias, true, tokenId);
+  const client = new Client({ name: "health-client", version: "1" });
+  const [ct, st] = InMemoryTransport.createLinkedPair();
+  await server.connect(st);
+  await client.connect(ct);
+  const call = async (args: Record<string, unknown>) => {
+    const r: any = await client.callTool({ name: "report_status", arguments: args });
+    return { isError: r.isError === true, body: JSON.parse(r.content?.[0]?.text ?? "null") };
+  };
+  return { call, close: async () => { await client.close(); await server.close(); } };
+}
+
+const HEALTH = {
+  bridge: "ok",
+  app_server: { ok: false, rtt_ms: null, last_error: "connection refused" },
+  tui: { ok: false, reason: "sleep-placeholder" },
+  model_auth: "revoked",
+};
+
+beforeEach(() => { cleanup(); seed(); });
+afterAll(cleanup);
+
+describe("#448 report_status.health passthrough", () => {
+  test("node's own token: stored and readable as reported", async () => {
+    const a = await connectAs(NODE_A, T_A);
+    try {
+      const r = await a.call({ resume_id: `sdk-${NODE_A_ID}`, alias: NODE_A, status: "idle", node_id: NODE_A_ID, health: HEALTH });
+      expect(r.isError).toBe(false);
+      expect(r.body.ok).not.toBe(false);
+    } finally { await a.close(); }
+    expect(readNodeHealth(NET, NODE_A)?.health).toEqual(HEALTH as any);
+  });
+
+  test("malformed health never rejects the report (node stays online); it reads as not reported", async () => {
+    const a = await connectAs(NODE_A, T_A);
+    try {
+      for (const bad of [42, "ok", { app_server: { ok: "yes" }, model_auth: "weird", tui: [] }, { app_server: { ok: true, rtt_ms: NaN, last_error: 5 } }]) {
+        const r = await a.call({ resume_id: `sdk-${NODE_A_ID}`, alias: NODE_A, status: "idle", node_id: NODE_A_ID, health: bad });
+        expect(r.isError).toBe(false);
+      }
+    } finally { await a.close(); }
+    expect(db.get<{ s: string }>("SELECT status AS s FROM sessions WHERE alias = ?1 AND network_id = ?2", NODE_A, NET)?.s).toBe("idle");
+    // 最后一份里 app_server.ok 合法、其余两格降级成 null —— 留下的是能证实的部分,不是编出来的。
+    expect(readNodeHealth(NET, NODE_A)?.health.app_server).toEqual({ ok: true, rtt_ms: null, last_error: null });
+  });
+
+  test("another node's token cannot speak for this node", async () => {
+    const b = await connectAs(NODE_B, T_B);
+    try {
+      await b.call({ resume_id: `sdk-${NODE_A_ID}`, alias: NODE_A, status: "idle", node_id: NODE_A_ID, health: HEALTH });
+    } finally { await b.close(); }
+    expect(readNodeHealth(NET, NODE_A)).toBeNull();
+  });
+
+  test("stale health reads as null, never as healthy", () => {
+    const t0 = 1_000_000;
+    recordNodeHealth(NET, NODE_A, { bridge: "ok", model_auth: "ok" }, t0);
+    expect(readNodeHealth(NET, NODE_A, t0 + 1_000)?.observed_ms_ago).toBe(1_000);
+    expect(readNodeHealth(NET, NODE_A, t0 + NODE_HEALTH_TTL_MS + 1)).toBeNull();
+  });
+
+  test("a report without health leaves the last one (heartbeats from older code paths do not erase it)", async () => {
+    const a = await connectAs(NODE_A, T_A);
+    try {
+      await a.call({ resume_id: `sdk-${NODE_A_ID}`, alias: NODE_A, status: "idle", node_id: NODE_A_ID, health: HEALTH });
+      await a.call({ resume_id: `sdk-${NODE_A_ID}`, alias: NODE_A, status: "working", node_id: NODE_A_ID });
+    } finally { await a.close(); }
+    expect(readNodeHealth(NET, NODE_A)?.health.model_auth).toBe("revoked");
+  });
+});

package/src/node-health-store.ts

@@ -0,0 +1,101 @@
+// #448 —— agent-node 上报的分层健康(`report_status.health`)的 Hub 侧透传。
+//
+// 「在线」只说明桥进程还活着:现场 ~40 台 codex 共存节点里,有的在 Hub 上 idle / in_flight=0,
+// 实际 app-server 端口没人监听、TUI pane 变成 `sleep infinity`、或登录态 "refresh token revoked"。
+// 节点把这几层分开报上来;这里只负责**收下、按节点存最新一份、在 /api/status 和 status_update 里带出去**。
+//
+// 刻意只放内存,不加列:它是**几秒钟一变的观测值**,不是需要跨重启保留的状态 —— Hub 重启后下一次
+// 心跳(≤3 分钟,翻转时立即)就会补上;持久化一份旧观测值反而会在节点死掉后继续说「app_server ok」。
+// 同理,超过 TTL 的那份按「没报」处理(null),不按「健康」处理。
+
+import { z } from "zod/v4";
+
+export const NODE_HEALTH_TTL_MS = 10 * 60_000;
+
+// 🔴 `.catch(undefined)`:这一格是纯诊断信息,**任何形状都不许拒掉整份 report_status**
+//    (zod 对象里一个已知字段验证失败 = 整份被拒 = 节点在 Hub 上失联,#1225 的形状)。
+//    合法的透传,其余一律当没报。子对象不 strict:新节点多报的键丢掉即可。
+export const nodeHealthSchema = z.object({
+  bridge: z.literal("ok").optional(),
+  app_server: z.object({
+    ok: z.boolean(),
+    rtt_ms: z.number().nullable().catch(null),
+    last_error: z.string().max(200).nullable().catch(null),
+  }).optional().catch(undefined),
+  tui: z.object({
+    ok: z.boolean(),
+    reason: z.string().max(40),
+  }).optional().catch(undefined),
+  model_auth: z.enum(["ok", "revoked", "expired", "unknown"]).optional().catch(undefined),
+  // #594 —— codex 节点的登录指纹 + 本机另有几台节点持有同一份登录(见下方 codexLoginSchema)。
+  // 🔴 线上 schema 故意是 unknown:report_status 的 JSON schema 每个节点每次 tools/list 都要下发,
+  //    #476 给它定了字节上限(tool-audience-http.test.ts)。严格校验在 recordNodeHealth 里做,效果相同:
+  //    不合规 → 这一格当没报,整份 report 照收。
+  codex_login: z.unknown().optional(),
+}).optional().catch(undefined);
+
+// #594 —— health.codex_login 的严格形状。纯展示/提示,**不是** degraded 层:共用登录今天还能跑,
+// 只是迟早互相顶掉(#1918);派发不因它拒绝。fingerprint 必须是 8 位小写 hex(sha256(refresh_token)
+// 截断,不是凭据)—— 节点误把 token 塞进来也存不下;计数是小的非负整数;codex_home 是节点自己的路径
+// (给客户端拼 `CODEX_HOME=<codex_home> codex login --device-auth`),拒控制字符。
+// 可选格各自降级;必需格(fingerprint / shared_with)不合规 → 整格丢弃。
+export const codexLoginSchema = z.object({
+  fingerprint: z.string().regex(/^[0-9a-f]{8}$/),
+  shared_with: z.number().int().min(0).max(100_000),
+  shared_home_with: z.number().int().min(0).max(100_000).optional().catch(undefined),
+  codex_home: z.string().max(1024).regex(/^[^\u0000-\u001f\u007f]*$/).optional().catch(undefined),
+});
+export type CodexLoginHealth = z.infer<typeof codexLoginSchema>;
+
+export type NodeHealth = Omit<NonNullable<z.infer<typeof nodeHealthSchema>>, "codex_login"> & { codex_login?: CodexLoginHealth };
+
+/** 线上收到的 health → 存的那份:codex_login 按严格形状过一遍,不合规就去掉这一格。 */
+export function normalizeNodeHealth(raw: NonNullable<z.infer<typeof nodeHealthSchema>>): NodeHealth {
+  const { codex_login: rawLogin, ...rest } = raw;
+  if (rawLogin === undefined) return rest;
+  const parsed = codexLoginSchema.safeParse(rawLogin);
+  return parsed.success ? { ...rest, codex_login: parsed.data } : rest;
+}
+
+const store = new Map<string, { health: NodeHealth; at: number }>();
+const keyOf = (networkId: string | null | undefined, alias: string) => `${networkId ?? "default"}\0${alias}`;
+
+// #431 —— /api/status 的记忆化(status-read-cache.ts)按这两样判断正文还对不对:任何一次上报 → 版本 +1;
+// 任何一份报告过 TTL 的那一刻(degraded / health 字段会消失)→ 由 nodeHealthNextExpiryAt 给出的时刻失效。
+let version = 0;
+export function nodeHealthVersion(): number { return version; }
+/** 最早一份还没过期的报告的过期时刻(ms);一份都没有 → Infinity。 */
+export function nodeHealthNextExpiryAt(now = Date.now()): number {
+  let next = Infinity;
+  for (const v of store.values()) {
+    const at = v.at + NODE_HEALTH_TTL_MS;
+    if (at >= now && at < next) next = at;
+  }
+  return next;
+}
+
+export function recordNodeHealth(
+  networkId: string | null | undefined, alias: string,
+  health: NodeHealth | NonNullable<z.infer<typeof nodeHealthSchema>>, now = Date.now(),
+): void {
+  version++;
+  store.set(keyOf(networkId, alias), { health: normalizeNodeHealth(health), at: now });
+  // 有界:按 TTL 顺手清掉过期的,别让改过名/删掉的节点在内存里攒着。
+  if (store.size > 256) {
+    for (const [k, v] of store) if (now - v.at > NODE_HEALTH_TTL_MS) store.delete(k);
+  }
+}
+
+/** 最新一份;没有或已过 TTL → null(读的人必须把它当「不知道」,而不是「健康」)。 */
+export function readNodeHealth(
+  networkId: string | null | undefined, alias: string, now = Date.now(),
+): { health: NodeHealth; observed_ms_ago: number } | null {
+  const hit = store.get(keyOf(networkId, alias));
+  if (!hit) return null;
+  const age = now - hit.at;
+  if (age > NODE_HEALTH_TTL_MS) { store.delete(keyOf(networkId, alias)); return null; }
+  return { health: hit.health, observed_ms_ago: Math.max(0, age) };
+}
+
+/** 测试用。 */
+export function clearNodeHealthStore(): void { store.clear(); version++; }