@sleep2agi/commhub-server 0.9.0-preview.85 → 0.9.0-preview.86

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sleep2agi/commhub-server",
-  "version": "0.9.0-preview.85",
+  "version": "0.9.0-preview.86",
   "description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
   "type": "module",
   "main": "src/index.ts",

package/src/node-health-dispatch-http.test.ts

@@ -0,0 +1,279 @@
+// #460(#448 子项)—— 降级节点拒收新任务。
+//
+// 钉住:
+//   1. 新鲜健康报告说某层坏了(app_server.ok=false / tui.ok=false / model_auth revoked|expired)
+//      → REST /api/task 409、MCP send_task / retry_task / reassign_task 返回 node_degraded,带层、原因、修法;
+//      定时任务的这一次 run 记 failed + error_code=node_degraded,不建任务行。
+//   2. 不知道 = 放行:没有报告(老 agent-node)、报告过期(> TTL)、各层都 ok、某层没报 —— 与改动前逐字相同的成功路径。
+//   3. 逃生口 force=true 只对用户令牌生效;节点令牌带 force 照样被拒(不让 agent 绕过对另一个 agent 的健康判断)。
+//   4. 老客户端请求形状(不带 force)在健康 / 未知节点上完全照旧 —— 兼容回放。
+//
+// 跑法:cd server && COMMHUB_DB=/tmp/x.db bun test src/node-health-dispatch-http.test.ts
+//       PG:COMMHUB_TEST_PG_URL=… COMMHUB_PG_EXPERIMENTAL=1(tests/test2123-hub-postgres-ladder 里注册)
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { mkdtempSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { createNetworkTokenForNode, register } from "./auth.js";
+import { db } from "./db.js";
+import { registerTools } from "./tools.js";
+import { clearNodeHealthStore, NODE_HEALTH_TTL_MS, recordNodeHealth } from "./node-health-store.js";
+import { degradedLayers } from "./node-health-guard.js";
+import { runDueScheduledTasks } from "./scheduled-tasks.js";
+
+const dir = mkdtempSync(join(tmpdir(), "anet-health-dispatch-"));
+const activeDbPath = process.env.COMMHUB_DB ?? (process.env.COMMHUB_TEST_PG_URL ? "postgres" : undefined);
+if (!activeDbPath) throw new Error("node-health-dispatch requires COMMHUB_DB (or COMMHUB_TEST_PG_URL) before module import");
+
+let server: any;
+let base = "";
+let ownerToken = "", ownerId = "", networkId = "";
+let peerNodeToken = "", peerNodeTokenId = "";
+const stamp = Date.now();
+const TARGET = `hd-target-${stamp}`;
+const TARGET_NODE = `n_hd_target_${stamp}`;
+const PEER = `hd-peer-${stamp}`;
+const PEER_NODE = `n_hd_peer_${stamp}`;
+let seq = 0;
+const text = (what: string) => `${what} #${++seq}`; // distinct content: the 5-minute dedup must not interfere
+
+async function api(token: string, path: string, init?: RequestInit) {
+  const res = await fetch(`${base}${path}`, {
+    ...init,
+    headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", ...(init?.headers || {}) },
+  });
+  return { status: res.status, body: await res.json() as any };
+}
+const postTask = (token: string, extra: Record<string, unknown> = {}) =>
+  api(token, "/api/task", { method: "POST", body: JSON.stringify({ alias: TARGET, task: text("rest"), network_id: networkId, ...extra }) });
+
+async function mcpAs(identity: { net: string | null; user: string; alias: string; isNetworkToken: boolean; tokenId: string }) {
+  const s = new McpServer({ name: "health-dispatch", version: "1" });
+  registerTools(s, undefined, identity.net, identity.user, identity.alias, identity.isNetworkToken, identity.tokenId);
+  const client = new Client({ name: "health-dispatch-client", version: "1" });
+  const [ct, st] = InMemoryTransport.createLinkedPair();
+  await s.connect(st);
+  await client.connect(ct);
+  const call = async (name: string, args: Record<string, unknown>) => {
+    const r: any = await client.callTool({ name, arguments: args });
+    const raw = r.content?.[0]?.text ?? "null";
+    try { return JSON.parse(raw); } catch { return { raw }; }
+  };
+  return { call, close: async () => { await client.close(); await s.close(); } };
+}
+
+const DOWN_APP = { bridge: "ok" as const, app_server: { ok: false, rtt_ms: null, last_error: "connect ECONNREFUSED 127.0.0.1:4500" }, model_auth: "ok" as const };
+const HEALTHY = { bridge: "ok" as const, app_server: { ok: true, rtt_ms: 3, last_error: null }, tui: { ok: true, reason: "running" }, model_auth: "ok" as const };
+const setHealth = (h: any, at?: number) => recordNodeHealth(networkId, TARGET, h, at);
+const taskCount = () => db.get<{ n: number }>("SELECT COUNT(*) AS n FROM tasks WHERE to_name = ?1 AND network_id = ?2", TARGET, networkId)!.n;
+
+beforeAll(async () => {
+  process.env.COMMHUB_UPLOADS_DIR = join(dir, "uploads");
+  process.env.HOST = "127.0.0.1";
+  const owner = register(`hd_owner_${stamp}`, "HealthDispatchOwner123!", undefined, "seed");
+  ownerToken = owner.token!; ownerId = owner.user!.user_id; networkId = owner.network_id!;
+  for (const [nodeId, alias] of [[TARGET_NODE, TARGET], [PEER_NODE, PEER]]) {
+    db.run(`INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, owner_user_id) VALUES (?1, ?2, ?2, 'codex-app-server', ?3, ?4)`, [nodeId, alias, networkId, ownerId]);
+    db.run(`INSERT INTO sessions (resume_id, alias, status, node_id, network_id) VALUES (?1, ?2, 'idle', ?3, ?4)`, [`r_${nodeId}`, alias, nodeId, networkId]);
+  }
+  const minted = createNetworkTokenForNode(ownerId, networkId, PEER, PEER_NODE);
+  peerNodeToken = minted.token!; peerNodeTokenId = minted.token_id!;
+  const mod: any = await import("./server.js");
+  server = mod.bootServer({ port: 0, hostname: "127.0.0.1" });
+  base = `http://127.0.0.1:${server.port}`;
+}, 30_000);
+afterAll(() => { server?.stop?.(true); clearNodeHealthStore(); });
+beforeEach(() => clearNodeHealthStore());
+
+describe("#460 which layers count as degraded", () => {
+  test("each failing layer is named; ok / missing layers are not", () => {
+    expect(degradedLayers(null)).toEqual([]);
+    expect(degradedLayers(HEALTHY)).toEqual([]);
+    expect(degradedLayers({ bridge: "ok", model_auth: "unknown" })).toEqual([]);
+    expect(degradedLayers(DOWN_APP).map(l => [l.layer, l.label])).toEqual([["app_server", "App Server 断开"]]);
+    expect(degradedLayers({ tui: { ok: false, reason: "sleep-placeholder" } }).map(l => [l.layer, l.reason, l.label])).toEqual([["tui", "sleep-placeholder", "TUI 被占位(sleep)"]]);
+    expect(degradedLayers({ model_auth: "revoked" }).map(l => l.label)).toEqual(["需要重新登录"]);
+    expect(degradedLayers({ model_auth: "expired" }).map(l => l.label)).toEqual(["登录已过期"]);
+    // 修法只指向本节点重新登录,不教人拷别人的凭据
+    expect(degradedLayers({ model_auth: "revoked" })[0].hint).toContain("不要拷别的节点的 auth.json");
+  });
+});
+
+describe("#460 REST /api/task", () => {
+  test("unknown health (old agent-node) → dispatched exactly as before", async () => {
+    const before = taskCount();
+    const r = await postTask(ownerToken);
+    expect(r.status).toBe(200);
+    expect(r.body.ok).toBe(true);
+    expect(taskCount()).toBe(before + 1);
+  });
+
+  test("healthy report → dispatched", async () => {
+    setHealth(HEALTHY);
+    expect((await postTask(ownerToken)).status).toBe(200);
+  });
+
+  test("App Server down → 409 node_degraded with layer, reason and hint; no task row", async () => {
+    setHealth(DOWN_APP);
+    const before = taskCount();
+    const r = await postTask(ownerToken);
+    expect(r.status).toBe(409);
+    expect(r.body.ok).toBe(false);
+    expect(r.body.error).toBe("node_degraded");
+    expect(r.body.alias).toBe(TARGET);
+    expect(r.body.layers.map((l: any) => l.layer)).toEqual(["app_server"]);
+    expect(r.body.layers[0].reason).toContain("ECONNREFUSED");
+    expect(r.body.message).toContain("App Server 断开");
+    expect(r.body.hint).toContain("force=true");
+    expect(r.body.force_allowed).toBe(true);
+    expect(typeof r.body.health_observed_ms_ago).toBe("number");
+    expect(taskCount()).toBe(before);
+  });
+
+  test("TUI gone and login revoked are refused too, all layers listed", async () => {
+    setHealth({ bridge: "ok", app_server: { ok: true, rtt_ms: 2, last_error: null }, tui: { ok: false, reason: "session-missing" }, model_auth: "revoked" });
+    const r = await postTask(ownerToken);
+    expect(r.status).toBe(409);
+    expect(r.body.layers.map((l: any) => l.layer)).toEqual(["tui", "model_auth"]);
+  });
+
+  test("expired login alone is refused", async () => {
+    setHealth({ bridge: "ok", model_auth: "expired" });
+    expect((await postTask(ownerToken)).body.error).toBe("node_degraded");
+  });
+
+  test("stale report (older than the TTL) = unknown → dispatched", async () => {
+    setHealth(DOWN_APP, Date.now() - NODE_HEALTH_TTL_MS - 1_000);
+    expect((await postTask(ownerToken)).status).toBe(200);
+  });
+
+  test("owner escape hatch: force=true with a user token dispatches", async () => {
+    setHealth(DOWN_APP);
+    const before = taskCount();
+    const r = await postTask(ownerToken, { force: true });
+    expect(r.status).toBe(200);
+    expect(taskCount()).toBe(before + 1);
+  });
+
+  test("a node token cannot force past another node's health", async () => {
+    setHealth(DOWN_APP);
+    const r = await api(peerNodeToken, "/api/task", { method: "POST", body: JSON.stringify({ alias: TARGET, task: text("node-force"), force: true }) });
+    expect(r.status).toBe(409);
+    expect(r.body.error).toBe("node_degraded");
+    expect(r.body.force_allowed).toBe(false);
+    expect(r.body.hint).not.toContain("force=true");
+  });
+
+  test("old-client request shapes are untouched on a healthy node (compat replay)", async () => {
+    setHealth(HEALTHY);
+    // the app's send (meta + client_request_id), a plain anet send, and a parent-linked dispatch
+    const shapes = [
+      { alias: TARGET, task: text("app"), network_id: networkId, priority: "normal", meta: { client_request_id: `cr_${stamp}_${seq}` } },
+      { alias: TARGET, task: text("anet"), network_id: networkId },
+      { alias: TARGET, task: text("ttl"), network_id: networkId, priority: "high", ttl_seconds: 600 },
+    ];
+    for (const shape of shapes) {
+      const r = await api(ownerToken, "/api/task", { method: "POST", body: JSON.stringify(shape) });
+      expect(r.status).toBe(200);
+      expect(r.body.ok).toBe(true);
+    }
+  });
+});
+
+describe("#460 MCP send_task / retry_task / reassign_task", () => {
+  const asOwner = () => ({ net: null, user: ownerId, alias: `hd_owner_${stamp}`, isNetworkToken: false, tokenId: "tok_hd_owner_login" });
+  const asPeerNode = () => ({ net: networkId, user: ownerId, alias: PEER, isNetworkToken: true, tokenId: peerNodeTokenId });
+
+  test("send_task from an agent to a degraded node → node_degraded (no queueing); force ignored for node tokens", async () => {
+    setHealth(DOWN_APP);
+    const peer = await mcpAs(asPeerNode());
+    try {
+      const before = taskCount();
+      const r = await peer.call("send_task", { alias: TARGET, task: text("mcp"), force: true });
+      expect(r.ok).toBe(false);
+      expect(r.error).toBe("node_degraded");
+      expect(r.force_allowed).toBe(false);
+      expect(taskCount()).toBe(before);
+    } finally { await peer.close(); }
+  });
+
+  test("send_task: unknown health → ok; owner force → ok", async () => {
+    const owner = await mcpAs(asOwner());
+    try {
+      expect((await owner.call("send_task", { alias: TARGET, task: text("mcp-unknown"), network_id: networkId })).ok).toBe(true);
+      setHealth({ bridge: "ok", model_auth: "revoked" });
+      expect((await owner.call("send_task", { alias: TARGET, task: text("mcp-refused"), network_id: networkId })).error).toBe("node_degraded");
+      expect((await owner.call("send_task", { alias: TARGET, task: text("mcp-forced"), network_id: networkId, force: true })).ok).toBe(true);
+    } finally { await owner.close(); }
+  });
+
+  test("retry_task and reassign_task onto a degraded node are refused", async () => {
+    const owner = await mcpAs(asOwner());
+    try {
+      const sent = await owner.call("send_task", { alias: PEER, task: text("to-peer"), network_id: networkId });
+      expect(sent.ok).toBe(true);
+      setHealth(DOWN_APP);
+      const re = await owner.call("reassign_task", { task_id: sent.task_id ?? sent.message_id, new_alias: TARGET, network_id: networkId });
+      expect(re.error).toBe("node_degraded");
+      const failed = await owner.call("send_task", { alias: TARGET, task: text("to-fail"), network_id: networkId, force: true });
+      const failedId = failed.task_id ?? failed.message_id;
+      db.run("UPDATE tasks SET status = 'failed' WHERE task_id = ?1", [failedId]);
+      const retry = await owner.call("retry_task", { task_id: failedId, network_id: networkId });
+      expect(retry.error).toBe("node_degraded");
+      clearNodeHealthStore();
+      expect((await owner.call("retry_task", { task_id: failedId, network_id: networkId })).ok).toBe(true);
+    } finally { await owner.close(); }
+  });
+});
+
+describe("#460 scheduled tasks", () => {
+  test("a due run on a degraded node fails with node_degraded and creates no task", async () => {
+    const created = await api(ownerToken, "/api/scheduled-tasks", {
+      method: "POST",
+      body: JSON.stringify({ network_id: networkId, name: `hd-schedule-${stamp}`, target_node_id: TARGET_NODE, task: text("scheduled"), timezone: "UTC", schedule: { type: "interval", every_seconds: 60 } }),
+    });
+    expect(created.status).toBe(201);
+    const scheduleId = created.body.schedule.schedule_id;
+    setHealth({ bridge: "ok", tui: { ok: false, reason: "pane-dead" }, model_auth: "ok" });
+    const before = taskCount();
+    db.run("UPDATE scheduled_tasks SET next_run_at = ?1 WHERE schedule_id = ?2", [new Date(Date.now() - 60_000).toISOString(), scheduleId]);
+    runDueScheduledTasks();
+    const run = db.get<{ status: string; error_code: string | null; error_message: string | null; task_id: string | null }>(
+      "SELECT status, error_code, error_message, task_id FROM scheduled_task_runs WHERE schedule_id = ?1 ORDER BY created_at DESC LIMIT 1", scheduleId);
+    expect(run?.status).toBe("failed");
+    expect(run?.error_code).toBe("node_degraded");
+    expect(run?.error_message).toContain("TUI 已退出");
+    expect(run?.task_id ?? null).toBeNull();
+    expect(taskCount()).toBe(before);
+  });
+});
+
+describe("#460 /api/status carries `degraded` only for degraded nodes", () => {
+  const row = async (light: boolean) => {
+    const r = await api(ownerToken, `/api/status?network_id=${encodeURIComponent(networkId)}${light ? "&light=1" : ""}`);
+    expect(r.status).toBe(200);
+    return r.body.sessions.find((x: any) => x.alias === TARGET);
+  };
+  test("unknown / healthy → no `degraded` key at all (light response bytes unchanged)", async () => {
+    expect("degraded" in (await row(true))).toBe(false);
+    setHealth(HEALTHY);
+    expect("degraded" in (await row(true))).toBe(false);
+    expect("degraded" in (await row(false))).toBe(false);
+  });
+  test("degraded → light and full rows list the failing layers with the app's labels", async () => {
+    setHealth({ bridge: "ok", app_server: { ok: false, rtt_ms: null, last_error: "refused" }, model_auth: "revoked" });
+    const want = [
+      { layer: "app_server", label: "App Server 断开", reason: "refused" },
+      { layer: "model_auth", label: "需要重新登录", reason: "revoked" },
+    ];
+    expect((await row(true)).degraded).toEqual(want);
+    expect((await row(false)).degraded).toEqual(want);
+  });
+  test("stale → no `degraded`", async () => {
+    setHealth(DOWN_APP, Date.now() - NODE_HEALTH_TTL_MS - 1_000);
+    expect("degraded" in (await row(true))).toBe(false);
+  });
+});

package/src/node-health-guard.ts

@@ -0,0 +1,105 @@
+// #460(#448 的子项)—— 降级节点拒收新任务。
+//
+// 现场:节点在 Hub 上 idle / in_flight=0,实际 app-server 端口没人听、TUI 变成 `sleep infinity`、
+// 或登录态 revoked —— 派过去的任务**静默排队**,谁都不知道它永远不会被执行。agent-node .94 起
+// 把这几层报上来(report_status.health,node-health-store.ts 只存内存、10 分钟 TTL);这里在**派发**
+// 那一刻读它:新鲜的健康报告说某一层坏了,就直接拒,带上是哪一层、怎么修。
+//
+// 🔴 只拒「确知坏了」的:没有报告(老 agent-node / claude-code 节点)、报告已过期(>TTL)、
+//    或某层没报(比如非共存节点没有 tui)= 不知道 = 放行。宁可像以前一样排队,
+//    也不能因为缺数据把一台正常的老节点挡在外面。
+// 🔴 只管**新任务派发**(send_task / REST /api/task / 定时任务 / retry / reassign / broadcast)。
+//    回复、send_message、ack 不经过这里 —— 降级节点发出的回复照常送达,发给它的普通消息也照常入库。
+// 逃生口:用户(不是节点令牌)可以带 force=true 强制派发 —— 比如明知 TUI 不在、但 bridge 能干活。
+//    节点令牌不能 force:不让 agent 绕过对另一个 agent 的健康判断。
+
+import { readNodeHealth, type NodeHealth } from "./node-health-store.js";
+
+export type DegradedLayer = {
+  layer: "app_server" | "tui" | "model_auth";
+  /** 机器可读的原因:app_server 的错误摘要 / tui 的 reason / model_auth 的状态。 */
+  reason: string;
+  /** 给人看的一句(中文,与 app 的「降级」徽标同一套措辞)。 */
+  label: string;
+  hint: string;
+};
+
+const TUI_LABEL: Record<string, string> = {
+  "session-missing": "TUI 会话不在",
+  "pane-dead": "TUI 已退出",
+  "sleep-placeholder": "TUI 被占位(sleep)",
+  "tmux-unavailable": "tmux 不可用",
+};
+
+/** 一份健康报告里确知坏掉的层;没报的层不算。 */
+export function degradedLayers(health: NodeHealth | null | undefined): DegradedLayer[] {
+  if (!health) return [];
+  const out: DegradedLayer[] = [];
+  if (health.app_server && health.app_server.ok === false) {
+    out.push({
+      layer: "app_server",
+      reason: (health.app_server.last_error || "unreachable").slice(0, 200),
+      label: "App Server 断开",
+      hint: "节点的 codex app-server 端口没有响应:在节点所在机器上重启该节点(anet node restart <别名>)",
+    });
+  }
+  if (health.tui && health.tui.ok === false) {
+    out.push({
+      layer: "tui",
+      reason: health.tui.reason,
+      label: TUI_LABEL[health.tui.reason] ?? "TUI 不可用",
+      hint: "共存 TUI 不在运行:在节点所在机器上重新拉起该节点的 TUI(anet node restart <别名>)",
+    });
+  }
+  if (health.model_auth === "revoked" || health.model_auth === "expired") {
+    out.push({
+      layer: "model_auth",
+      reason: health.model_auth,
+      label: health.model_auth === "revoked" ? "需要重新登录" : "登录已过期",
+      hint: "该节点自己的 CODEX_HOME 需要重新登录(在那台机器上:CODEX_HOME=<节点目录>/codex-home codex login);不要拷别的节点的 auth.json",
+    });
+  }
+  return out;
+}
+
+export type HealthGuardResult =
+  | { ok: true; forced?: true }
+  | {
+    ok: false;
+    error: "node_degraded";
+    alias: string;
+    layers: DegradedLayer[];
+    message: string;
+    hint: string;
+    health_observed_ms_ago: number;
+    /** 能不能用 force=true 绕过(只有用户令牌能)。 */
+    force_allowed: boolean;
+  };
+
+/**
+ * 派发前的健康门。networkId 用与 report_status 写入时同一个键(sessionNetId = 网络 id ?? "default")。
+ * force 只在 forceAllowed(用户令牌)时生效。
+ */
+export function assertNodeHealthy(
+  alias: string,
+  networkId: string | null | undefined,
+  opts: { force?: boolean; forceAllowed: boolean; now?: number },
+): HealthGuardResult {
+  if (!alias) return { ok: true };
+  const hit = readNodeHealth(networkId ?? "default", alias, opts.now);
+  if (!hit) return { ok: true };
+  const layers = degradedLayers(hit.health);
+  if (!layers.length) return { ok: true };
+  if (opts.force && opts.forceAllowed) return { ok: true, forced: true };
+  const labels = layers.map(l => l.label).join("、");
+  return {
+    ok: false,
+    error: "node_degraded",
+    alias,
+    layers,
+    message: `节点 ${alias} 处于降级状态(${labels}),任务不会被执行,已拒绝派发。`,
+    hint: layers[0].hint + (opts.forceAllowed ? "。确需强制派发可带 force=true。" : ""),
+    health_observed_ms_ago: hit.observed_ms_ago,
+    force_allowed: opts.forceAllowed,
+  };
+}

package/src/scheduled-tasks.ts

@@ -1,5 +1,6 @@
 import { db, logTaskEvent } from "./db.js";
 import { assertNodeActive } from "./lifecycle-guard.js";
+import { assertNodeHealthy } from "./node-health-guard.js";
 import { addNetworkScope, canRestWriteNetwork, singleNetworkId, type RestNetworkScope } from "./network-scope.js";
 import { canMessageAgent } from "./agent-access.js";
 import { hasSubscribers, pushEvent, pushNetworkObserverEvent } from "./push.js";
@@ -290,6 +291,19 @@ export function dispatchScheduledOccurrence(row: ScheduledRow, scheduledFor: str
       return;
     }
 
+    // #460 — the target reports a degraded layer (App Server down / TUI gone / login revoked): the run fails
+    // with that reason instead of queueing a task nobody will execute. Unknown / stale health runs as before.
+    const health = assertNodeHealthy(node.alias, row.network_id, { forceAllowed: false });
+    if (!health.ok) {
+      finalStatus = "failed";
+      db.run(
+        "UPDATE scheduled_task_runs SET status = 'failed', error_code = 'node_degraded', error_message = ?1, completed_at = datetime('now') WHERE run_id = ?2",
+        [health.message.slice(0, 500), runId],
+      );
+      if (advanceSchedule) advance(row, scheduledFor, advanceAfter);
+      return;
+    }
+
     const taskId = crypto.randomUUID();
     // claude-code 裸节点的 node-server 上报 report_status 不带 node_id,它的 sessions 行
     // node_id 永远是 NULL —— 只按 node_id 找会话,这类节点每一次都判成 queued、门铃不响,

package/src/server.ts

@@ -2,6 +2,7 @@ import { buildServeErrorResponse } from "./serve-error.js";
 import { maybeGzipResponse, trimLightTask } from "./http-gzip";
 import { statusAliasResolverRead } from "./status-alias-resolver.js";
 import { readNodeHealth } from "./node-health-store.js";
+import { assertNodeHealthy, degradedLayers } from "./node-health-guard.js";
 import { auditDepartment, createDepartment, deleteDepartment, listDepartments, setMemberDepartment, updateDepartment } from "./departments.js";
 import { redactMessageRow } from "./redact-tokens.js";
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
@@ -750,8 +751,18 @@ const TaskSchema = z.object({
   // stays unified regardless of transport (REST or MCP).
   attachments: z.any().optional(),
   meta: z.any().optional(),
+  // #460 — dispatch even when the target reports a degraded health layer. Honoured for user
+  // tokens only (a node token cannot override another node's health). Old clients never send it.
+  force: z.boolean().optional(),
 });
 
+/** #460 — `{ degraded: [{ layer, label, reason }] }` when the node's fresh health says a layer is down, else `{}`. */
+function degradedField(networkId: string | null | undefined, alias: string): { degraded?: Array<{ layer: string; label: string; reason: string }> } {
+  const h = readNodeHealth(networkId ?? "default", alias);
+  const layers = degradedLayers(h?.health);
+  return layers.length ? { degraded: layers.map(l => ({ layer: l.layer, label: l.label, reason: l.reason })) } : {};
+}
+
 const BroadcastSchema = z.object({
   message: z.string().min(1).max(10000),
   filter_server: z.string().max(200).optional(),
@@ -2317,6 +2328,10 @@ return Bun.serve({
             // Only when filtering by node_id (or for the old alias resolver), so the unfiltered light response
             // keeps its exact bytes.
             ...(withNodeId ? { node_id: s.node_id ?? null } : {}),
+            // #460 — the app's list reads the light projection; a node whose fresh health says a layer is down
+            // carries the failing layers here (the same ones dispatch refuses with node_degraded). Absent for every
+            // healthy / unknown node, so the response is byte-identical unless something is actually degraded.
+            ...degradedField(s.network_id, s.alias),
           };
         }
         const externalSchedules = (() => {
@@ -2359,6 +2374,7 @@ return Bun.serve({
             const h = readNodeHealth(s.network_id, s.alias);
             return { health: h?.health ?? null, health_observed_ms_ago: h?.observed_ms_ago ?? null };
           })(),
+          ...degradedField(s.network_id, s.alias),
         };
       });
       const summary = sessions.reduce((acc: any, session: any) => {
@@ -3211,6 +3227,12 @@ return Bun.serve({
           return withCors(req, Response.json(lc, { status: 409 }));
         }
       }
+      // #460 — refuse dispatch to a node whose fresh health says a layer is down (409, same as the
+      // lifecycle guard above). Unknown / stale health = allowed: old nodes keep working.
+      {
+        const hg = assertNodeHealthy(targetAlias, taskNetId ?? null, { force: body.force === true, forceAllowed: !restAuth?.networkId });
+        if (!hg.ok) return withCors(req, Response.json(hg, { status: 409 }));
+      }
       // Mirror send_task MCP: write inbox + tasks rows in a single
       // transaction so the dispatch is visible to dashboard's Tasks page
       // and the parent_task_id lineage chain. Previously this endpoint

package/src/tools.ts

@@ -2,6 +2,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { parseDbTimestampMs } from "./db-timestamp.js";
 import { z } from "zod/v4";
 import { nodeHealthSchema, recordNodeHealth } from "./node-health-store.js";
+import { assertNodeHealthy } from "./node-health-guard.js";
 import { parseAliasFilter } from "./alias-filter.js";
 import { createHash } from "node:crypto";
 import { db, uuidv4, logTaskEvent, chainReplyToParent, hashToken, generateId, generateNetworkToken, syncScheduledRunForTask } from "./db.js";
@@ -1632,8 +1633,9 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
       network_id: z.string().max(200).optional().describe("Network scope"),
       parent_task_id: z.string().max(200).optional().describe("Parent task this dispatch is on behalf of. When the child task replies the hub will auto-chain the answer to the parent task's originator, so the user sees the final result even if the intermediate session ends."),
       meta: z.any().optional().describe("Optional structured task metadata, e.g. { attachments: [{ type, path, url, mime, name, size }] }."),
+      force: z.boolean().optional().describe("#460 — dispatch even when the target reports a degraded health layer (user tokens only; ignored for node tokens)"),
     },
-    async ({ alias, task, priority, context, from_session: _fromIn, ttl_seconds, network_id: netId, parent_task_id: parentIn, meta }) => { const fromMismatch = fromIdentityMismatchReply(_fromIn); if (fromMismatch) return fromMismatch; const from_session = defaultFrom(_fromIn);
+    async ({ alias, task, priority, context, from_session: _fromIn, ttl_seconds, network_id: netId, parent_task_id: parentIn, meta, force }) => { const fromMismatch = fromIdentityMismatchReply(_fromIn); if (fromMismatch) return fromMismatch; const from_session = defaultFrom(_fromIn);
       const effectiveNetId = getNetworkId(netId);
       const authOrigin: TaskAuthOrigin = callerTokenIsNetwork
         ? "node"
@@ -1787,6 +1789,11 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
         const lc = assertNodeActive(targetAlias, effectiveNetId ?? null);
         if (!lc.ok) return { content: [{ type: "text" as const, text: JSON.stringify(lc) }] };
       }
+      // #460 — a node whose fresh health says a layer is down will not run the task: refuse instead of queueing silently.
+      {
+        const hg = assertNodeHealthy(targetAlias, effectiveNetId ?? null, { force, forceAllowed: !callerTokenIsNetwork });
+        if (!hg.ok) return { content: [{ type: "text" as const, text: JSON.stringify(hg) }] };
+      }
 
       console.log(`[${ts()}] ${from_session} → send_task → ${targetAlias}: ${task.slice(0, 60)}${priority === "high" ? " [HIGH]" : ""}${canonical.renamed ? ` [renamed from ${alias}]` : ""}`);
       const fromNodeId = resolveNodeIdForAlias(from_session, effectiveNetId);
@@ -2346,8 +2353,9 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
       task_id: z.string().min(1).max(200).describe("Task ID to retry"),
       from_session: z.string().max(200).optional(),
       network_id: z.string().max(200).optional().describe("Network scope (auto-resolved for single-network user tokens)"),
+      force: z.boolean().optional().describe("#460 — dispatch even when the target reports a degraded health layer (user tokens only; ignored for node tokens)"),
     },
-    async ({ task_id, from_session: _fromIn, network_id: netId }) => { const fromMismatch = fromIdentityMismatchReply(_fromIn); if (fromMismatch) return fromMismatch; const from_session = defaultFrom(_fromIn);
+    async ({ task_id, from_session: _fromIn, network_id: netId, force }) => { const fromMismatch = fromIdentityMismatchReply(_fromIn); if (fromMismatch) return fromMismatch; const from_session = defaultFrom(_fromIn);
       const effectiveNetId = getNetworkId(netId);
       if (!canWrite(effectiveNetId)) return writeDeniedReply(effectiveNetId);
       console.log(`[${ts()}] ${from_session} → retry_task → ${task_id.slice(0, 8)}`);
@@ -2367,6 +2375,11 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
         const lc = assertNodeActive(task.to_name, effectiveNetId ?? task.network_id ?? null);
         if (!lc.ok) return { content: [{ type: "text" as const, text: JSON.stringify(lc) }] };
       }
+      // #460 — a node whose fresh health says a layer is down will not run the task: refuse instead of queueing silently.
+      {
+        const hg = assertNodeHealthy(task.to_name, effectiveNetId ?? task.network_id ?? null, { force, forceAllowed: !callerTokenIsNetwork });
+        if (!hg.ok) return { content: [{ type: "text" as const, text: JSON.stringify(hg) }] };
+      }
       db.transaction(() => {
         // Reset task status
         const updateParams: any[] = [task_id];
@@ -2559,8 +2572,9 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
       new_alias: z.string().min(1).max(200).describe("Target agent alias"),
       from_session: z.string().max(200).optional(),
       network_id: z.string().max(200).optional().describe("Network scope (auto-resolved for single-network user tokens)"),
+      force: z.boolean().optional().describe("#460 — dispatch even when the target reports a degraded health layer (user tokens only; ignored for node tokens)"),
     },
-    async ({ task_id, new_alias, from_session: _fromIn, network_id: netId }) => { const fromMismatch = fromIdentityMismatchReply(_fromIn); if (fromMismatch) return fromMismatch; const from_session = defaultFrom(_fromIn);
+    async ({ task_id, new_alias, from_session: _fromIn, network_id: netId, force }) => { const fromMismatch = fromIdentityMismatchReply(_fromIn); if (fromMismatch) return fromMismatch; const from_session = defaultFrom(_fromIn);
       const effectiveNetId = getNetworkId(netId);
       if (!canWrite(effectiveNetId)) return writeDeniedReply(effectiveNetId);
       console.log(`[${ts()}] ${from_session} → reassign_task → ${task_id.slice(0, 8)} → ${new_alias}`);
@@ -2582,6 +2596,11 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
         const lc = assertNodeActive(reassignedAlias, effectiveNetId ?? task.network_id ?? null);
         if (!lc.ok) return { content: [{ type: "text" as const, text: JSON.stringify(lc) }] };
       }
+      // #460 — a node whose fresh health says a layer is down will not run the task: refuse instead of queueing silently.
+      {
+        const hg = assertNodeHealthy(reassignedAlias, effectiveNetId ?? task.network_id ?? null, { force, forceAllowed: !callerTokenIsNetwork });
+        if (!hg.ok) return { content: [{ type: "text" as const, text: JSON.stringify(hg) }] };
+      }
       db.transaction(() => {
         // Ack old inbox to prevent original agent from picking it up
         const inboxParams: any[] = [task_id];