@sleep2agi/commhub-server 0.9.0-preview.1 → 0.9.0-preview.101

package/src/rules-file-transport.test.ts

@@ -0,0 +1,237 @@
+// app#225 — 节点规则文件远程读写：走真 MCP in-process transport 的端到端。
+//
+// 验的是 #225 验收里 hub 能验的那几条：
+//   1. 客户端只用 node_id 就能发起读/写；工具的 inputSchema 里**没有**任何
+//      路径/文件名字段（验收第 5 条「不能借路径参数写任意文件」在 hub 侧的形状）。
+//   2. 节点用网络 token + alias 拉到的请求里也没有路径字段；ack 后客户端轮询拿到内容。
+//   3. SEC-1：跨网络节点发不了、别的节点 ack 不了、用户 token 拉不了。
+//   4. 单飞 + 超时回收。
+//
+// 跑法：cd server && COMMHUB_DB=/tmp/rules.db bun test src/rules-file-transport.test.ts
+import { afterAll, beforeEach, describe, expect, test } from "bun:test";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { db } from "./db.js";
+import { registerTools } from "./tools.js";
+
+const NET = "net_rules_file";
+const OTHER_NET = "net_rules_file_other";
+const USER = "u_rules_file_owner";
+const NODE_ID = "node_rules_file_claude";
+const NODE_ALIAS = "rules-file-claude";
+const OTHER_NODE_ID = "node_rules_file_other";
+const OTHER_NODE_ALIAS = "rules-file-other";
+const NODE_TOKEN = "tok_rules_file_node";
+const OTHER_NODE_TOKEN = "tok_rules_file_other_node";
+
+function cleanup() {
+  for (const net of [NET, OTHER_NET]) {
+    try { db.run("DELETE FROM node_rules_requests WHERE network_id = ?1", [net]); } catch {}
+    try { db.run("DELETE FROM audit_log WHERE network_id = ?1", [net]); } catch {}
+    try { db.run("DELETE FROM nodes WHERE network_id = ?1", [net]); } catch {}
+    try { db.run("DELETE FROM api_tokens WHERE network_id = ?1", [net]); } catch {}
+    try { db.run("DELETE FROM network_members WHERE network_id = ?1", [net]); } catch {}
+    try { db.run("DELETE FROM networks WHERE network_id = ?1", [net]); } catch {}
+  }
+  try { db.run("DELETE FROM users WHERE user_id = ?1", [USER]); } catch {}
+}
+
+function seedWorld() {
+  db.run(
+    `INSERT INTO users (user_id, username, password_hash, role, created_at) VALUES (?1, ?2, 'x', 'user', datetime('now'))`,
+    [USER, USER],
+  );
+  for (const net of [NET, OTHER_NET]) {
+    db.run(`INSERT INTO networks (network_id, network_name, owner_id, created_at) VALUES (?1, ?2, ?3, datetime('now'))`, [net, net, USER]);
+    db.run(`INSERT INTO network_members (user_id, network_id, role, joined_at) VALUES (?1, ?2, 'owner', datetime('now'))`, [USER, net]);
+  }
+  db.run(
+    `INSERT INTO nodes (node_id, node_name, alias, network_id, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, datetime('now'), datetime('now'))`,
+    [NODE_ID, NODE_ALIAS, NODE_ALIAS, NET],
+  );
+  db.run(
+    `INSERT INTO nodes (node_id, node_name, alias, network_id, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, datetime('now'), datetime('now'))`,
+    [OTHER_NODE_ID, OTHER_NODE_ALIAS, OTHER_NODE_ALIAS, OTHER_NET],
+  );
+  db.run(
+    `INSERT INTO api_tokens (token_id, user_id, network_id, scope, name, token_hash, expires_at, revoked_at) VALUES (?1, ?2, ?3, 'network', ?4, ?5, NULL, NULL)`,
+    [NODE_TOKEN, USER, NET, `node:${NODE_ALIAS}`, "hash_rules_file_node"],
+  );
+  db.run(
+    `INSERT INTO api_tokens (token_id, user_id, network_id, scope, name, token_hash, expires_at, revoked_at) VALUES (?1, ?2, ?3, 'network', ?4, ?5, NULL, NULL)`,
+    [OTHER_NODE_TOKEN, USER, OTHER_NET, `node:${OTHER_NODE_ALIAS}`, "hash_rules_file_other"],
+  );
+}
+
+type Identity = { net: string; alias: string; isNetworkToken: boolean; tokenId: string };
+const asUser: Identity = { net: NET, alias: USER, isNetworkToken: false, tokenId: "tok_rules_file_user" };
+const asNode: Identity = { net: NET, alias: NODE_ALIAS, isNetworkToken: true, tokenId: NODE_TOKEN };
+const asOtherNode: Identity = { net: OTHER_NET, alias: OTHER_NODE_ALIAS, isNetworkToken: true, tokenId: OTHER_NODE_TOKEN };
+
+async function connect(id: Identity) {
+  const server = new McpServer({ name: "rules-file-test", version: "1" });
+  registerTools(server, undefined, id.net, USER, id.alias, id.isNetworkToken, id.tokenId);
+  const client = new Client({ name: "rules-file-client", version: "1" });
+  const [ct, st] = InMemoryTransport.createLinkedPair();
+  await server.connect(st);
+  await client.connect(ct);
+  const call = async (name: string, args: Record<string, unknown>) => {
+    const r: any = await client.callTool({ name, arguments: args });
+    const first = r.content?.[0];
+    if (!first || first.type !== "text") throw new Error(`no text result from ${name}`);
+    return JSON.parse(first.text);
+  };
+  const close = async () => { await client.close(); await server.close(); };
+  return { client, server, call, close };
+}
+
+beforeEach(() => { cleanup(); seedWorld(); });
+afterAll(cleanup);
+
+describe("app#225 rules file — schema has no path", () => {
+  test("read/write tool inputs are node id + content only; no path / file_name / dir field anywhere", async () => {
+    // 🔴 不走 client.listTools():main 上 report_status / send_desktop_message /
+    // update_node_config / create_node 四个存量工具的 zod 形状让 SDK 的
+    // toJSONSchema 抛 `schema._zod`,listTools 整个失败(与本 PR 无关,实验见
+    // PR 正文)。这里直接看 McpServer 登记表里的 zod shape —— 它就是 hub 校验
+    // 入参用的那份,比 JSON schema 更接近真相。
+    const u = await connect(asUser);
+    try {
+      const reg: Record<string, any> = (u.server as any)._registeredTools;
+      const shapeKeys = (name: string): string[] => {
+        expect(reg[name], name).toBeDefined();
+        const schema = reg[name].inputSchema;
+        const shape = schema?.shape ?? schema?._def?.shape?.() ?? schema ?? {};
+        return Object.keys(shape).sort();
+      };
+      // `alias` (app#225 follow-up) names a TARGET node/session, not a file —
+      // the no-path regex below still covers it.
+      expect(shapeKeys("read_node_rules_file")).toEqual(["alias", "child_node_id", "network_id", "node_id"]);
+      expect(shapeKeys("write_node_rules_file")).toEqual(["alias", "child_node_id", "content", "network_id", "node_id"]);
+      expect(shapeKeys("get_rules_file_request")).toEqual([]);
+      // ack 的 file_name 是节点**回报**它用了哪个文件,不是客户端**指定**用哪个文件。
+      expect(shapeKeys("ack_rules_file_request")).toEqual(["content", "error", "exists", "file_name", "request_id", "status"]);
+      expect(shapeKeys("get_rules_file_result")).toEqual(["network_id", "request_id"]);
+      for (const name of ["read_node_rules_file", "write_node_rules_file", "get_rules_file_request", "get_rules_file_result"]) {
+        for (const k of shapeKeys(name)) expect(k, `${name}.${k}`).not.toMatch(/path|dir|file|cwd/i);
+      }
+    } finally { await u.close(); }
+  }, 20_000);
+});
+
+describe("app#225 rules file — read round trip", () => {
+  test("user enqueues read → node pulls (no path in request) → acks content → user polls done", async () => {
+    const u = await connect(asUser);
+    const n = await connect(asNode);
+    try {
+      const enq = await u.call("read_node_rules_file", { node_id: NODE_ID });
+      expect(enq).toMatchObject({ ok: true, op: "read" });
+      expect(enq.request_id).toMatch(/^rf_/);
+
+      expect(await u.call("get_rules_file_result", { request_id: enq.request_id })).toMatchObject({ ok: true, status: "pending", op: "read" });
+
+      const pulled = await n.call("get_rules_file_request", {});
+      expect(pulled.ok).toBe(true);
+      expect(Object.keys(pulled.request).sort()).toEqual(["op", "request_id"]);
+      expect(pulled.request).toEqual({ request_id: enq.request_id, op: "read" });
+      // 第二次拉:已经 in_progress,不再重复下发
+      expect(await n.call("get_rules_file_request", {})).toEqual({ ok: true, request: null });
+
+      expect(await n.call("ack_rules_file_request", {
+        request_id: enq.request_id, status: "done", file_name: "CLAUDE.md", exists: true, content: "# 规则\n",
+      })).toMatchObject({ ok: true, status: "done" });
+
+      const res = await u.call("get_rules_file_result", { request_id: enq.request_id });
+      expect(res).toMatchObject({ ok: true, status: "done", op: "read", file_name: "CLAUDE.md", exists: true, content: "# 规则\n", error: null });
+    } finally { await u.close(); await n.close(); }
+  }, 20_000);
+
+  test("write: content travels to the node verbatim; result carries no content back", async () => {
+    const u = await connect(asUser);
+    const n = await connect(asNode);
+    try {
+      const body = "# new rules\n\n- 不碰生产 DB\n";
+      const enq = await u.call("write_node_rules_file", { node_id: NODE_ID, content: body });
+      expect(enq).toMatchObject({ ok: true, op: "write" });
+      const pulled = await n.call("get_rules_file_request", {});
+      expect(pulled.request).toEqual({ request_id: enq.request_id, op: "write", content: body });
+      await n.call("ack_rules_file_request", { request_id: enq.request_id, status: "done", file_name: "CLAUDE.md", exists: true });
+      const res = await u.call("get_rules_file_result", { request_id: enq.request_id });
+      expect(res).toMatchObject({ ok: true, status: "done", op: "write", file_name: "CLAUDE.md" });
+      expect(res.content).toBeUndefined();
+    } finally { await u.close(); await n.close(); }
+  }, 20_000);
+
+  test("node failure is surfaced with its reason", async () => {
+    const u = await connect(asUser);
+    const n = await connect(asNode);
+    try {
+      const enq = await u.call("write_node_rules_file", { node_id: NODE_ID, content: "x" });
+      await n.call("get_rules_file_request", {});
+      await n.call("ack_rules_file_request", { request_id: enq.request_id, status: "failed", file_name: "CLAUDE.md", error: "EACCES: permission denied" });
+      expect(await u.call("get_rules_file_result", { request_id: enq.request_id })).toMatchObject({ ok: true, status: "failed", error: "EACCES: permission denied" });
+    } finally { await u.close(); await n.close(); }
+  }, 20_000);
+});
+
+describe("app#225 rules file — SEC-1 and lifecycle", () => {
+  test("cross-network node cannot be targeted", async () => {
+    const u = await connect(asUser);
+    try {
+      expect(await u.call("read_node_rules_file", { node_id: OTHER_NODE_ID })).toMatchObject({ ok: false, error: "cross_network_node" });
+      expect(db.get<{ c: number }>("SELECT COUNT(*) AS c FROM node_rules_requests WHERE node_id = ?1", OTHER_NODE_ID)?.c).toBe(0);
+    } finally { await u.close(); }
+  }, 20_000);
+
+  test("a node in another network can neither pull nor ack this node's request", async () => {
+    const u = await connect(asUser);
+    const o = await connect(asOtherNode);
+    try {
+      const enq = await u.call("read_node_rules_file", { node_id: NODE_ID });
+      expect(await o.call("get_rules_file_request", {})).toEqual({ ok: true, request: null });
+      expect(await o.call("ack_rules_file_request", { request_id: enq.request_id, status: "done", content: "stolen" })).toMatchObject({ ok: true, ignored: "unknown_or_foreign_request" });
+      expect(db.get<{ status: string }>("SELECT status FROM node_rules_requests WHERE request_id = ?1", enq.request_id)?.status).toBe("pending");
+    } finally { await u.close(); await o.close(); }
+  }, 20_000);
+
+  test("user token (not a network token) cannot use the node-side pull/ack", async () => {
+    const u = await connect(asUser);
+    try {
+      expect(await u.call("get_rules_file_request", {})).toMatchObject({ ok: false, error: "network_token_required" });
+      expect(await u.call("ack_rules_file_request", { request_id: "rf_nope", status: "done" })).toMatchObject({ ok: false, error: "network_token_required" });
+    } finally { await u.close(); }
+  }, 20_000);
+
+  test("single flight: a second request while one is pending is refused; a stale one is superseded", async () => {
+    const u = await connect(asUser);
+    try {
+      const first = await u.call("read_node_rules_file", { node_id: NODE_ID });
+      expect(await u.call("read_node_rules_file", { node_id: NODE_ID })).toMatchObject({ ok: false, error: "request_in_flight", existing_request_id: first.request_id });
+      db.run("UPDATE node_rules_requests SET created_at = ?1 WHERE request_id = ?2", [Date.now() - 120_000, first.request_id]);
+      const second = await u.call("read_node_rules_file", { node_id: NODE_ID });
+      expect(second.ok).toBe(true);
+      expect(db.get<{ status: string }>("SELECT status FROM node_rules_requests WHERE request_id = ?1", first.request_id)?.status).toBe("timeout");
+    } finally { await u.close(); }
+  }, 20_000);
+
+  test("polling a request the node never answered flips it to timeout with a readable reason", async () => {
+    const u = await connect(asUser);
+    try {
+      const enq = await u.call("read_node_rules_file", { node_id: NODE_ID });
+      db.run("UPDATE node_rules_requests SET created_at = ?1 WHERE request_id = ?2", [Date.now() - 120_000, enq.request_id]);
+      const res = await u.call("get_rules_file_result", { request_id: enq.request_id });
+      expect(res).toMatchObject({ ok: true, status: "timeout" });
+      expect(res.error).toMatch(/did not answer/);
+    } finally { await u.close(); }
+  }, 20_000);
+
+  test("oversized content is refused before anything is queued", async () => {
+    const u = await connect(asUser);
+    try {
+      const r: any = await u.client.callTool({ name: "write_node_rules_file", arguments: { node_id: NODE_ID, content: "x".repeat(256 * 1024 + 1) } });
+      expect(r.isError).toBe(true);
+      expect(db.get<{ c: number }>("SELECT COUNT(*) AS c FROM node_rules_requests WHERE node_id = ?1", NODE_ID)?.c).toBe(0);
+    } finally { await u.close(); }
+  }, 20_000);
+});

package/src/runtime-label.test.ts

@@ -0,0 +1,22 @@
+import { describe, expect, test } from "bun:test";
+import { normalizeSessionRuntime } from "./runtime-label";
+
+describe("session runtime projection", () => {
+  test("preserves exact co-presence and CLI runtimes", () => {
+    expect(normalizeSessionRuntime("agent-node:codex-app-server")).toBe("codex-app-server");
+    expect(normalizeSessionRuntime("agent-node:grok-build-cli")).toBe("grok-build-cli");
+  });
+  test("keeps legacy labels backward-compatible", () => {
+    expect(normalizeSessionRuntime("agent-node:codex")).toBe("codex-sdk");
+    expect(normalizeSessionRuntime("agent-node:claude")).toBe("claude-agent-sdk");
+    expect(normalizeSessionRuntime("agent-node:grok")).toBe("grok-build-acp");
+    expect(normalizeSessionRuntime("agent-node:opencode")).toBe("opencode-cli");
+    expect(normalizeSessionRuntime("claude-code")).toBe("claude-code-cli");
+  });
+  test("does not invent unknown runtimes", () => {
+    expect(normalizeSessionRuntime("agent-node:future-runtime")).toBeNull();
+    expect(normalizeSessionRuntime("agent-node:codex-app-server-evil")).toBeNull();
+    expect(normalizeSessionRuntime("agent-node:grok-build-cli-preview")).toBeNull();
+    expect(normalizeSessionRuntime(null)).toBeNull();
+  });
+});

package/src/runtime-label.ts

@@ -0,0 +1,21 @@
+/** Preserve exact agent-node runtimes before applying legacy family aliases. */
+export function normalizeSessionRuntime(agent: unknown): string | null {
+  if (typeof agent !== "string" || agent.length === 0) return null;
+  if (agent === "claude-code") return "claude-code-cli";
+  const reported = agent.startsWith("agent-node:") ? agent.slice("agent-node:".length) : "";
+  const exact: Record<string, string> = {
+    "codex-app-server": "codex-app-server",
+    "grok-build-cli": "grok-build-cli",
+    opencode: "opencode-cli",
+    "opencode-cli": "opencode-cli",
+    codex: "codex-sdk",
+    "codex-sdk": "codex-sdk",
+    claude: "claude-agent-sdk",
+    "claude-agent-sdk": "claude-agent-sdk",
+    grok: "grok-build-acp",
+    "grok-build-acp": "grok-build-acp",
+  };
+  if (exact[reported]) return exact[reported];
+  if (agent === "http-api" || agent === "http" || agent === "api") return "http-api";
+  return null;
+}

package/src/scheduled-doorbell-nodeless-session.test.ts

@@ -0,0 +1,130 @@
+import { afterAll, beforeEach, describe, expect, test } from "bun:test";
+import { db } from "./db.js";
+import { createSSEStream } from "./push.js";
+import { dispatchScheduledOccurrence } from "./scheduled-tasks.js";
+
+// A claude-code node's channel server reports status without node_id, so its
+// sessions row has node_id NULL. The scheduler used to look the session up by
+// node_id only, judged every occurrence "queued" and never rang the doorbell:
+// the task sat in the inbox until an unrelated push drained it (20–36 min in
+// production).
+
+const NET = "net_sched_doorbell_nodeless";
+const NODE = "n_sched_doorbell_cc";
+const ALIAS = "sched-doorbell-cc";
+
+function cleanup(): void {
+  for (const table of ["task_events", "inbox", "scheduled_task_runs", "scheduled_tasks", "tasks", "sessions", "nodes"]) {
+    try { db.run(`DELETE FROM ${table} WHERE network_id = ?1`, [NET]); } catch {}
+  }
+}
+
+function seedNode(): void {
+  db.run(
+    `INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, lifecycle_state)
+     VALUES (?1, ?2, ?2, 'claude-code-cli', ?3, 'active')`,
+    [NODE, ALIAS, NET],
+  );
+}
+
+function seedSession(status: string, nodeId: string | null): void {
+  db.run(
+    `INSERT INTO sessions (resume_id, alias, status, node_id, network_id, agent, updated_at, last_seen_at)
+     VALUES (?1, ?2, ?3, ?4, ?5, 'claude-code', datetime('now'), datetime('now'))`,
+    [`cc-${crypto.randomUUID()}`, ALIAS, status, nodeId, NET],
+  );
+}
+
+function seedSchedule(): any {
+  const scheduleId = `sched_${crypto.randomUUID()}`;
+  db.run(
+    `INSERT INTO scheduled_tasks
+       (schedule_id, network_id, name, target_node_id, target_alias, task_content, priority,
+        schedule_type, schedule_json, timezone, status, next_run_at)
+     VALUES (?1, ?2, 'doorbell', ?3, ?4, 'scheduled doorbell task', 'normal', 'interval',
+             '{"type":"interval","every_seconds":3600}', 'UTC', 'active', datetime('now', '+1 hour'))`,
+    [scheduleId, NET, NODE, ALIAS],
+  );
+  return db.get("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", scheduleId);
+}
+
+/** Subscribe like the channel server does and collect decoded SSE frames. */
+function subscribe(): { frames: any[]; close: () => Promise<void> } {
+  const res = createSSEStream(ALIAS, NET);
+  const reader = res.body!.getReader();
+  const decoder = new TextDecoder();
+  const frames: any[] = [];
+  let buffer = "";
+  let open = true;
+  (async () => {
+    while (open) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      buffer += decoder.decode(value, { stream: true });
+      const blocks = buffer.split("\n\n");
+      buffer = blocks.pop() || "";
+      for (const block of blocks) {
+        const line = block.split("\n").find((l) => l.startsWith("data: "));
+        if (line) frames.push(JSON.parse(line.slice(6)));
+      }
+    }
+  })().catch(() => {});
+  return {
+    frames,
+    close: async () => {
+      open = false;
+      await reader.cancel().catch(() => {});
+    },
+  };
+}
+
+async function settle(): Promise<void> {
+  await new Promise((resolve) => setTimeout(resolve, 20));
+}
+
+beforeEach(() => {
+  cleanup();
+  seedNode();
+});
+
+afterAll(cleanup);
+
+describe("scheduler doorbell for sessions reported without node_id", () => {
+  test("idle node_id-less session is delivered and its subscriber receives new_task", async () => {
+    seedSession("idle", null);
+    const sub = subscribe();
+    try {
+      await settle();
+      const result = dispatchScheduledOccurrence(seedSchedule(), new Date().toISOString(), false);
+      await settle();
+      expect(result.status).toBe("delivered");
+      expect(db.get<{ status: string }>("SELECT status FROM scheduled_task_runs WHERE run_id = ?1", result.runId)?.status).toBe("delivered");
+      const wake = sub.frames.filter((f) => f.type === "new_task");
+      expect(wake).toHaveLength(1);
+      expect(wake[0].from).toBe("scheduler");
+      expect(wake[0].inbox_count).toBe(1);
+    } finally {
+      await sub.close();
+    }
+  });
+
+  test("a live subscriber counts even when the session row says offline", async () => {
+    seedSession("offline", null);
+    const sub = subscribe();
+    try {
+      await settle();
+      const result = dispatchScheduledOccurrence(seedSchedule(), new Date().toISOString(), false);
+      await settle();
+      expect(result.status).toBe("delivered");
+      expect(sub.frames.filter((f) => f.type === "new_task")).toHaveLength(1);
+    } finally {
+      await sub.close();
+    }
+  });
+
+  test("offline session with nobody listening stays queued", () => {
+    seedSession("offline", null);
+    const result = dispatchScheduledOccurrence(seedSchedule(), new Date().toISOString(), false);
+    expect(result.status).toBe("queued");
+  });
+});

package/src/scheduled-failures-http.test.ts

@@ -0,0 +1,233 @@
+// #523 —— 定时任务连续失败:告诉创建者(可选自动暂停)。
+//
+// 钉住:
+//   1. 连续 5 次失败(派出去的任务以 failed 结束)→ 给创建者恰好 1 条通知(别的成员 0 条),第 6–10 次不再发;
+//      通知里有排程名、目标节点、失败次数、最近一次的错误(截断)和怎么暂停。
+//   2. 派发时就失败(目标节点不存在)同样计数。
+//   3. 中间成功一次 → 重新上膛:再连续 5 次失败才发第二条。
+//   4. 一直失败:24 小时内不重发;上次告警早于 24 小时 → 再发一条。
+//   5. 默认不自动暂停;COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS=3 → 第 3 次失败后排程 paused、next_run_at 清空、
+//      revision+1,并通知一次。
+//   6. GET /runs 返回 consecutive_failures / failure_alert_threshold / last_failure_alert_at,
+//      任务以 failed 结束的 run 的 error_message 取自任务的回复正文。
+//
+// 跑法:cd server && COMMHUB_DB=/tmp/x.db bun test src/scheduled-failures-http.test.ts
+//       PG:COMMHUB_TEST_PG_URL=… COMMHUB_PG_EXPERIMENTAL=1(tests/test2123-hub-postgres-ladder 里注册)
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { addNetworkMember, createNetworkTokenForNode, register } from "./auth.js";
+import { db } from "./db.js";
+import { dispatchScheduledOccurrence, handleScheduledTaskRequest } from "./scheduled-tasks.js";
+import { SCHEDULE_FAILING_NOTICE_KIND, failureStreak } from "./scheduled-failures.js";
+import { registerTools } from "./tools.js";
+
+const activeDbPath = process.env.COMMHUB_DB ?? (process.env.COMMHUB_TEST_PG_URL ? "postgres" : undefined);
+if (!activeDbPath) throw new Error("scheduled-failures requires COMMHUB_DB (or COMMHUB_TEST_PG_URL) before module import");
+
+const stamp = Date.now();
+const ALIAS = `fail-node-${stamp}`;
+const NODE_ID = `n_fail_${stamp}`;
+let NET = "", creatorId = "", otherId = "";
+const ENV_KEYS = ["COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS", "COMMHUB_SCHEDULE_FAILURE_RENOTICE_SEC", "COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS"];
+const SAFETY_ERROR = "执行出错: upstream model refused the request (safety system). " + "x".repeat(600);
+
+function seedSchedule(name = "前后端分离巡检", targetNodeId = NODE_ID): any {
+  const scheduleId = `sched_${crypto.randomUUID()}`;
+  db.run(
+    `INSERT INTO scheduled_tasks
+       (schedule_id, network_id, created_by, name, target_node_id, target_alias, task_content, priority,
+        schedule_type, schedule_json, timezone, status, next_run_at)
+     VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'scheduled task', 'normal', 'interval',
+             '{"type":"interval","every_seconds":120}', 'Asia/Shanghai', 'active', datetime('now', '+2 minutes'))`,
+    [scheduleId, NET, creatorId, name, targetNodeId, ALIAS],
+  );
+  return reload(scheduleId);
+}
+const reload = (scheduleId: string) => db.get<any>("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", scheduleId);
+
+let slot = 0;
+const nextSlot = () => new Date(Date.UTC(2026, 9, 4, 0, 0, 0) + (++slot) * 120_000).toISOString();
+const fire = (schedule: any, now = new Date()) => dispatchScheduledOccurrence(schedule, nextSlot(), false, now);
+
+const notices = (userId: string) => db.all<{ title: string; content: string; meta_json: string; from_session: string; severity: string }>(
+  "SELECT title, content, meta_json, from_session, severity FROM user_inbox WHERE user_id = ?1 AND network_id = ?2 AND kind = ?3 ORDER BY created_at, message_id",
+  userId, NET, SCHEDULE_FAILING_NOTICE_KIND,
+);
+const noticesFor = (userId: string, scheduleId: string) => notices(userId).filter((n) => JSON.parse(n.meta_json).schedule_failing.schedule_id === scheduleId);
+
+async function nodeReply(args: Record<string, unknown>): Promise<any> {
+  const mcp = new McpServer({ name: "sched-fail", version: "0" }) as any;
+  const handlers: Record<string, (a: any) => Promise<any>> = {};
+  const original = mcp.tool.bind(mcp);
+  mcp.tool = (name: string, ...rest: any[]) => {
+    const handler = rest.at(-1);
+    if (typeof handler === "function") handlers[name] = handler;
+    return original(name, ...rest);
+  };
+  registerTools(mcp, undefined, NET, creatorId, ALIAS, true, null);
+  const out = await handlers.send_reply(args);
+  return JSON.parse(out.content[0].text);
+}
+
+/** 派一次,节点以 failed(或 replied)回复。 */
+async function runOnce(schedule: any, outcome: "failed" | "replied" = "failed"): Promise<{ runId: string; taskId: string }> {
+  const r = fire(schedule);
+  expect(r.status).toBe("delivered");
+  const out = await nodeReply({ alias: "scheduler", text: outcome === "failed" ? SAFETY_ERROR : "done", in_reply_to: r.taskId, status: outcome, from_session: ALIAS });
+  expect(out.ok).toBe(true);
+  return { runId: r.runId, taskId: r.taskId! };
+}
+
+async function getRuns(scheduleId: string): Promise<any> {
+  const url = new URL(`http://hub.test/api/scheduled-tasks/${scheduleId}/runs`);
+  const res = await handleScheduledTaskRequest({
+    req: new Request(url, { method: "GET" }), url,
+    auth: { userId: creatorId, networkId: NET, username: "creator" },
+    isAdmin: false, isNodeToken: false, scope: { networkId: NET, networkIds: [NET] } as any,
+  });
+  expect(res!.status).toBe(200);
+  return res!.json();
+}
+
+beforeAll(() => {
+  const creator = register(`fail_creator_${stamp}`, "FailCreator123!", undefined, "seed");
+  expect(creator.ok).toBe(true);
+  NET = creator.network_id!; creatorId = creator.user!.user_id;
+  const other = register(`fail_other_${stamp}`, "FailOther123!", undefined, "seed");
+  otherId = other.user!.user_id;
+  expect(addNetworkMember(NET, otherId, "member", creatorId, { agentAccess: "all" }).ok).toBe(true);
+  expect(createNetworkTokenForNode(creatorId, NET, ALIAS).ok).toBe(true);
+  db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id) VALUES (?1, ?2, ?2, 'claude-code', ?3)", [NODE_ID, ALIAS, NET]);
+  db.run("INSERT INTO sessions (resume_id, alias, status, node_id, network_id) VALUES (?1, ?2, 'idle', ?3, ?4)", [`r_fail_${stamp}`, ALIAS, NODE_ID, NET]);
+});
+beforeEach(() => { for (const k of ENV_KEYS) delete process.env[k]; });
+afterEach(() => { for (const k of ENV_KEYS) delete process.env[k]; });
+afterAll(() => { db.run("DELETE FROM user_inbox WHERE network_id = ?1 AND kind = ?2", [NET, SCHEDULE_FAILING_NOTICE_KIND]); });
+
+describe("#523 consecutive-failure alert", () => {
+  test("5 failed runs → exactly one notice to the creator with name/target/count/error/how to pause; runs 6–10 stay quiet", async () => {
+    const s = seedSchedule();
+    for (let i = 0; i < 4; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(0);
+    await runOnce(s);
+    const mine = noticesFor(creatorId, s.schedule_id);
+    expect(mine.length).toBe(1);
+    expect(noticesFor(otherId, s.schedule_id).length).toBe(0);
+    const n = mine[0];
+    expect(n.from_session).toBe(ALIAS);
+    expect(n.title).toBe("定时任务连续失败");
+    expect(n.severity).toBe("error");
+    for (const frag of ["前后端分离巡检", ALIAS, "连续 5 次", "safety system", "task_failed", "暂停", s.schedule_id]) expect(n.content).toContain(frag);
+    expect(n.content).not.toContain("x".repeat(400)); // 错误被截断
+    expect(JSON.parse(n.meta_json).schedule_failing).toEqual({ reason: "failing", schedule_id: s.schedule_id, consecutive_failures: 5, error_code: "task_failed" });
+    for (let i = 0; i < 5; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    expect(reload(s.schedule_id).status).toBe("active"); // 默认不自动暂停
+  });
+
+  test("dispatch-time failures (target node gone) count too", () => {
+    const s = seedSchedule("孤儿排程", `n_missing_${stamp}`);
+    for (let i = 0; i < 5; i++) expect(fire(s).status).toBe("failed");
+    const mine = noticesFor(creatorId, s.schedule_id);
+    expect(mine.length).toBe(1);
+    expect(mine[0].content).toContain("target_node_not_found");
+    expect(mine[0].content).toContain("The bound node no longer exists");
+  });
+
+  test("a success in between re-arms: the next 5 failures notify again", async () => {
+    const s = seedSchedule();
+    for (let i = 0; i < 5; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    await runOnce(s, "replied");
+    expect(failureStreak(s.schedule_id).consecutiveFailures).toBe(0);
+    for (let i = 0; i < 4; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(2);
+  });
+
+  test("still failing: no repeat within 24h; one more once the last alert is older than 24h", async () => {
+    const s = seedSchedule();
+    for (let i = 0; i < 5; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    db.run("UPDATE scheduled_tasks SET failure_alert_at = ?1 WHERE schedule_id = ?2", [new Date(Date.now() - 23 * 3_600_000).toISOString(), s.schedule_id]);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    db.run("UPDATE scheduled_tasks SET failure_alert_at = ?1 WHERE schedule_id = ?2", [new Date(Date.now() - 25 * 3_600_000).toISOString(), s.schedule_id]);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(2);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(2);
+  });
+
+  test("COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS changes N; no creator → no notice", async () => {
+    process.env.COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS = "2";
+    const s = seedSchedule();
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(0);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+
+    const orphan = seedSchedule();
+    db.run("UPDATE scheduled_tasks SET created_by = NULL WHERE schedule_id = ?1", [orphan.schedule_id]);
+    const row = reload(orphan.schedule_id);
+    for (let i = 0; i < 3; i++) await runOnce(row);
+    expect(noticesFor(creatorId, orphan.schedule_id).length).toBe(0);
+  });
+});
+
+describe("#523 auto-pause (opt-in)", () => {
+  test("off by default: 10 failures leave the schedule active", async () => {
+    const s = seedSchedule();
+    for (let i = 0; i < 10; i++) await runOnce(s);
+    const after = reload(s.schedule_id);
+    expect(after.status).toBe("active");
+    expect(after.next_run_at).not.toBeNull();
+  });
+
+  test("COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS=3 pauses after the 3rd failure and tells the creator once", async () => {
+    process.env.COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS = "3";
+    const s = seedSchedule();
+    await runOnce(s); await runOnce(s);
+    expect(reload(s.schedule_id).status).toBe("active");
+    await runOnce(s);
+    const after = reload(s.schedule_id);
+    expect(after.status).toBe("paused");
+    expect(after.next_run_at).toBeNull();
+    expect(after.revision).toBe(s.revision + 1);
+    const mine = noticesFor(creatorId, s.schedule_id);
+    expect(mine.length).toBe(1);
+    expect(mine[0].title).toBe("定时任务连续失败,已自动暂停");
+    expect(JSON.parse(mine[0].meta_json).schedule_failing.reason).toBe("paused");
+    // 手动 run-now 再失败两次到 N=5:同一段失败,已经通知过 → 不再发
+    await runOnce(after); await runOnce(after);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+  });
+});
+
+describe("#523 GET /runs", () => {
+  test("exposes consecutive_failures and the failed task's reason as error_message", async () => {
+    const s = seedSchedule();
+    await runOnce(s, "replied");
+    const fails = [await runOnce(s), await runOnce(s)];
+    const body = await getRuns(s.schedule_id);
+    expect(body.ok).toBe(true);
+    expect(body.consecutive_failures).toBe(2);
+    expect(body.failure_alert_threshold).toBe(5);
+    expect(body.last_failure_alert_at).toBeNull();
+    expect(body.runs.length).toBe(3);
+    const failed = body.runs.find((r: any) => r.run_id === fails[1].runId);
+    expect(failed.status).toBe("failed");
+    expect(failed.error_code).toBe("task_failed");
+    expect(failed.error_message).toContain("safety system");
+    expect(failed.error_message.length).toBeLessThanOrEqual(500);
+    for (const k of ["scheduled_for", "created_at", "completed_at", "task_id"]) expect(k in failed).toBe(true);
+    const ok = body.runs.find((r: any) => r.status === "replied");
+    expect(ok.error_message).toBeNull();
+
+    for (let i = 0; i < 3; i++) await runOnce(s);
+    const again = await getRuns(s.schedule_id);
+    expect(again.consecutive_failures).toBe(5);
+    expect(typeof again.last_failure_alert_at).toBe("string");
+  });
+});