@sleep2agi/commhub-server 0.9.0-preview.184 → 0.9.0-preview.186

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sleep2agi/commhub-server",
3
- "version": "0.9.0-preview.184",
3
+ "version": "0.9.0-preview.186",
4
4
  "description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
5
5
  "type": "module",
6
6
  "main": "src/index.ts",
@@ -0,0 +1,80 @@
1
+ // #2840 —— 连续确定性失败 → 一条告警;且不因 timeout 或一次性失败而告警。
2
+ // 校准依据见模块头注释:只算确定性错误时 7 天 6 条(0.9/天);含 timeout 则是 35 条。
3
+ import { describe, expect, test, beforeAll, afterAll } from "bun:test";
4
+ import { mkdtempSync, rmSync } from "fs";
5
+ import { join } from "path";
6
+ import { tmpdir } from "os";
7
+ import { register } from "./auth.js";
8
+ import { db } from "./db.js";
9
+ import { scanNodeFailureAlerts, classifyDeterministicFailure, NODE_FAILURE_ALERT_KIND } from "./node-failure-alert.js";
10
+
11
+ const DIR = mkdtempSync(join(tmpdir(), "anet-nfa-"));
12
+ let NET = "", UID = "", NODE = "";
13
+
14
+ function task(to: string, status: string, result: string, minutesAgo: number) {
15
+ const id = `t_nfa_${crypto.randomUUID()}`;
16
+ db.run(
17
+ `INSERT INTO tasks (task_id, from_name, to_name, priority, status, content, result, created_at, completed_at, network_id)
18
+ VALUES (?1,'scheduler',?2,'normal',?3,'c',?4, datetime('now', ?5), datetime('now', ?5), ?6)`,
19
+ [id, to, status, result, `-${minutesAgo} minutes`, NET],
20
+ );
21
+ }
22
+
23
+ const alerts = () => db.all<{ id: string }>(
24
+ "SELECT message_id AS id FROM user_inbox WHERE kind = ?1", NODE_FAILURE_ALERT_KIND,
25
+ );
26
+
27
+ beforeAll(() => {
28
+ process.env.COMMHUB_DB ||= join(DIR, "hub.db");
29
+ const a = register(`nfa_owner_${Date.now()}`, "NfaPassw0rd!x", undefined, "Owner");
30
+ expect(a.ok).toBe(true);
31
+ NET = a.network_id!; UID = a.user!.user_id;
32
+ NODE = `nfa-node-${Date.now()}`;
33
+ db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, owner_user_id) VALUES (?1,?2,?2,'codex',?3,?4)",
34
+ [`n_${NODE}`, NODE, NET, UID]);
35
+ });
36
+ afterAll(() => { try { rmSync(DIR, { recursive: true, force: true }); } catch {} });
37
+
38
+ describe("#2840 节点连续确定性失败告警", () => {
39
+ test("分类:只认确定性错误,timeout 与任务内容错误都不算", () => {
40
+ expect(classifyDeterministicFailure("codex-app-server 错误: Missing environment variable: `DEEPSEEK_KEY`")).toBe("missing_env");
41
+ expect(classifyDeterministicFailure("spawn ENOENT")).toBe("binary_missing");
42
+ expect(classifyDeterministicFailure("401 Unauthorized")).toBe("login_required");
43
+ expect(classifyDeterministicFailure("codex request 'turn/start' timed out")).toBeNull();
44
+ expect(classifyDeterministicFailure("任务内容写错了")).toBeNull();
45
+ expect(classifyDeterministicFailure(null)).toBeNull();
46
+ });
47
+
48
+ test("连续 5 次缺环境变量 → 恰好一条告警;再次扫描不重复发", () => {
49
+ for (let i = 5; i >= 1; i--) task(NODE, "failed", "Missing environment variable: `DEEPSEEK_KEY`", i * 3);
50
+ const first = scanNodeFailureAlerts();
51
+ expect(first.nodes).toContain(NODE);
52
+ expect(alerts().length).toBe(1);
53
+ const again = scanNodeFailureAlerts();
54
+ expect(again.nodes).not.toContain(NODE);
55
+ expect(alerts().length).toBe(1);
56
+ });
57
+
58
+ test("timeout 连续 10 次不告警(这是校准结论的回归钉)", () => {
59
+ const other = `nfa-timeout-${Date.now()}`;
60
+ db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, owner_user_id) VALUES (?1,?2,?2,'codex',?3,?4)",
61
+ [`n_${other}`, other, NET, UID]);
62
+ for (let i = 10; i >= 1; i--) task(other, "failed", "turn/start timed out", i * 2);
63
+ const r = scanNodeFailureAlerts();
64
+ expect(r.nodes).not.toContain(other);
65
+ expect(alerts().length).toBe(1); // 仍然只有前面那一条
66
+ });
67
+
68
+ test("中间出现一条成功就断开连续,不告警", () => {
69
+ const reset = `nfa-reset-${Date.now()}`;
70
+ db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, owner_user_id) VALUES (?1,?2,?2,'codex',?3,?4)",
71
+ [`n_${reset}`, reset, NET, UID]);
72
+ task(reset, "failed", "Missing environment variable: `X`", 30);
73
+ task(reset, "failed", "Missing environment variable: `X`", 29);
74
+ task(reset, "replied", "ok 正常完成", 28);
75
+ task(reset, "failed", "Missing environment variable: `X`", 27);
76
+ task(reset, "failed", "Missing environment variable: `X`", 26);
77
+ const r = scanNodeFailureAlerts();
78
+ expect(r.nodes).not.toContain(reset);
79
+ });
80
+ });
@@ -0,0 +1,104 @@
1
+ // #2840 —— 节点「假活」告警:连续确定性失败 → 一条人类可见的通知。
2
+ //
3
+ // 现场(2026-10-28 只读统计):TMA客户端鲸 因为缺 `DEEPSEEK_KEY`,在 **9 天里失败 1433 次**,
4
+ // 期间没有任何告警。节点在线、进程在、版本正常、`get_session_status` 返回 idle,
5
+ // 失败只写在任务回执的 result 文本里 —— 巡检若只看状态就会判「没回复 = 没在做」。
6
+ //
7
+ // 设计约束(用近 7 天生产数据回放校准过,见 #2840):
8
+ // · **只对确定性错误告警**:缺环境变量 / 二进制缺失 / 登录失效 —— 这类不会自愈,必须人处理。
9
+ // 把 `timeout` 也算进去会让 7 天出现 35 条告警(其中 29 条是 timeout),等于制造新噪音源。
10
+ // · 阈值 5 次连续(同节点 + 同类别)。校准结果:只算确定性错误时 7 天 6 条(0.9/天),
11
+ // 而 TMA客户端鲸 那 1433 次会被压缩成 5 条 —— 压缩比 287:1。
12
+ // · 同一次故障只告一次:按 (节点, 类别, 首现时刻) 去重;彻底恢复(出现一条成功)后才重新武装。
13
+ // · 走已有 user_inbox 通道(与 schedule_stuck 同一套),不新造通知面。
14
+ import { db } from "./db.js";
15
+ import { sendAgentNotice } from "./agent-notice.js";
16
+
17
+ export const NODE_FAILURE_ALERT_KIND = "node_failure_alert";
18
+ /** 连续多少次确定性失败才告警。 */
19
+ export const ALERT_STREAK_THRESHOLD = 5;
20
+ /** 只看这么久以内的任务(更早的失败已经不可操作)。 */
21
+ export const SCAN_WINDOW_HOURS = 24;
22
+
23
+ type Category = "missing_env" | "binary_missing" | "login_required";
24
+
25
+ const PATTERNS: Array<{ category: Category; re: RegExp; label: string; hint: string }> = [
26
+ { category: "missing_env", re: /Missing environment variable/i, label: "缺环境变量",
27
+ hint: "在节点所在机器补上该变量后重启节点(anet node start <alias>)" },
28
+ { category: "binary_missing", re: /ENOENT|No such file or directory/i, label: "二进制/文件缺失",
29
+ hint: "检查该节点的运行时是否安装完整,必要时重装后重启节点" },
30
+ { category: "login_required", re: /login required|Unauthorized|\b401\b/i, label: "凭据/登录失效",
31
+ hint: "重新登录该节点(anet node codex login-status 可先自检)" },
32
+ ];
33
+
34
+ /** 只认确定性错误;timeout / 任务内容错误一律返回 null(那类会自愈或与节点无关)。 */
35
+ export function classifyDeterministicFailure(result: string | null | undefined): Category | null {
36
+ if (!result) return null;
37
+ for (const p of PATTERNS) if (p.re.test(result)) return p.category;
38
+ return null;
39
+ }
40
+
41
+ export type AlertScanResult = { alerts: number; nodes: string[] };
42
+
43
+ /**
44
+ * 扫一遍最近的失败,给「连续确定性失败」的节点发一次性告警。
45
+ * 纯读 + 发通知;不改任何任务/排程状态。
46
+ */
47
+ export function scanNodeFailureAlerts(now = new Date()): AlertScanResult {
48
+ const since = new Date(now.getTime() - SCAN_WINDOW_HOURS * 3_600_000).toISOString().slice(0, 19).replace("T", " ");
49
+ const rows = db.all<{ to_name: string; network_id: string | null; created_at: string; status: string; result: string | null }>(
50
+ `SELECT to_name, network_id, created_at, status, result FROM tasks
51
+ WHERE created_at >= ?1 AND to_name IS NOT NULL
52
+ ORDER BY to_name, created_at`,
53
+ since,
54
+ );
55
+ // 按节点切段:streak 只数「连续」——中间出现任何非该类失败的成功/失败都要断开。
56
+ const streaks = new Map<string, { category: Category; count: number; firstAt: string; networkId: string | null; sample: string }>();
57
+ const cur = new Map<string, { category: Category; count: number; firstAt: string }>();
58
+ for (const r of rows) {
59
+ const cat = r.status === "failed" ? classifyDeterministicFailure(r.result) : null;
60
+ const node = r.to_name;
61
+ const prev = cur.get(node);
62
+ if (!cat) {
63
+ // 序列断了:连续计数和「已完成段落」都要清,否则一条成功之后旧段落仍会被告警。
64
+ cur.delete(node);
65
+ streaks.delete(node);
66
+ continue;
67
+ }
68
+ if (prev && prev.category === cat) {
69
+ prev.count += 1;
70
+ streaks.set(node, { category: cat, count: prev.count, firstAt: prev.firstAt, networkId: r.network_id, sample: String(r.result ?? "").slice(0, 200) });
71
+ } else {
72
+ cur.set(node, { category: cat, count: 1, firstAt: r.created_at });
73
+ streaks.delete(node);
74
+ }
75
+ }
76
+ const out: string[] = [];
77
+ for (const [node, s] of streaks) {
78
+ if (s.count < ALERT_STREAK_THRESHOLD) continue;
79
+ const meta = { node, category: s.category, first_seen: s.firstAt };
80
+ const dup = db.get<{ n: number }>(
81
+ `SELECT COUNT(*) AS n FROM user_inbox WHERE kind = ?1 AND from_session = ?2
82
+ AND json_extract(meta_json, '$.node_failure_alert.category') = ?3
83
+ AND json_extract(meta_json, '$.node_failure_alert.first_seen') = ?4`,
84
+ NODE_FAILURE_ALERT_KIND, node, s.category, s.firstAt,
85
+ )?.n ?? 0;
86
+ if (dup > 0) continue;
87
+ const owner = db.get<{ owner_user_id: string | null }>("SELECT owner_user_id FROM nodes WHERE alias = ?1 AND network_id = ?2", node, s.networkId)?.owner_user_id;
88
+ if (!owner || !s.networkId) continue;
89
+ const label = PATTERNS.find((p) => p.category === s.category)!;
90
+ const sent = sendAgentNotice({
91
+ networkId: s.networkId,
92
+ userId: owner,
93
+ fromAlias: node,
94
+ kind: NODE_FAILURE_ALERT_KIND,
95
+ title: `节点连续失败:${label.label}`,
96
+ text: `${node} 连续 ${s.count} 次以「${label.label}」失败(自 ${s.firstAt} 起)。\n\n最近一次回执节选:\n${s.sample}\n\n处理建议:${label.hint}`,
97
+ severity: "error",
98
+ meta: { node_failure_alert: meta },
99
+ idPrefix: "dm_nfa_",
100
+ });
101
+ if (sent) out.push(node);
102
+ }
103
+ return { alerts: out.length, nodes: out };
104
+ }
@@ -256,6 +256,19 @@ describe("GET /api/networks/:id/node-permission-report", () => {
256
256
  // #2823 —— 调用面计数:拒绝日志给不了分母。
257
257
  // 生产 30 天 `node_permission_log` 0 行,分不清「没人越界」和「没人碰过这些工具」,
258
258
  // 而 #525 要切 enforce,恰恰需要一个分母。
259
+ describe("#2823 计数表的上限必须覆盖整个保留期", () => {
260
+ // 撞上限的后果是**静默归零**(callLogFull 只给已有行加次数、不插新行),
261
+ // 而这正是 #525 用来判「要不要切 enforce」的判据 —— 读到归零会把「没人调用」当结论。
262
+ // 生产实测 75 行/小时(.184 上线首小时);这里按 100 行/小时留余量。
263
+ test("TOOL_CALL_MAX_ROWS 覆盖 LOG_RETENTION_DAYS × 100 行/小时,否则会在保留期内静默停表", async () => {
264
+ const { TOOL_CALL_MAX_ROWS, LOG_RETENTION_DAYS } = await import("./node-permissions.js");
265
+ const worstCasePerHour = 100;
266
+ expect(TOOL_CALL_MAX_ROWS).toBeGreaterThanOrEqual(LOG_RETENTION_DAYS * 24 * worstCasePerHour);
267
+ // 反向验证这条测试有判别力:旧的 20 000 必须不满足
268
+ expect(20_000).toBeLessThan(LOG_RETENTION_DAYS * 24 * worstCasePerHour);
269
+ });
270
+ });
271
+
259
272
  describe("#2823 调用面计数(node_tool_calls)", () => {
260
273
  const callHits = (nodeId: string, tool: string) => Number(
261
274
  db.get<{ h: number | string | null }>(
@@ -73,7 +73,21 @@ db.exec(`
73
73
  `);
74
74
 
75
75
  let callLogFull = false;
76
- export const TOOL_CALL_MAX_ROWS = 20_000;
76
+ /**
77
+ * 行数上限。**刻意比 `LOG_MAX_ROWS`(20 000) 高一个数量级**。
78
+ *
79
+ * 为什么不能沿用 20 000:这张表的键是 `(network, node, tool, hour)` —— 每过一个小时
80
+ * 就是一整批新键。生产实测(.184 上线 1 小时):75 行/小时 ⇒ 1 800 行/天,
81
+ * 20 000 ÷ 1 800 ≈ **11 天**,而保留期是 30 天。
82
+ *
83
+ * 撞上限的后果不是「变慢」,而是 `callLogFull` 只给已有行加次数、**不再插新行**,
84
+ * 于是新的一小时一行都写不进去 —— 计数**静默归零**,不报错、不告警。
85
+ * 而这张表正是 #525「要不要切 enforce」的判据,读到归零会把「没人调用」当成结论。
86
+ *
87
+ * 200 000 行 ≈ 26 MB(实测 91 行 = 12 KB),对这张表完全可接受;
88
+ * 按当前速率可覆盖 30 天保留期且留 2 倍余量。
89
+ */
90
+ export const TOOL_CALL_MAX_ROWS = 200_000;
77
91
 
78
92
  /** 计一次「到达权限闸」的工具调用。绝不影响请求本身 —— 出错只写日志。 */
79
93
  export function recordNodeToolCall(id: NodeIdentity, tool: string): void {
package/src/server.ts CHANGED
@@ -84,6 +84,7 @@ import { startRetentionSweeper } from "./retention.js";
84
84
  import { sweepEmptyNodeRows } from "./empty-node-sweeper.js";
85
85
  import { versionStats } from "./version-stats.js";
86
86
  import { startStaleSessionSweeper } from "./stale-sweeper.js";
87
+ import { scanNodeFailureAlerts } from "./node-failure-alert.js";
87
88
  import {
88
89
  AUDIT_LOG_REST_SELECT,
89
90
  COMPLETION_REST_SELECT,
@@ -5784,6 +5785,21 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5784
5785
  // every COMMHUB_STALE_SWEEP_SECONDS (default 60s).
5785
5786
  const staleSweeperTimer = startStaleSessionSweeper();
5786
5787
 
5788
+ // #2840 —— 节点「假活」告警:连续 5 次确定性失败(缺环境变量 / 二进制缺失 / 登录失效)
5789
+ // 给节点主人发一条一次性通知。校准见 node-failure-alert.ts 头注释:
5790
+ // 只算确定性错误时 7 天 6 条(0.9/天);把 timeout 也算进去会变成 35 条。
5791
+ const nodeFailureAlertMs = Number(process.env.COMMHUB_NODE_FAILURE_ALERT_MS) > 0
5792
+ ? Number(process.env.COMMHUB_NODE_FAILURE_ALERT_MS) : 5 * 60 * 1000;
5793
+ const nodeFailureAlertTimer = setInterval(() => {
5794
+ try {
5795
+ const r = scanNodeFailureAlerts();
5796
+ if (r.alerts > 0) console.log(`[commhub node-failure-alert] sent ${r.alerts} alert(s): ${r.nodes.join(", ")}`);
5797
+ } catch (e: any) {
5798
+ console.log(`[commhub node-failure-alert] failed: ${e?.message ?? e}`);
5799
+ }
5800
+ }, nodeFailureAlertMs);
5801
+ (nodeFailureAlertTimer as any).unref?.();
5802
+
5787
5803
  // #476 — the two legacy module-level intervals, now owned by startHub
5788
5804
  // like every other periodic job: rate-limit map cleanup (memory only)
5789
5805
  // and task expiration patrol (writes DB — must never run on import).
@@ -5825,6 +5841,7 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5825
5841
  clearInterval(aliasBaselineTimer);
5826
5842
  clearTimeout(deferredBackfillTimer);
5827
5843
  clearInterval(staleSweeperTimer);
5844
+ clearInterval(nodeFailureAlertTimer);
5828
5845
  clearInterval(rateLimitSweepTimer);
5829
5846
  clearInterval(taskPatrolTimer);
5830
5847
  clearInterval(deliveredStalePatrolTimer);