@sleep2agi/commhub-server 0.9.0-preview.185 → 0.9.0-preview.187

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sleep2agi/commhub-server",
3
- "version": "0.9.0-preview.185",
3
+ "version": "0.9.0-preview.187",
4
4
  "description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
5
5
  "type": "module",
6
6
  "main": "src/index.ts",
package/src/db.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { refreshScheduledBackoff } from "./scheduled-backoff.js";
1
+ import { refreshScheduledBackoff } from "./scheduled-backoff.js"; import { scheduledFailureCode } from "./scheduled-failure-code.js";
2
2
  import { createAdapter, type DbAdapter } from "./db-adapter"; import { withStatementHook } from "./status-read-cache"; // #431 写 sessions 的语句让 /api/status 缓存失效
3
3
 
4
4
  export const db: DbAdapter = withStatementHook(createAdapter());
@@ -2345,6 +2345,7 @@ type ScheduledTaskLifecycleRow = {
2345
2345
  network_id: string | null;
2346
2346
  status: string;
2347
2347
  completed_at: string | null;
2348
+ result: string | null;
2348
2349
  };
2349
2350
 
2350
2351
  type ScheduledTaskBinding = { run_id: string; schedule_id: string };
@@ -2368,7 +2369,7 @@ export function syncScheduledRunForTask(taskId: string, expectedNetworkId?: stri
2368
2369
 
2369
2370
  function syncScheduledRunForTaskInTransaction(taskId: string, expectedNetworkId?: string | null): { matched: boolean; status?: string } {
2370
2371
  const task = db.get<ScheduledTaskLifecycleRow>(
2371
- `SELECT task_id, network_id, status, completed_at
2372
+ `SELECT task_id, network_id, status, completed_at, result
2372
2373
  FROM tasks
2373
2374
  WHERE task_id = ?1
2374
2375
  AND (CAST(?2 AS TEXT) IS NULL OR network_id = ?2)`,
@@ -2389,7 +2390,7 @@ function syncScheduledRunForTaskInTransaction(taskId: string, expectedNetworkId?
2389
2390
  const binding = bindings[0];
2390
2391
 
2391
2392
  if (SCHEDULED_RUN_TERMINAL_STATUSES.has(task.status)) {
2392
- const errorCode = task.status === "replied" ? null : `task_${task.status}`;
2393
+ const errorCode = scheduledFailureCode(task.status, task.result);
2393
2394
  const updated = db.run(
2394
2395
  `UPDATE scheduled_task_runs
2395
2396
  SET status = ?1,
@@ -0,0 +1,80 @@
1
+ // #2840 —— 连续确定性失败 → 一条告警;且不因 timeout 或一次性失败而告警。
2
+ // 校准依据见模块头注释:只算确定性错误时 7 天 6 条(0.9/天);含 timeout 则是 35 条。
3
+ import { describe, expect, test, beforeAll, afterAll } from "bun:test";
4
+ import { mkdtempSync, rmSync } from "fs";
5
+ import { join } from "path";
6
+ import { tmpdir } from "os";
7
+ import { register } from "./auth.js";
8
+ import { db } from "./db.js";
9
+ import { scanNodeFailureAlerts, classifyDeterministicFailure, NODE_FAILURE_ALERT_KIND } from "./node-failure-alert.js";
10
+
11
+ const DIR = mkdtempSync(join(tmpdir(), "anet-nfa-"));
12
+ let NET = "", UID = "", NODE = "";
13
+
14
+ function task(to: string, status: string, result: string, minutesAgo: number) {
15
+ const id = `t_nfa_${crypto.randomUUID()}`;
16
+ db.run(
17
+ `INSERT INTO tasks (task_id, from_name, to_name, priority, status, content, result, created_at, completed_at, network_id)
18
+ VALUES (?1,'scheduler',?2,'normal',?3,'c',?4, datetime('now', ?5), datetime('now', ?5), ?6)`,
19
+ [id, to, status, result, `-${minutesAgo} minutes`, NET],
20
+ );
21
+ }
22
+
23
+ const alerts = () => db.all<{ id: string }>(
24
+ "SELECT message_id AS id FROM user_inbox WHERE kind = ?1", NODE_FAILURE_ALERT_KIND,
25
+ );
26
+
27
+ beforeAll(() => {
28
+ process.env.COMMHUB_DB ||= join(DIR, "hub.db");
29
+ const a = register(`nfa_owner_${Date.now()}`, "NfaPassw0rd!x", undefined, "Owner");
30
+ expect(a.ok).toBe(true);
31
+ NET = a.network_id!; UID = a.user!.user_id;
32
+ NODE = `nfa-node-${Date.now()}`;
33
+ db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, owner_user_id) VALUES (?1,?2,?2,'codex',?3,?4)",
34
+ [`n_${NODE}`, NODE, NET, UID]);
35
+ });
36
+ afterAll(() => { try { rmSync(DIR, { recursive: true, force: true }); } catch {} });
37
+
38
+ describe("#2840 节点连续确定性失败告警", () => {
39
+ test("分类:只认确定性错误,timeout 与任务内容错误都不算", () => {
40
+ expect(classifyDeterministicFailure("codex-app-server 错误: Missing environment variable: `DEEPSEEK_KEY`")).toBe("missing_env");
41
+ expect(classifyDeterministicFailure("spawn ENOENT")).toBe("binary_missing");
42
+ expect(classifyDeterministicFailure("401 Unauthorized")).toBe("login_required");
43
+ expect(classifyDeterministicFailure("codex request 'turn/start' timed out")).toBeNull();
44
+ expect(classifyDeterministicFailure("任务内容写错了")).toBeNull();
45
+ expect(classifyDeterministicFailure(null)).toBeNull();
46
+ });
47
+
48
+ test("连续 5 次缺环境变量 → 恰好一条告警;再次扫描不重复发", () => {
49
+ for (let i = 5; i >= 1; i--) task(NODE, "failed", "Missing environment variable: `DEEPSEEK_KEY`", i * 3);
50
+ const first = scanNodeFailureAlerts();
51
+ expect(first.nodes).toContain(NODE);
52
+ expect(alerts().length).toBe(1);
53
+ const again = scanNodeFailureAlerts();
54
+ expect(again.nodes).not.toContain(NODE);
55
+ expect(alerts().length).toBe(1);
56
+ });
57
+
58
+ test("timeout 连续 10 次不告警(这是校准结论的回归钉)", () => {
59
+ const other = `nfa-timeout-${Date.now()}`;
60
+ db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, owner_user_id) VALUES (?1,?2,?2,'codex',?3,?4)",
61
+ [`n_${other}`, other, NET, UID]);
62
+ for (let i = 10; i >= 1; i--) task(other, "failed", "turn/start timed out", i * 2);
63
+ const r = scanNodeFailureAlerts();
64
+ expect(r.nodes).not.toContain(other);
65
+ expect(alerts().length).toBe(1); // 仍然只有前面那一条
66
+ });
67
+
68
+ test("中间出现一条成功就断开连续,不告警", () => {
69
+ const reset = `nfa-reset-${Date.now()}`;
70
+ db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, owner_user_id) VALUES (?1,?2,?2,'codex',?3,?4)",
71
+ [`n_${reset}`, reset, NET, UID]);
72
+ task(reset, "failed", "Missing environment variable: `X`", 30);
73
+ task(reset, "failed", "Missing environment variable: `X`", 29);
74
+ task(reset, "replied", "ok 正常完成", 28);
75
+ task(reset, "failed", "Missing environment variable: `X`", 27);
76
+ task(reset, "failed", "Missing environment variable: `X`", 26);
77
+ const r = scanNodeFailureAlerts();
78
+ expect(r.nodes).not.toContain(reset);
79
+ });
80
+ });
@@ -0,0 +1,104 @@
1
+ // #2840 —— 节点「假活」告警:连续确定性失败 → 一条人类可见的通知。
2
+ //
3
+ // 现场(2026-10-28 只读统计):TMA客户端鲸 因为缺 `DEEPSEEK_KEY`,在 **9 天里失败 1433 次**,
4
+ // 期间没有任何告警。节点在线、进程在、版本正常、`get_session_status` 返回 idle,
5
+ // 失败只写在任务回执的 result 文本里 —— 巡检若只看状态就会判「没回复 = 没在做」。
6
+ //
7
+ // 设计约束(用近 7 天生产数据回放校准过,见 #2840):
8
+ // · **只对确定性错误告警**:缺环境变量 / 二进制缺失 / 登录失效 —— 这类不会自愈,必须人处理。
9
+ // 把 `timeout` 也算进去会让 7 天出现 35 条告警(其中 29 条是 timeout),等于制造新噪音源。
10
+ // · 阈值 5 次连续(同节点 + 同类别)。校准结果:只算确定性错误时 7 天 6 条(0.9/天),
11
+ // 而 TMA客户端鲸 那 1433 次会被压缩成 5 条 —— 压缩比 287:1。
12
+ // · 同一次故障只告一次:按 (节点, 类别, 首现时刻) 去重;彻底恢复(出现一条成功)后才重新武装。
13
+ // · 走已有 user_inbox 通道(与 schedule_stuck 同一套),不新造通知面。
14
+ import { db } from "./db.js";
15
+ import { sendAgentNotice } from "./agent-notice.js";
16
+
17
+ export const NODE_FAILURE_ALERT_KIND = "node_failure_alert";
18
+ /** 连续多少次确定性失败才告警。 */
19
+ export const ALERT_STREAK_THRESHOLD = 5;
20
+ /** 只看这么久以内的任务(更早的失败已经不可操作)。 */
21
+ export const SCAN_WINDOW_HOURS = 24;
22
+
23
+ type Category = "missing_env" | "binary_missing" | "login_required";
24
+
25
+ const PATTERNS: Array<{ category: Category; re: RegExp; label: string; hint: string }> = [
26
+ { category: "missing_env", re: /Missing environment variable/i, label: "缺环境变量",
27
+ hint: "在节点所在机器补上该变量后重启节点(anet node start <alias>)" },
28
+ { category: "binary_missing", re: /ENOENT|No such file or directory/i, label: "二进制/文件缺失",
29
+ hint: "检查该节点的运行时是否安装完整,必要时重装后重启节点" },
30
+ { category: "login_required", re: /login required|Unauthorized|\b401\b/i, label: "凭据/登录失效",
31
+ hint: "重新登录该节点(anet node codex login-status 可先自检)" },
32
+ ];
33
+
34
+ /** 只认确定性错误;timeout / 任务内容错误一律返回 null(那类会自愈或与节点无关)。 */
35
+ export function classifyDeterministicFailure(result: string | null | undefined): Category | null {
36
+ if (!result) return null;
37
+ for (const p of PATTERNS) if (p.re.test(result)) return p.category;
38
+ return null;
39
+ }
40
+
41
+ export type AlertScanResult = { alerts: number; nodes: string[] };
42
+
43
+ /**
44
+ * 扫一遍最近的失败,给「连续确定性失败」的节点发一次性告警。
45
+ * 纯读 + 发通知;不改任何任务/排程状态。
46
+ */
47
+ export function scanNodeFailureAlerts(now = new Date()): AlertScanResult {
48
+ const since = new Date(now.getTime() - SCAN_WINDOW_HOURS * 3_600_000).toISOString().slice(0, 19).replace("T", " ");
49
+ const rows = db.all<{ to_name: string; network_id: string | null; created_at: string; status: string; result: string | null }>(
50
+ `SELECT to_name, network_id, created_at, status, result FROM tasks
51
+ WHERE created_at >= ?1 AND to_name IS NOT NULL
52
+ ORDER BY to_name, created_at`,
53
+ since,
54
+ );
55
+ // 按节点切段:streak 只数「连续」——中间出现任何非该类失败的成功/失败都要断开。
56
+ const streaks = new Map<string, { category: Category; count: number; firstAt: string; networkId: string | null; sample: string }>();
57
+ const cur = new Map<string, { category: Category; count: number; firstAt: string }>();
58
+ for (const r of rows) {
59
+ const cat = r.status === "failed" ? classifyDeterministicFailure(r.result) : null;
60
+ const node = r.to_name;
61
+ const prev = cur.get(node);
62
+ if (!cat) {
63
+ // 序列断了:连续计数和「已完成段落」都要清,否则一条成功之后旧段落仍会被告警。
64
+ cur.delete(node);
65
+ streaks.delete(node);
66
+ continue;
67
+ }
68
+ if (prev && prev.category === cat) {
69
+ prev.count += 1;
70
+ streaks.set(node, { category: cat, count: prev.count, firstAt: prev.firstAt, networkId: r.network_id, sample: String(r.result ?? "").slice(0, 200) });
71
+ } else {
72
+ cur.set(node, { category: cat, count: 1, firstAt: r.created_at });
73
+ streaks.delete(node);
74
+ }
75
+ }
76
+ const out: string[] = [];
77
+ for (const [node, s] of streaks) {
78
+ if (s.count < ALERT_STREAK_THRESHOLD) continue;
79
+ const meta = { node, category: s.category, first_seen: s.firstAt };
80
+ const dup = db.get<{ n: number }>(
81
+ `SELECT COUNT(*) AS n FROM user_inbox WHERE kind = ?1 AND from_session = ?2
82
+ AND json_extract(meta_json, '$.node_failure_alert.category') = ?3
83
+ AND json_extract(meta_json, '$.node_failure_alert.first_seen') = ?4`,
84
+ NODE_FAILURE_ALERT_KIND, node, s.category, s.firstAt,
85
+ )?.n ?? 0;
86
+ if (dup > 0) continue;
87
+ const owner = db.get<{ owner_user_id: string | null }>("SELECT owner_user_id FROM nodes WHERE alias = ?1 AND network_id = ?2", node, s.networkId)?.owner_user_id;
88
+ if (!owner || !s.networkId) continue;
89
+ const label = PATTERNS.find((p) => p.category === s.category)!;
90
+ const sent = sendAgentNotice({
91
+ networkId: s.networkId,
92
+ userId: owner,
93
+ fromAlias: node,
94
+ kind: NODE_FAILURE_ALERT_KIND,
95
+ title: `节点连续失败:${label.label}`,
96
+ text: `${node} 连续 ${s.count} 次以「${label.label}」失败(自 ${s.firstAt} 起)。\n\n最近一次回执节选:\n${s.sample}\n\n处理建议:${label.hint}`,
97
+ severity: "error",
98
+ meta: { node_failure_alert: meta },
99
+ idPrefix: "dm_nfa_",
100
+ });
101
+ if (sent) out.push(node);
102
+ }
103
+ return { alerts: out.length, nodes: out };
104
+ }
@@ -0,0 +1,25 @@
1
+ // #1879 —— 失败分类:共享会话忙与真失败分开。
2
+ import { describe, expect, test } from "bun:test";
3
+ import { scheduledFailureCode } from "./scheduled-failure-code.js";
4
+
5
+ describe("scheduledFailureCode", () => {
6
+ test("共享会话忙 → task_failed_busy(两种情况都覆盖)", () => {
7
+ const notSubmitted = "[TMA云盘欧] opencode 任务未提交:共享会话在 30 分钟 内一直处于忙碌状态(可能有人正在 TUI 里跑一轮),本任务没有发出";
8
+ expect(scheduledFailureCode("failed", notSubmitted)).toBe("task_failed_busy");
9
+ const stillRunning = "[通信客户端欧] ⏳ opencode 任务仍在节点的 TUI 会话里运行,没有被中止;bridge 等待回复已达 30 分钟";
10
+ expect(scheduledFailureCode("failed", stillRunning)).toBe("task_failed_busy");
11
+ });
12
+
13
+ test("真失败仍然是 task_failed(不能被分类吞掉)", () => {
14
+ expect(scheduledFailureCode("failed", "runtime failed")).toBe("task_failed");
15
+ expect(scheduledFailureCode("failed", null)).toBe("task_failed");
16
+ // 提到「共享会话」但没有任何忙碌措辞 ⇒ 仍然算真失败,不能被分类吞掉
17
+ expect(scheduledFailureCode("failed", "共享会话已释放,任务失败原因不明")).toBe("task_failed");
18
+ });
19
+
20
+ test("非 failed 状态不受影响", () => {
21
+ expect(scheduledFailureCode("replied", "共享会话忙")).toBeNull();
22
+ expect(scheduledFailureCode("cancelled", "共享会话一直处于忙碌")).toBe("task_cancelled");
23
+ expect(scheduledFailureCode("expired", null)).toBe("task_expired");
24
+ });
25
+ });
@@ -0,0 +1,20 @@
1
+ // #1879 —— 定时任务失败分类(纯函数,不 import db,便于单测)。
2
+ //
3
+ // 生产 24h(2026-10-28):144 条 failed run 的 error_code 全是 `task_failed`,
4
+ // 但其中 125 条(66%)的真实原因是「共享会话被人的 TUI 占着,任务 30 分钟内没提交上去」。
5
+ // 混在一个桶里,报表分不出来,也没法给它们配不同策略。
6
+ //
7
+ // 只做分类,**不改等待语义**:近 24h 有 108 个任务是等到 >25 分钟才成功的,
8
+ // 「忙就立刻失败」会打掉它们。
9
+ //
10
+ // `failureStreak()` 按 status='failed' 计数(不是按 error_code),所以新增一个 code
11
+ // 不影响「连续 N 次失败告警 / 自动暂停」的阈值。
12
+ export function scheduledFailureCode(status: string, result: string | null): string | null {
13
+ if (status === "replied") return null;
14
+ const base = `task_${status}`;
15
+ if (status !== "failed") return base;
16
+ const text = result ?? "";
17
+ if (/共享会话/.test(text) && /忙碌|一直处于|未获提交/.test(text)) return "task_failed_busy";
18
+ if (/仍在节点的 TUI 会话里运行|等待回复已达\s*30\s*分钟/.test(text)) return "task_failed_busy";
19
+ return base;
20
+ }
package/src/server.ts CHANGED
@@ -84,6 +84,7 @@ import { startRetentionSweeper } from "./retention.js";
84
84
  import { sweepEmptyNodeRows } from "./empty-node-sweeper.js";
85
85
  import { versionStats } from "./version-stats.js";
86
86
  import { startStaleSessionSweeper } from "./stale-sweeper.js";
87
+ import { scanNodeFailureAlerts } from "./node-failure-alert.js";
87
88
  import {
88
89
  AUDIT_LOG_REST_SELECT,
89
90
  COMPLETION_REST_SELECT,
@@ -5784,6 +5785,21 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5784
5785
  // every COMMHUB_STALE_SWEEP_SECONDS (default 60s).
5785
5786
  const staleSweeperTimer = startStaleSessionSweeper();
5786
5787
 
5788
+ // #2840 —— 节点「假活」告警:连续 5 次确定性失败(缺环境变量 / 二进制缺失 / 登录失效)
5789
+ // 给节点主人发一条一次性通知。校准见 node-failure-alert.ts 头注释:
5790
+ // 只算确定性错误时 7 天 6 条(0.9/天);把 timeout 也算进去会变成 35 条。
5791
+ const nodeFailureAlertMs = Number(process.env.COMMHUB_NODE_FAILURE_ALERT_MS) > 0
5792
+ ? Number(process.env.COMMHUB_NODE_FAILURE_ALERT_MS) : 5 * 60 * 1000;
5793
+ const nodeFailureAlertTimer = setInterval(() => {
5794
+ try {
5795
+ const r = scanNodeFailureAlerts();
5796
+ if (r.alerts > 0) console.log(`[commhub node-failure-alert] sent ${r.alerts} alert(s): ${r.nodes.join(", ")}`);
5797
+ } catch (e: any) {
5798
+ console.log(`[commhub node-failure-alert] failed: ${e?.message ?? e}`);
5799
+ }
5800
+ }, nodeFailureAlertMs);
5801
+ (nodeFailureAlertTimer as any).unref?.();
5802
+
5787
5803
  // #476 — the two legacy module-level intervals, now owned by startHub
5788
5804
  // like every other periodic job: rate-limit map cleanup (memory only)
5789
5805
  // and task expiration patrol (writes DB — must never run on import).
@@ -5825,6 +5841,7 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5825
5841
  clearInterval(aliasBaselineTimer);
5826
5842
  clearTimeout(deferredBackfillTimer);
5827
5843
  clearInterval(staleSweeperTimer);
5844
+ clearInterval(nodeFailureAlertTimer);
5828
5845
  clearInterval(rateLimitSweepTimer);
5829
5846
  clearInterval(taskPatrolTimer);
5830
5847
  clearInterval(deliveredStalePatrolTimer);