@sleep2agi/commhub-server 0.9.0-preview.196 → 0.9.0-preview.198
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/node-failure-alert.test.ts +60 -1
- package/src/node-failure-alert.ts +62 -1
- package/src/server.ts +8 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sleep2agi/commhub-server",
|
|
3
|
-
"version": "0.9.0-preview.
|
|
3
|
+
"version": "0.9.0-preview.198",
|
|
4
4
|
"description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -6,7 +6,7 @@ import { join } from "path";
|
|
|
6
6
|
import { tmpdir } from "os";
|
|
7
7
|
import { register } from "./auth.js";
|
|
8
8
|
import { db } from "./db.js";
|
|
9
|
-
import { scanNodeFailureAlerts, classifyDeterministicFailure, NODE_FAILURE_ALERT_KIND } from "./node-failure-alert.js";
|
|
9
|
+
import { scanNodeFailureAlerts, scanModelAuthBlockedNodes, classifyDeterministicFailure, NODE_FAILURE_ALERT_KIND, MODEL_AUTH_BLOCKED_CATEGORY } from "./node-failure-alert.js";
|
|
10
10
|
|
|
11
11
|
const DIR = mkdtempSync(join(tmpdir(), "anet-nfa-"));
|
|
12
12
|
let NET = "", UID = "", NODE = "";
|
|
@@ -40,6 +40,14 @@ describe("#2840 节点连续确定性失败告警", () => {
|
|
|
40
40
|
expect(classifyDeterministicFailure("codex-app-server 错误: Missing environment variable: `DEEPSEEK_KEY`")).toBe("missing_env");
|
|
41
41
|
expect(classifyDeterministicFailure("spawn ENOENT")).toBe("binary_missing");
|
|
42
42
|
expect(classifyDeterministicFailure("401 Unauthorized")).toBe("login_required");
|
|
43
|
+
// #2964 —— 这是生产里真实出现的那一句(agent-node 的 codex-health 生成),
|
|
44
|
+
// 原来的正则在它上面返回 null ⇒ 凭据过期的节点一条告警都发不出去。
|
|
45
|
+
expect(classifyDeterministicFailure(
|
|
46
|
+
"codex-app-server 错误: model login expired: this node's CODEX_HOME (/home/user/.anet/nodes/n/codex-home) needs a fresh login — log in again there: CODEX_HOME=/x codex login. Not taking work until then (do not copy another node's auth.json).",
|
|
47
|
+
)).toBe("login_required");
|
|
48
|
+
expect(classifyDeterministicFailure("model login revoked: this node's CODEX_HOME needs a fresh login")).toBe("login_required");
|
|
49
|
+
// 反例:非登录类的失败不能被吸进来
|
|
50
|
+
expect(classifyDeterministicFailure("model login ok")).toBeNull();
|
|
43
51
|
expect(classifyDeterministicFailure("codex request 'turn/start' timed out")).toBeNull();
|
|
44
52
|
// #2891 —— 额度耗尽单列一类;限流(会自愈)**不能**混进来
|
|
45
53
|
expect(classifyDeterministicFailure("opencode 错误: 模型调用失败(provider.quota: The usage limit has been reached)")).toBe("quota_exhausted");
|
|
@@ -89,3 +97,54 @@ describe("#2840 节点连续确定性失败告警", () => {
|
|
|
89
97
|
expect(r.nodes).not.toContain(reset);
|
|
90
98
|
});
|
|
91
99
|
});
|
|
100
|
+
|
|
101
|
+
// #2964 后半 —— 节点「在线但干不了活」:状态是 error、原因是登录失效,
|
|
102
|
+
// 但它一条任务都不接,所以「连续失败」那条告警路径永远不会触发。
|
|
103
|
+
describe("#2964 登录失效节点(不接活)", () => {
|
|
104
|
+
const session = (alias: string, status: string, task: string | null) => {
|
|
105
|
+
db.run(
|
|
106
|
+
`INSERT OR REPLACE INTO sessions (resume_id, alias, status, task, network_id, updated_at)
|
|
107
|
+
VALUES (?1, ?2, ?3, ?4, ?5, datetime('now'))`,
|
|
108
|
+
[`s_${alias}`, alias, status, task, NET],
|
|
109
|
+
);
|
|
110
|
+
db.run(
|
|
111
|
+
`INSERT OR IGNORE INTO nodes (node_id, node_name, alias, runtime, network_id, owner_user_id)
|
|
112
|
+
VALUES (?1, ?2, ?2, 'codex', ?3, ?4)`,
|
|
113
|
+
[`n_${alias}`, alias, NET, UID],
|
|
114
|
+
);
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
test("status=error 且原因是登录失效 → 告警一次;同一天再扫不重复", () => {
|
|
118
|
+
const alias = `nfa-auth-${Date.now()}`;
|
|
119
|
+
session(alias, "error", "model login expired: this node's CODEX_HOME needs a fresh login — log in again there");
|
|
120
|
+
const before = alerts().length;
|
|
121
|
+
expect(scanModelAuthBlockedNodes().nodes).toContain(alias);
|
|
122
|
+
expect(alerts().length).toBe(before + 1);
|
|
123
|
+
// 每节点每天一条:再扫一次不应新增
|
|
124
|
+
expect(scanModelAuthBlockedNodes().alerts).toBe(0);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test("idle 的节点不告警(没坏就别吵)", () => {
|
|
128
|
+
const alias = `nfa-auth-idle-${Date.now()}`;
|
|
129
|
+
session(alias, "idle", "model login expired: needs a fresh login");
|
|
130
|
+
expect(scanModelAuthBlockedNodes().nodes).not.toContain(alias);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test("error 但原因不是登录类(例如代码异常)不告警", () => {
|
|
134
|
+
const alias = `nfa-auth-other-${Date.now()}`;
|
|
135
|
+
session(alias, "error", "TypeError: cannot read properties of undefined (reading 'x')");
|
|
136
|
+
expect(scanModelAuthBlockedNodes().nodes).not.toContain(alias);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
test("告警里带类别与当天日期(去重键)", () => {
|
|
140
|
+
const row = db.get<{ cat: string; day: string }>(
|
|
141
|
+
`SELECT json_extract(meta_json,'$.node_failure_alert.category') AS cat,
|
|
142
|
+
json_extract(meta_json,'$.node_failure_alert.first_seen') AS day
|
|
143
|
+
FROM user_inbox WHERE kind = ?1
|
|
144
|
+
AND json_extract(meta_json,'$.node_failure_alert.category') = ?2 LIMIT 1`,
|
|
145
|
+
NODE_FAILURE_ALERT_KIND, MODEL_AUTH_BLOCKED_CATEGORY,
|
|
146
|
+
);
|
|
147
|
+
expect(row?.cat).toBe(MODEL_AUTH_BLOCKED_CATEGORY);
|
|
148
|
+
expect(row?.day).toMatch(/^\d{4}-\d{2}-\d{2}$/);
|
|
149
|
+
});
|
|
150
|
+
});
|
|
@@ -27,7 +27,13 @@ const PATTERNS: Array<{ category: Category; re: RegExp; label: string; hint: str
|
|
|
27
27
|
hint: "在节点所在机器补上该变量后重启节点(anet node start <alias>)" },
|
|
28
28
|
{ category: "binary_missing", re: /ENOENT|No such file or directory/i, label: "二进制/文件缺失",
|
|
29
29
|
hint: "检查该节点的运行时是否安装完整,必要时重装后重启节点" },
|
|
30
|
-
|
|
30
|
+
// #2964 —— 真实的登录失效文案长这样(agent-node/src/runtime/codex-health.ts 生成):
|
|
31
|
+
// "model login expired: this node's CODEX_HOME (/…/codex-home) needs a fresh login — log in again there: …"
|
|
32
|
+
// 原来的 /login required|Unauthorized|\b401\b/ **匹配不到这一句**:它写的是
|
|
33
|
+
// "needs a fresh login",不是 "login required"。实测 2026-10-29 一个凭据过期的节点
|
|
34
|
+
// 启动后每轮都以这句话失败,而 #2840 的告警一条都不会发 —— 分类器看不见它。
|
|
35
|
+
// 补两种状态词(expired / revoked)与那句固定结尾,仍然只认登录态,不扩大到泛化的 auth 字样。
|
|
36
|
+
{ category: "login_required", re: /login required|Unauthorized|\b401\b|model login (?:expired|revoked)|needs a fresh login/i, label: "凭据/登录失效",
|
|
31
37
|
hint: "重新登录该节点(anet node codex login-status 可先自检)" },
|
|
32
38
|
// #2891 —— 额度耗尽:与前三类同性质(不会自愈、必须人处理),但**必须与「限流」分开**。
|
|
33
39
|
//
|
|
@@ -43,6 +49,61 @@ const PATTERNS: Array<{ category: Category; re: RegExp; label: string; hint: str
|
|
|
43
49
|
hint: "该节点的上游额度已用完:换 key / 充值 / 换 provider 后重启节点;限流(rate limit)不属此类,会自愈" },
|
|
44
50
|
];
|
|
45
51
|
|
|
52
|
+
/** #2964 后半 —— 节点完全不接活时的告警类别(与「连续失败」区分开)。 */
|
|
53
|
+
export const MODEL_AUTH_BLOCKED_CATEGORY = "login_expired";
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* #2964 后半 —— 扫「在线但干不了活」的节点。
|
|
57
|
+
*
|
|
58
|
+
* 前一半(分类器认得出登录文案)只覆盖「节点产生了失败」的情况。
|
|
59
|
+
* 还有一种更安静的状态:节点启动后登录已失效,**一条任务都不接**,
|
|
60
|
+
* 于是 `#2840` 的「连续失败」永远不会触发 —— 但它同样干不了活。
|
|
61
|
+
*
|
|
62
|
+
* hub 侧的事实来源:agent-node 的 `gateStatusOnModelAuth` 会把这种节点的状态报成
|
|
63
|
+
* `error`,并把原因写进 `sessions.task`。这里直接读它,不做任何推断。
|
|
64
|
+
*
|
|
65
|
+
* 去重粒度:每节点每天最多一条(`first_seen` 用日期),因为这不是「某一次失败」,
|
|
66
|
+
* 而是一个需要人处理的状态;每隔 5 分钟提醒一次只会变成噪音。
|
|
67
|
+
*/
|
|
68
|
+
export function scanModelAuthBlockedNodes(now = new Date()): AlertScanResult {
|
|
69
|
+
const day = now.toISOString().slice(0, 10);
|
|
70
|
+
const rows = db.all<{ alias: string | null; network_id: string | null; task: string | null }>(
|
|
71
|
+
`SELECT alias, network_id, task FROM sessions WHERE status = 'error' AND task IS NOT NULL`,
|
|
72
|
+
);
|
|
73
|
+
const out: string[] = [];
|
|
74
|
+
for (const r of rows) {
|
|
75
|
+
const node = r.alias;
|
|
76
|
+
if (!node || !r.network_id) continue;
|
|
77
|
+
if (classifyDeterministicFailure(r.task) !== "login_required") continue;
|
|
78
|
+
const dup = db.get<{ n: number }>(
|
|
79
|
+
`SELECT COUNT(*) AS n FROM user_inbox WHERE kind = ?1 AND from_session = ?2
|
|
80
|
+
AND json_extract(meta_json, '$.node_failure_alert.category') = ?3
|
|
81
|
+
AND json_extract(meta_json, '$.node_failure_alert.first_seen') = ?4`,
|
|
82
|
+
NODE_FAILURE_ALERT_KIND, node, MODEL_AUTH_BLOCKED_CATEGORY, day,
|
|
83
|
+
)?.n ?? 0;
|
|
84
|
+
if (dup > 0) continue;
|
|
85
|
+
const owner = db.get<{ owner_user_id: string | null }>(
|
|
86
|
+
"SELECT owner_user_id FROM nodes WHERE alias = ?1 AND network_id = ?2", node, r.network_id,
|
|
87
|
+
)?.owner_user_id;
|
|
88
|
+
if (!owner) continue;
|
|
89
|
+
const sent = sendAgentNotice({
|
|
90
|
+
networkId: r.network_id,
|
|
91
|
+
userId: owner,
|
|
92
|
+
fromAlias: node,
|
|
93
|
+
kind: NODE_FAILURE_ALERT_KIND,
|
|
94
|
+
title: "节点在线但不会接活:登录失效",
|
|
95
|
+
text: `${node} 当前状态是 error,原因是登录失效。它不会接任何任务(不是慢,是干不了)。\n\n`
|
|
96
|
+
+ `节点自述:\n${String(r.task).slice(0, 300)}\n\n`
|
|
97
|
+
+ `处理建议:在该节点自己的 CODEX_HOME 里重新登录(不要拷贝别的节点的 auth.json),然后重启节点。`,
|
|
98
|
+
severity: "error",
|
|
99
|
+
meta: { node_failure_alert: { node, category: MODEL_AUTH_BLOCKED_CATEGORY, first_seen: day } },
|
|
100
|
+
idPrefix: "dm_nfa_",
|
|
101
|
+
});
|
|
102
|
+
if (sent) out.push(node);
|
|
103
|
+
}
|
|
104
|
+
return { alerts: out.length, nodes: out };
|
|
105
|
+
}
|
|
106
|
+
|
|
46
107
|
/** 只认确定性错误;timeout / 任务内容错误一律返回 null(那类会自愈或与节点无关)。 */
|
|
47
108
|
export function classifyDeterministicFailure(result: string | null | undefined): Category | null {
|
|
48
109
|
if (!result) return null;
|
package/src/server.ts
CHANGED
|
@@ -85,7 +85,7 @@ import { startRetentionSweeper } from "./retention.js";
|
|
|
85
85
|
import { sweepEmptyNodeRows } from "./empty-node-sweeper.js";
|
|
86
86
|
import { versionStats } from "./version-stats.js";
|
|
87
87
|
import { startStaleSessionSweeper } from "./stale-sweeper.js";
|
|
88
|
-
import { scanNodeFailureAlerts } from "./node-failure-alert.js";
|
|
88
|
+
import { scanNodeFailureAlerts, scanModelAuthBlockedNodes } from "./node-failure-alert.js";
|
|
89
89
|
import { scanStarvedSchedules } from "./schedule-starved.js";
|
|
90
90
|
import { scanNodeBusyCircuit } from "./node-serviceability.js";
|
|
91
91
|
import {
|
|
@@ -5823,6 +5823,13 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
|
|
|
5823
5823
|
} catch (e: any) {
|
|
5824
5824
|
console.log(`[commhub node-failure-alert] failed: ${e?.message ?? e}`);
|
|
5825
5825
|
}
|
|
5826
|
+
// #2964 —— 「在线但干不了活」:登录失效的节点一条任务都不会接,连续失败那条路不会触发。
|
|
5827
|
+
try {
|
|
5828
|
+
const b = scanModelAuthBlockedNodes();
|
|
5829
|
+
if (b.alerts > 0) console.log(`[commhub node-auth-blocked-alert] sent ${b.alerts} alert(s): ${b.nodes.join(", ")}`);
|
|
5830
|
+
} catch (e: any) {
|
|
5831
|
+
console.log(`[commhub node-auth-blocked-alert] failed: ${e?.message ?? e}`);
|
|
5832
|
+
}
|
|
5826
5833
|
}, nodeFailureAlertMs);
|
|
5827
5834
|
(nodeFailureAlertTimer as any).unref?.();
|
|
5828
5835
|
|