@sleep2agi/commhub-server 0.9.0-preview.191 → 0.9.0-preview.193
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/node-failure-alert.test.ts +11 -0
- package/src/node-failure-alert.ts +13 -1
- package/src/node-serviceability.test.ts +19 -0
- package/src/node-serviceability.ts +29 -1
- package/src/rest-explicit-columns-http.test.ts +44 -0
- package/src/scheduled-failure-code.test.ts +49 -1
- package/src/scheduled-failure-code.ts +85 -0
- package/src/scheduled-failures-http.test.ts +46 -0
- package/src/scheduled-failures.ts +16 -3
- package/src/scheduled-tasks.ts +30 -2
- package/src/server.ts +6 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sleep2agi/commhub-server",
|
|
3
|
-
"version": "0.9.0-preview.
|
|
3
|
+
"version": "0.9.0-preview.193",
|
|
4
4
|
"description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -41,6 +41,17 @@ describe("#2840 节点连续确定性失败告警", () => {
|
|
|
41
41
|
expect(classifyDeterministicFailure("spawn ENOENT")).toBe("binary_missing");
|
|
42
42
|
expect(classifyDeterministicFailure("401 Unauthorized")).toBe("login_required");
|
|
43
43
|
expect(classifyDeterministicFailure("codex request 'turn/start' timed out")).toBeNull();
|
|
44
|
+
// #2891 —— 额度耗尽单列一类;限流(会自愈)**不能**混进来
|
|
45
|
+
expect(classifyDeterministicFailure("opencode 错误: 模型调用失败(provider.quota: The usage limit has been reached)")).toBe("quota_exhausted");
|
|
46
|
+
expect(classifyDeterministicFailure("provider.quota: Free usage limit reached.")).toBe("quota_exhausted");
|
|
47
|
+
// 限流不是额度:等几秒就好,不该惊动人
|
|
48
|
+
expect(classifyDeterministicFailure("429 Too Many Requests (rate limit)")).toBeNull();
|
|
49
|
+
// 任务正文里**提到**额度(不是错误原文)→ 不能命中
|
|
50
|
+
expect(classifyDeterministicFailure("本轮汇报:我把 provider.quota 的含义写进了 runbook")).toBeNull();
|
|
51
|
+
// 已知限制:正文**逐字引用**错误原文(含冒号)时仍会命中。
|
|
52
|
+
// 生产 7 天语料里这样的假阳性 1/507,且告警要「连续 5 次」才发,故不改判据;
|
|
53
|
+
// 这里把它钉成测试,避免以后有人误以为这是完备的语义判断。
|
|
54
|
+
expect(classifyDeterministicFailure("引用:provider.quota: The usage limit has been reached")).toBe("quota_exhausted");
|
|
44
55
|
expect(classifyDeterministicFailure("任务内容写错了")).toBeNull();
|
|
45
56
|
expect(classifyDeterministicFailure(null)).toBeNull();
|
|
46
57
|
});
|
|
@@ -20,7 +20,7 @@ export const ALERT_STREAK_THRESHOLD = 5;
|
|
|
20
20
|
/** 只看这么久以内的任务(更早的失败已经不可操作)。 */
|
|
21
21
|
export const SCAN_WINDOW_HOURS = 24;
|
|
22
22
|
|
|
23
|
-
type Category = "missing_env" | "binary_missing" | "login_required";
|
|
23
|
+
type Category = "missing_env" | "binary_missing" | "login_required" | "quota_exhausted";
|
|
24
24
|
|
|
25
25
|
const PATTERNS: Array<{ category: Category; re: RegExp; label: string; hint: string }> = [
|
|
26
26
|
{ category: "missing_env", re: /Missing environment variable/i, label: "缺环境变量",
|
|
@@ -29,6 +29,18 @@ const PATTERNS: Array<{ category: Category; re: RegExp; label: string; hint: str
|
|
|
29
29
|
hint: "检查该节点的运行时是否安装完整,必要时重装后重启节点" },
|
|
30
30
|
{ category: "login_required", re: /login required|Unauthorized|\b401\b/i, label: "凭据/登录失效",
|
|
31
31
|
hint: "重新登录该节点(anet node codex login-status 可先自检)" },
|
|
32
|
+
// #2891 —— 额度耗尽:与前三类同性质(不会自愈、必须人处理),但**必须与「限流」分开**。
|
|
33
|
+
//
|
|
34
|
+
// 校准(2026-10-28,近 7 天 2825 条失败):
|
|
35
|
+
// provider.quota / usage limit has been reached → 507 条(占全部失败 18%)
|
|
36
|
+
// 限流类文案(rate limit / 429 / too many requests) → 2 条
|
|
37
|
+
// ⇒ 宽泛匹配「额度|limit|quota」会到 513,多出来的 6 条是**任务正文在讨论额度**(假阳性),
|
|
38
|
+
// 所以这里只认 provider.quota / usage limit has been reached 这两句原文。
|
|
39
|
+
//
|
|
40
|
+
// 为什么不能把限流并进来:限流会自愈(等几秒重试即可),额度不会。
|
|
41
|
+
// 混成一类会让告警失去「必须人处理」这个唯一有用的性质。
|
|
42
|
+
{ category: "quota_exhausted", re: /provider\.quota:\s|usage limit has been reached/i, label: "模型额度耗尽",
|
|
43
|
+
hint: "该节点的上游额度已用完:换 key / 充值 / 换 provider 后重启节点;限流(rate limit)不属此类,会自愈" },
|
|
32
44
|
];
|
|
33
45
|
|
|
34
46
|
/** 只认确定性错误;timeout / 任务内容错误一律返回 null(那类会自愈或与节点无关)。 */
|
|
@@ -50,6 +50,25 @@ describe("节点忙熔断:三项判据", () => {
|
|
|
50
50
|
expect(r.enabled).toBe(false); // 默认关闭
|
|
51
51
|
});
|
|
52
52
|
|
|
53
|
+
// 熔断后不能「永不投递」:否则节点拿不到成功记录 ⇒ 永远解不开(死锁)。
|
|
54
|
+
test("熔断期间按探针间隔放行一条(自愈通道)", () => {
|
|
55
|
+
process.env.COMMHUB_NODE_BUSY_CIRCUIT = "1";
|
|
56
|
+
// 黑洞节点:刚投递过 ⇒ 拦
|
|
57
|
+
db.run(`INSERT INTO tasks (task_id, content, from_name, to_name, network_id, status, created_at)
|
|
58
|
+
VALUES ('t_probe_now','x','hub','黑洞节点',?1,'failed',datetime('now'))`, [NET]);
|
|
59
|
+
const blocked = nodeCanAcceptDispatch("黑洞节点", NET);
|
|
60
|
+
expect(blocked.ok).toBe(false);
|
|
61
|
+
expect(blocked.probe).toBeUndefined();
|
|
62
|
+
// 把最近投递推到 2 小时前(默认探针间隔 1h)⇒ 放行,且标记 probe
|
|
63
|
+
// 注意:前面的用例也给这个节点插过 created_at=now 的任务,MAX(created_at) 会取到它们
|
|
64
|
+
db.run("UPDATE tasks SET created_at = datetime('now','-2 hours') WHERE to_name = '黑洞节点'");
|
|
65
|
+
const probe = nodeCanAcceptDispatch("黑洞节点", NET);
|
|
66
|
+
expect(probe.ok).toBe(true);
|
|
67
|
+
expect(probe.probe).toBe(true);
|
|
68
|
+
// 还原,避免影响后续用例(它们依赖「最近刚投递过」这个前态)
|
|
69
|
+
db.run("UPDATE tasks SET created_at = datetime('now') WHERE to_name = '黑洞节点'");
|
|
70
|
+
});
|
|
71
|
+
|
|
53
72
|
test("默认关闭时投递判断恒放行(不改任何既有行为)", () => {
|
|
54
73
|
delete process.env.COMMHUB_NODE_BUSY_CIRCUIT;
|
|
55
74
|
expect(circuitEnabled()).toBe(false);
|
|
@@ -31,6 +31,28 @@ export function circuitEnabled(): boolean {
|
|
|
31
31
|
}
|
|
32
32
|
export function circuitWindowHours(): number { return envInt("COMMHUB_NODE_BUSY_CIRCUIT_HOURS", 6, 1); }
|
|
33
33
|
export function circuitMinFails(): number { return envInt("COMMHUB_NODE_BUSY_CIRCUIT_FAILS", 5, 1); }
|
|
34
|
+
/**
|
|
35
|
+
* 熔断期间**放探针**的间隔(小时)。
|
|
36
|
+
*
|
|
37
|
+
* 🔴 为什么必须有:熔断后完全不再投递 ⇒ 节点永远拿不到成功记录 ⇒ 永远解不开(死锁)。
|
|
38
|
+
* 所以每 N 小时放一条探针任务:它若成功,replied 计数不再为 0,熔断自动解除。
|
|
39
|
+
*/
|
|
40
|
+
export function circuitProbeHours(): number { return envInt("COMMHUB_NODE_BUSY_CIRCUIT_PROBE_HOURS", 1, 1); }
|
|
41
|
+
|
|
42
|
+
/** 该节点最近一次**被投递**的时刻(ms)。没有投递过返回 null。 */
|
|
43
|
+
function lastDeliveryMs(alias: string, networkId: string | null): number | null {
|
|
44
|
+
try {
|
|
45
|
+
const row = db.get<{ t: string | null }>(
|
|
46
|
+
"SELECT MAX(created_at) AS t FROM tasks WHERE to_name = ?1 AND (?2 IS NULL OR network_id = ?2)",
|
|
47
|
+
alias, networkId,
|
|
48
|
+
);
|
|
49
|
+
if (!row?.t) return null;
|
|
50
|
+
const ms = Date.parse(String(row.t).replace(" ", "T") + "Z");
|
|
51
|
+
return Number.isFinite(ms) ? ms : null;
|
|
52
|
+
} catch {
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
34
56
|
|
|
35
57
|
export type NodeServingStats = { alias: string; network_id: string | null; delivered: number; replied: number; busy_failed: number };
|
|
36
58
|
|
|
@@ -69,7 +91,7 @@ export function scanNodeBusyCircuit(now = new Date()): CircuitScanResult {
|
|
|
69
91
|
* 这个节点现在能不能投递。默认永远 true(未开启 = 不拦截)。
|
|
70
92
|
* 供排程派发判断;纯读,不改任何状态。
|
|
71
93
|
*/
|
|
72
|
-
export function nodeCanAcceptDispatch(alias: string, networkId: string | null): { ok: boolean; reason?: string; busy_failed?: number } {
|
|
94
|
+
export function nodeCanAcceptDispatch(alias: string, networkId: string | null): { ok: boolean; reason?: string; busy_failed?: number; probe?: boolean } {
|
|
73
95
|
if (!circuitEnabled()) return { ok: true };
|
|
74
96
|
const windowHours = circuitWindowHours();
|
|
75
97
|
const since = new Date(Date.now() - windowHours * 3_600_000).toISOString().slice(0, 19).replace("T", " ");
|
|
@@ -88,6 +110,12 @@ export function nodeCanAcceptDispatch(alias: string, networkId: string | null):
|
|
|
88
110
|
if (!row) return { ok: true };
|
|
89
111
|
const stats = { delivered: Number(row.delivered ?? 0), replied: Number(row.replied ?? 0), busy_failed: Number(row.busy_failed ?? 0) };
|
|
90
112
|
if (!isNotServing(stats)) return { ok: true };
|
|
113
|
+
// 熔断期间每 N 小时放一条探针,否则节点永远拿不到成功记录、永远解不开。
|
|
114
|
+
const probeHours = circuitProbeHours();
|
|
115
|
+
const last = lastDeliveryMs(alias, networkId);
|
|
116
|
+
if (last === null || Date.now() - last >= probeHours * 3_600_000) {
|
|
117
|
+
return { ok: true, probe: true, busy_failed: stats.busy_failed };
|
|
118
|
+
}
|
|
91
119
|
return {
|
|
92
120
|
ok: false,
|
|
93
121
|
reason: `节点近 ${windowHours}h 内 ${stats.busy_failed} 次忙类失败、0 次成功(投递也只会白等),已跳过`,
|
|
@@ -197,6 +197,50 @@ describe("#311 REST response projections are stable across future ALTER TABLE",
|
|
|
197
197
|
expect(detail.task[INTERNAL_SENTINEL]).toBeUndefined();
|
|
198
198
|
});
|
|
199
199
|
|
|
200
|
+
// #2884 —— 共享会话忙的失败:任务详情/列表要把「排队」与「会话等待」分开报。
|
|
201
|
+
// 现场那条 412 分钟的任务**不是排程任务**(scheduled_task_runs 里没有它),
|
|
202
|
+
// 所以投影必须落在任务读端点,不能只改排程 run。
|
|
203
|
+
test("#2884 busy failures expose queue vs session-wait split (non-scheduled tasks too)", async () => {
|
|
204
|
+
const BUSY = "[n] opencode 任务未提交:共享会话在 30 分钟 内一直处于忙碌状态,本任务没有发出";
|
|
205
|
+
// 时间用 JS 算再传参:`datetime('now','-N minutes')` 是 SQLite 方言,
|
|
206
|
+
// PG ladder 里会直接报错 (#2884)。completed_at 留空表示「现在」,由断言侧不需要它。
|
|
207
|
+
const minutesAgo = (m: number) => new Date(Date.now() - m * 60_000).toISOString().replace("T", " ").replace(/\..+/, "");
|
|
208
|
+
const nowTs = minutesAgo(0);
|
|
209
|
+
db.run(
|
|
210
|
+
`INSERT INTO tasks (task_id, from_name, to_name, status, content, priority, result,
|
|
211
|
+
created_at, started_at, completed_at, network_id)
|
|
212
|
+
VALUES ('task-busy-shape', 'sender', 'rest-shape-node', 'failed', 'busy', 'normal', ?1, ?2, ?3, ?4, ?5)`,
|
|
213
|
+
[BUSY, minutesAgo(412), minutesAgo(30), nowTs, networkId],
|
|
214
|
+
);
|
|
215
|
+
// 对照组:同一形状但原因不是会话忙 ⇒ 不能被加拆解
|
|
216
|
+
db.run(
|
|
217
|
+
`INSERT INTO tasks (task_id, from_name, to_name, status, content, priority, result,
|
|
218
|
+
created_at, started_at, completed_at, network_id)
|
|
219
|
+
VALUES ('task-plain-fail', 'sender', 'rest-shape-node', 'failed', 'plain', 'normal',
|
|
220
|
+
'执行出错: upstream refused', ?1, ?2, ?3, ?4)`,
|
|
221
|
+
[minutesAgo(400), minutesAgo(370), nowTs, networkId],
|
|
222
|
+
);
|
|
223
|
+
|
|
224
|
+
const detail = await api(`/api/tasks/task-busy-shape?network_id=${networkId}`);
|
|
225
|
+
expect(detail.task.status).toBe("failed");
|
|
226
|
+
expect(detail.task.result).toContain("共享会话");
|
|
227
|
+
expect(detail.task.result).toContain("排队 382 分钟");
|
|
228
|
+
expect(detail.task.result).toContain("会话等待 30 分钟");
|
|
229
|
+
expect(detail.task.result).toContain("不代表总共等了多久");
|
|
230
|
+
// 幂等:再读一次不重复追加
|
|
231
|
+
const again = await api(`/api/tasks/task-busy-shape?network_id=${networkId}`);
|
|
232
|
+
expect(again.task.result.match(/实测总耗时/g)?.length).toBe(1);
|
|
233
|
+
|
|
234
|
+
const plain = await api(`/api/tasks/task-plain-fail?network_id=${networkId}`);
|
|
235
|
+
expect(plain.task.result).not.toContain("排队");
|
|
236
|
+
|
|
237
|
+
// 列表端点走同一条投影
|
|
238
|
+
const list = await api(`/api/tasks?network_id=${networkId}&task_id=task-busy-shape&skip_stats=1`);
|
|
239
|
+
expect(list.tasks[0].result).toContain("排队 382 分钟");
|
|
240
|
+
// 响应键不变(不新增字段)
|
|
241
|
+
expect(sortedKeys(list.tasks[0])).toEqual(sorted(GOLDEN_RESPONSE_KEYS.task));
|
|
242
|
+
});
|
|
243
|
+
|
|
200
244
|
test("audit, task-event, and completion rows are explicit", async () => {
|
|
201
245
|
const audit = await api("/api/audit-log?action=shape_action");
|
|
202
246
|
expect(sortedKeys(audit.logs[0])).toEqual(sorted(GOLDEN_RESPONSE_KEYS.audit));
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// #1879 —— 失败分类:共享会话忙与真失败分开。
|
|
2
2
|
import { describe, expect, test } from "bun:test";
|
|
3
|
-
import { scheduledFailureCode } from "./scheduled-failure-code.js";
|
|
3
|
+
import { scheduledFailureCode, busyElapsedNote } from "./scheduled-failure-code.js";
|
|
4
4
|
|
|
5
5
|
describe("scheduledFailureCode", () => {
|
|
6
6
|
test("共享会话忙 → task_failed_busy(两种情况都覆盖)", () => {
|
|
@@ -23,3 +23,51 @@ describe("scheduledFailureCode", () => {
|
|
|
23
23
|
expect(scheduledFailureCode("expired", null)).toBe("task_expired");
|
|
24
24
|
});
|
|
25
25
|
});
|
|
26
|
+
|
|
27
|
+
// #2884 —— 两个量必须分开报:排队(创建→开始处理)与会话等待(开始处理→失败)。
|
|
28
|
+
describe("busyElapsedNote", () => {
|
|
29
|
+
test("生产里那条 412 分钟:拆成排队 382 + 会话等待 30.1", () => {
|
|
30
|
+
// 原始数据取自 tasks 表(2026-10-27,TMA需求欧)
|
|
31
|
+
const note = busyElapsedNote("2026-10-27 13:54:48", "2026-10-27 20:16:52", "2026-10-27 20:46:57");
|
|
32
|
+
expect(note).toBeTruthy();
|
|
33
|
+
expect(note!).toContain("实测总耗时 412 分钟");
|
|
34
|
+
expect(note!).toContain("排队 382 分钟");
|
|
35
|
+
expect(note!).toContain("会话等待 30.1 分钟");
|
|
36
|
+
// 关键:不能让人以为 30 分钟就是全部
|
|
37
|
+
expect(note!).toContain("不代表总共等了多久");
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test("没有被排队的(排队 0):照样把两段都说出来,不省略", () => {
|
|
41
|
+
const note = busyElapsedNote("2026-10-28 12:11:07", "2026-10-28 12:11:08", "2026-10-28 12:41:10");
|
|
42
|
+
expect(note!).toContain("排队 0 分钟");
|
|
43
|
+
expect(note!).toContain("会话等待 30 分钟");
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test("节点没上报开始处理时刻:只说总计,不硬凑分子", () => {
|
|
47
|
+
const note = busyElapsedNote("2026-10-27 13:54:48", null, "2026-10-27 20:46:57");
|
|
48
|
+
expect(note!).toContain("实测总耗时 412 分钟");
|
|
49
|
+
expect(note!).toContain("无法拆出排队与会话等待");
|
|
50
|
+
expect(note!).not.toContain("会话等待 30");
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("时刻缺失或倒挂 ⇒ null(宁可不写,也不编一个负数)", () => {
|
|
54
|
+
expect(busyElapsedNote(null, null, "2026-10-27 20:46:57")).toBeNull();
|
|
55
|
+
expect(busyElapsedNote("2026-10-27 13:54:48", null, null)).toBeNull();
|
|
56
|
+
expect(busyElapsedNote("2026-10-27 20:00:00", null, "2026-10-27 13:00:00")).toBeNull();
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test("跨库:PG 可能给 Date / 毫秒数,不能静默失效", () => {
|
|
60
|
+
const created = new Date(Date.now() - 412 * 60_000);
|
|
61
|
+
const started = new Date(Date.now() - 30 * 60_000);
|
|
62
|
+
const completed = new Date();
|
|
63
|
+
const fromDates = busyElapsedNote(created, started, completed);
|
|
64
|
+
expect(fromDates).toContain("排队 382 分钟");
|
|
65
|
+
expect(fromDates).toContain("会话等待 30 分钟");
|
|
66
|
+
|
|
67
|
+
const fromNumbers = busyElapsedNote(created.getTime(), started.getTime(), completed.getTime());
|
|
68
|
+
expect(fromNumbers).toContain("排队 382 分钟");
|
|
69
|
+
|
|
70
|
+
// 空串也要当「没有」,不能 parse 成 1970
|
|
71
|
+
expect(busyElapsedNote("", "", "")).toBeNull();
|
|
72
|
+
});
|
|
73
|
+
});
|
|
@@ -18,3 +18,88 @@ export function scheduledFailureCode(status: string, result: string | null): str
|
|
|
18
18
|
if (/仍在节点的 TUI 会话里运行|等待回复已达\s*30\s*分钟/.test(text)) return "task_failed_busy";
|
|
19
19
|
return base;
|
|
20
20
|
}
|
|
21
|
+
|
|
22
|
+
// #2884 —— 「排了多久」和「会话等了多久」是两个量,别用同一个数字代表。
|
|
23
|
+
//
|
|
24
|
+
// 现场(2026-10-27 实测,tasks 表 created_at → started_at → completed_at):
|
|
25
|
+
// 一条 412 分钟才失败的任务,拆开是 **排队 382 分钟 + 会话等待 30.1 分钟**。
|
|
26
|
+
// 失败文案只说了「30 分钟」,因为 30 分钟是「轮到它之后等共享会话空出来」的预算;
|
|
27
|
+
// 在轮到自己**之前**的排队既没有上限、也没有被写进文案。
|
|
28
|
+
//
|
|
29
|
+
// 后果不是文案不好看,是报表失真:按 error_code 聚合时这 180 条(94%)都被算成
|
|
30
|
+
// 「共享会话忙」,于是「排队几小时」这一层被彻底掩盖 —— 而它才是能操作的那一层
|
|
31
|
+
// (加节点 / 减排程 / 拉间隔),会话忙只能等人离开 TUI。
|
|
32
|
+
//
|
|
33
|
+
// 这里只做**如实测量与上报**,不改等待语义:fail fast 会改变派发行为,
|
|
34
|
+
// 要单独定阈值(排队上限独立成卡)。阈值是不变量,得先有数再选。
|
|
35
|
+
|
|
36
|
+
function parseDbTime(v: string | Date | number | null | undefined): number | null {
|
|
37
|
+
if (v === null || v === undefined || v === "") return null;
|
|
38
|
+
// 跨库:SQLite 给字符串(`YYYY-MM-DD HH:MM:SS`,UTC 无 Z),
|
|
39
|
+
// PG 的 timestamp 列可能直接给 Date 或毫秒数。两种都认,
|
|
40
|
+
// 否则这条投影在 PG 上会静默失效(返回 null ⇒ 不显示拆解),
|
|
41
|
+
// 而「静默失效」比「报错」难发现得多。
|
|
42
|
+
if (v instanceof Date) return Number.isFinite(v.getTime()) ? v.getTime() : null;
|
|
43
|
+
if (typeof v === "number") return Number.isFinite(v) ? v : null;
|
|
44
|
+
// 字符串里的空格要换掉:Date.parse 对 `... 20:16:52` 会按本地时区解释。
|
|
45
|
+
const ms = Date.parse(`${String(v).trim().replace(" ", "T")}Z`);
|
|
46
|
+
return Number.isFinite(ms) ? ms : null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function fmtMinutes(ms: number): string {
|
|
50
|
+
const min = ms / 60_000;
|
|
51
|
+
if (min >= 100) return `${Math.round(min)} 分钟`;
|
|
52
|
+
return `${(Math.round(min * 10) / 10).toString()} 分钟`;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* 把一条失败任务拆成「排队 / 会话等待」两段,供 run 记录与任务详情展示。
|
|
57
|
+
*
|
|
58
|
+
* 读不出时刻就不编:宁可只说总计,也不把两段硬凑成 30 分钟。
|
|
59
|
+
*
|
|
60
|
+
* #2884 —— 为什么排程与非排程任务都要覆盖:
|
|
61
|
+
* 现场那条 412 分钟的任务**不是排程任务**(scheduled_task_runs 里查不到),是直连任务。
|
|
62
|
+
* 只改排程 run 的投影,最该被修的那一类反而看不到拆解。
|
|
63
|
+
*/
|
|
64
|
+
export function busyElapsedNote(
|
|
65
|
+
createdAt: string | Date | number | null | undefined,
|
|
66
|
+
startedAt: string | Date | number | null | undefined,
|
|
67
|
+
completedAt: string | Date | number | null | undefined,
|
|
68
|
+
): string | null {
|
|
69
|
+
const created = parseDbTime(createdAt);
|
|
70
|
+
const completed = parseDbTime(completedAt);
|
|
71
|
+
if (created === null || completed === null) return null;
|
|
72
|
+
const total = completed - created;
|
|
73
|
+
if (!(total >= 0)) return null;
|
|
74
|
+
const started = parseDbTime(startedAt);
|
|
75
|
+
if (started === null) {
|
|
76
|
+
return `实测总耗时 ${fmtMinutes(total)}(节点未上报开始处理的时刻,无法拆出排队与会话等待)`;
|
|
77
|
+
}
|
|
78
|
+
const queued = Math.max(0, started - created);
|
|
79
|
+
const waited = Math.max(0, completed - started);
|
|
80
|
+
return `实测总耗时 ${fmtMinutes(total)}:排队 ${fmtMinutes(queued)}(任务创建→节点开始处理)`
|
|
81
|
+
+ ` + 会话等待 ${fmtMinutes(waited)}(节点开始处理→失败)。`
|
|
82
|
+
+ `失败文案里的等待上限只代表后一段,不代表总共等了多久。`;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** 幂等标记:已经加过拆解就不再重复追加。 */
|
|
86
|
+
const BUSY_NOTE_MARK = "实测总耗时";
|
|
87
|
+
export const BUSY_NOTE_MARKER = BUSY_NOTE_MARK;
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* #2884 —— 在读取投影上补「排队 / 会话等待」拆解(幂等,不改写入路径)。
|
|
91
|
+
*
|
|
92
|
+
* 只对 `task_failed_busy` 生效:其它失败原因的耗时与「排队」无关,
|
|
93
|
+
* 追加会让失败的真正原因更难看清。
|
|
94
|
+
*/
|
|
95
|
+
export function withBusyElapsed<T extends {
|
|
96
|
+
status?: string | null; result?: string | null;
|
|
97
|
+
created_at?: string | null; started_at?: string | null; completed_at?: string | null;
|
|
98
|
+
}>(task: T): T {
|
|
99
|
+
if (!task || scheduledFailureCode(task.status ?? null, task.result ?? null) !== "task_failed_busy") return task;
|
|
100
|
+
const current = typeof task.result === "string" ? task.result : null;
|
|
101
|
+
if (current && current.includes(BUSY_NOTE_MARK)) return task; // 已加过
|
|
102
|
+
const note = busyElapsedNote(task.created_at, task.started_at, task.completed_at);
|
|
103
|
+
if (!note) return task;
|
|
104
|
+
return { ...task, result: [current, note].filter(Boolean).join(" ") };
|
|
105
|
+
}
|
|
@@ -231,3 +231,49 @@ describe("#523 GET /runs", () => {
|
|
|
231
231
|
expect(typeof again.last_failure_alert_at).toBe("string");
|
|
232
232
|
});
|
|
233
233
|
});
|
|
234
|
+
|
|
235
|
+
// #2884 —— 共享会话忙:run 记录要把「排队」和「会话等待」两个量分开报。
|
|
236
|
+
// 生产实测(2026-10-27, TMA需求欧):一条 412 分钟才失败的任务,拆开是排队 382 + 会话等待 30。
|
|
237
|
+
// 文案只说过「30 分钟」,于是报表把所有长等待都算成「会话忙」,掩盖了「排队几小时」这一层。
|
|
238
|
+
describe("#2884 busy runs report queue and session wait separately", () => {
|
|
239
|
+
const BUSY = "[TMA需求欧] opencode 任务未提交:共享会话在 30 分钟 内一直处于忙碌状态(可能有人正在 TUI 里跑一轮),本任务没有发出";
|
|
240
|
+
|
|
241
|
+
test("error_message carries both numbers; the node's own wording is kept", async () => {
|
|
242
|
+
const s = seedSchedule("忙的情况");
|
|
243
|
+
const r = fire(s);
|
|
244
|
+
expect(r.status).toBe("delivered");
|
|
245
|
+
// 造出「排了 382 分钟、轮到后等了 30 分钟」:把创建时刻前移 412 分钟、开始处理前移 30 分钟,
|
|
246
|
+
// 回复会把 completed_at 设成现在。
|
|
247
|
+
// 时间用 JS 算再传参:`datetime('now','-N minutes')` 是 SQLite 方言,
|
|
248
|
+
// 这套用例在 PG ladder 里也跑(test2123),方言会让它直接报错 (#2884)。
|
|
249
|
+
const minutesAgo = (m: number) => new Date(Date.now() - m * 60_000).toISOString().replace("T", " ").replace(/\..+/, "");
|
|
250
|
+
db.run(
|
|
251
|
+
"UPDATE tasks SET created_at = ?2, started_at = ?3 WHERE task_id = ?1",
|
|
252
|
+
[r.taskId, minutesAgo(412), minutesAgo(30)],
|
|
253
|
+
);
|
|
254
|
+
const out = await nodeReply({ alias: "scheduler", text: BUSY, in_reply_to: r.taskId, status: "failed", from_session: ALIAS });
|
|
255
|
+
expect(out.ok).toBe(true);
|
|
256
|
+
|
|
257
|
+
const body = await getRuns(s.schedule_id);
|
|
258
|
+
const run = body.runs.find((x: any) => x.run_id === r.runId);
|
|
259
|
+
expect(run.status).toBe("failed");
|
|
260
|
+
expect(run.error_code).toBe("task_failed_busy");
|
|
261
|
+
// 两个量分开
|
|
262
|
+
expect(run.error_message).toContain("排队 382 分钟");
|
|
263
|
+
expect(run.error_message).toContain("会话等待 30 分钟");
|
|
264
|
+
// 节点原文没有被读数覆盖
|
|
265
|
+
expect(run.error_message).toContain("共享会话");
|
|
266
|
+
// 不能让人把 30 分钟读成总时长
|
|
267
|
+
expect(run.error_message).toContain("不代表总共等了多久");
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
test("not-busy failures never get the breakdown appended", async () => {
|
|
271
|
+
const s = seedSchedule("真失败");
|
|
272
|
+
const { runId } = await runOnce(s);
|
|
273
|
+
const body = await getRuns(s.schedule_id);
|
|
274
|
+
const run = body.runs.find((x: any) => x.run_id === runId);
|
|
275
|
+
expect(run.error_code).toBe("task_failed");
|
|
276
|
+
expect(run.error_message).toContain("safety system");
|
|
277
|
+
expect(run.error_message).not.toContain("排队");
|
|
278
|
+
});
|
|
279
|
+
});
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
import { db, setScheduledRunTerminalHook } from "./db.js";
|
|
21
21
|
import { sendAgentNotice } from "./agent-notice.js";
|
|
22
22
|
import { parseHubTimestamp } from "./hub-timestamp.js";
|
|
23
|
+
import { busyElapsedNote } from "./scheduled-failure-code.js";
|
|
23
24
|
|
|
24
25
|
export const SCHEDULE_FAILING_NOTICE_KIND = "schedule_failing";
|
|
25
26
|
const ERROR_PREVIEW_CHARS = 300;
|
|
@@ -73,8 +74,12 @@ export type FailureNotice = {
|
|
|
73
74
|
};
|
|
74
75
|
|
|
75
76
|
function latestFailure(scheduleId: string): { error_code: string | null; error_message: string | null } {
|
|
76
|
-
const row = db.get<{
|
|
77
|
-
|
|
77
|
+
const row = db.get<{
|
|
78
|
+
error_code: string | null; error_message: string | null; result: string | null;
|
|
79
|
+
task_created_at: string | null; task_started_at: string | null; task_completed_at: string | null;
|
|
80
|
+
}>(
|
|
81
|
+
`SELECT r.error_code, r.error_message, t.result,
|
|
82
|
+
t.created_at AS task_created_at, t.started_at AS task_started_at, t.completed_at AS task_completed_at
|
|
78
83
|
FROM scheduled_task_runs r
|
|
79
84
|
LEFT JOIN tasks t ON t.task_id = r.task_id AND t.network_id = r.network_id
|
|
80
85
|
WHERE r.schedule_id = ?1 AND r.status = 'failed'
|
|
@@ -82,7 +87,15 @@ function latestFailure(scheduleId: string): { error_code: string | null; error_m
|
|
|
82
87
|
scheduleId,
|
|
83
88
|
);
|
|
84
89
|
// 任务以 failed 结束时 run.error_message 为空,原因在任务的回复正文里。
|
|
85
|
-
|
|
90
|
+
// #2884 —— 共享会话忙的那一类,再把「排队 / 会话等待」两段实测补在后面:
|
|
91
|
+
// 只读拼接,不改写入路径,所以节点原文不会被覆盖。
|
|
92
|
+
const base = row?.error_message || row?.result || null;
|
|
93
|
+
if (row?.error_code !== "task_failed_busy") return { error_code: row?.error_code ?? null, error_message: base };
|
|
94
|
+
const note = busyElapsedNote(row?.task_created_at, row?.task_started_at, row?.task_completed_at);
|
|
95
|
+
return {
|
|
96
|
+
error_code: row?.error_code ?? null,
|
|
97
|
+
error_message: note ? [base, note].filter(Boolean).join(" ") : base,
|
|
98
|
+
};
|
|
86
99
|
}
|
|
87
100
|
|
|
88
101
|
function truncate(s: string, max: number): string {
|
package/src/scheduled-tasks.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { sessionTaskOnDispatch } from "./session-task-on-dispatch.js";
|
|
|
6
6
|
import { TASK_CONTENT_MAX, sessionTaskPreview } from "./shared/task-content-limit.js";
|
|
7
7
|
import { assertNodeActive } from "./lifecycle-guard.js";
|
|
8
8
|
import { assertNodeHealthy } from "./node-health-guard.js";
|
|
9
|
+
import { nodeCanAcceptDispatch } from "./node-serviceability.js";
|
|
9
10
|
import { addNetworkScope, canRestWriteNetwork, singleNetworkId, type RestNetworkScope } from "./network-scope.js";
|
|
10
11
|
import { canMessageAgent, type AgentRef } from "./agent-access.js";
|
|
11
12
|
import { dispatchVerdict } from "./node-permissions.js";
|
|
@@ -14,6 +15,7 @@ import { SCHEDULED_TASK_STORAGE_SELECT } from "./rest-projections.js";
|
|
|
14
15
|
import { blockerTimedOut, blockerTimeoutClock, expireStuckBlocker, sendStuckNotice, skipsBlockedBy, stuckNoticeSkips, stuckTimeoutMs, type StuckNotice } from "./scheduled-stuck.js";
|
|
15
16
|
import { applyNoProgressBackoff, evaluateScheduleFailures, failureSummary } from "./scheduled-failures.js";
|
|
16
17
|
import { parseHubTimestamp } from "./hub-timestamp.js";
|
|
18
|
+
import { busyElapsedNote } from "./scheduled-failure-code.js";
|
|
17
19
|
import { scheduledTargetOffline, sendOfflineNotice, type OfflineNotice } from "./scheduled-offline.js";
|
|
18
20
|
|
|
19
21
|
export type ScheduleSpec =
|
|
@@ -264,14 +266,22 @@ export function cancelSchedule(row: ScheduledRow): void {
|
|
|
264
266
|
export function scheduleRuns(row: ScheduledRow, limit: number): Record<string, unknown> {
|
|
265
267
|
const runs = db.all(
|
|
266
268
|
// #523 — a task that ended `failed` leaves run.error_message empty; its reason is the task's reply text.
|
|
269
|
+
// #2884 — 共享会话忙的失败额外带上实测拆解(排队 / 会话等待两个量分开报)。
|
|
267
270
|
`SELECT r.run_id, r.schedule_id, r.scheduled_for, r.task_id, r.status, r.error_code,
|
|
268
271
|
CASE WHEN r.error_message IS NULL AND r.status = 'failed' THEN SUBSTR(t.result, 1, 500) ELSE r.error_message END AS error_message,
|
|
269
|
-
r.blocked_by_task_id, r.blocked_by_state, r.created_at, r.completed_at, r.work_outcome, r.schedule_revision, r.work_outcome_node_id
|
|
272
|
+
r.blocked_by_task_id, r.blocked_by_state, r.created_at, r.completed_at, r.work_outcome, r.schedule_revision, r.work_outcome_node_id,
|
|
273
|
+
t.created_at AS task_created_at, t.started_at AS task_started_at, t.completed_at AS task_completed_at
|
|
270
274
|
FROM scheduled_task_runs r
|
|
271
275
|
LEFT JOIN tasks t ON t.task_id = r.task_id AND t.network_id = r.network_id
|
|
272
276
|
WHERE r.schedule_id = ?1 AND r.network_id = ?2 ORDER BY r.created_at DESC, r.scheduled_for DESC LIMIT ?3`,
|
|
273
277
|
row.schedule_id, row.network_id, limit,
|
|
274
|
-
)
|
|
278
|
+
).map((r: any) => {
|
|
279
|
+
const note = r.error_code === "task_failed_busy"
|
|
280
|
+
? busyElapsedNote(r.task_created_at, r.task_started_at, r.task_completed_at)
|
|
281
|
+
: null;
|
|
282
|
+
const { task_created_at, task_started_at, task_completed_at, ...rest } = r;
|
|
283
|
+
return note ? { ...rest, error_message: [r.error_message, note].filter(Boolean).join(" ") } : rest;
|
|
284
|
+
});
|
|
275
285
|
return { runs, ...failureSummary(row.schedule_id) };
|
|
276
286
|
}
|
|
277
287
|
|
|
@@ -587,6 +597,24 @@ export function dispatchScheduledOccurrence(row: ScheduledRow, scheduledFor: str
|
|
|
587
597
|
return;
|
|
588
598
|
}
|
|
589
599
|
|
|
600
|
+
// #2880 —— 节点级忙熔断(默认关闭;COMMHUB_NODE_BUSY_CIRCUIT=1 才生效)。
|
|
601
|
+
// 与 lifecycle/health 的区别:节点没坏、也没停,只是最近一直没有产出(忙类失败多且 0 成功),
|
|
602
|
+
// 投进去必然白等 ~30 分钟。所以记 **skipped** 而不是 failed —— 这不是故障,是「现在别投」。
|
|
603
|
+
// 熔断期间每 COMMHUB_NODE_BUSY_CIRCUIT_PROBE_HOURS 小时会放一条探针(见 node-serviceability.ts),
|
|
604
|
+
// 避免「不投递 ⇒ 拿不到成功 ⇒ 永远解不开」的死锁。
|
|
605
|
+
{
|
|
606
|
+
const circuit = nodeCanAcceptDispatch(node.alias, row.network_id);
|
|
607
|
+
if (!circuit.ok) {
|
|
608
|
+
finalStatus = "skipped";
|
|
609
|
+
db.run(
|
|
610
|
+
"UPDATE scheduled_task_runs SET status = 'skipped', error_code = 'target_busy_no_output', error_message = ?1, completed_at = datetime('now') WHERE run_id = ?2",
|
|
611
|
+
[String(circuit.reason ?? "target_busy_no_output").slice(0, 500), runId],
|
|
612
|
+
);
|
|
613
|
+
if (advanceSchedule) advance(row, scheduledFor, advanceAfter);
|
|
614
|
+
return;
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
|
|
590
618
|
// #460 — the target reports a degraded layer (App Server down / TUI gone / login revoked): the run fails
|
|
591
619
|
// with that reason instead of queueing a task nobody will execute. Unknown / stale health runs as before.
|
|
592
620
|
const health = assertNodeHealthy(node.alias, row.network_id, { forceAllowed: false });
|
package/src/server.ts
CHANGED
|
@@ -43,6 +43,7 @@ import { expireStaleOpenTasks } from "./task-stale-open.js";
|
|
|
43
43
|
import { expireDeliveredToOfflineTargets } from "./task-offline-delivery.js";
|
|
44
44
|
import { flagOrphanTasks } from "./task-orphan.js";
|
|
45
45
|
import { addAgentNetworkScope, addHumanNetworkScope, addNetworkScope, addOwnTrafficScope, addAgentTimelineScope, canRestWriteNetwork, canRestWriteNetworkAsHuman, getUserNetworkIds, resolveRestNetworkScope, resolveRestWriteNetworkId, singleNetworkId, type RestNetworkScope } from "./network-scope.js";
|
|
46
|
+
import { withBusyElapsed } from "./scheduled-failure-code.js";
|
|
46
47
|
import { pickAliasNetwork } from "./rest-alias-network-fallback.js";
|
|
47
48
|
import { lifecycleProjections, lifecycleRequestResponse } from "./node-lifecycle-read.js";
|
|
48
49
|
import { listAdoptionCandidates, snapshotReportsAdoptionDiscovery } from "./adoption-candidates.js";
|
|
@@ -5430,10 +5431,12 @@ return Bun.serve({
|
|
|
5430
5431
|
let sql = `SELECT ${TASK_REST_SELECT} FROM tasks WHERE task_id = ?1`;
|
|
5431
5432
|
sql = addAgentTimelineScope(sql, params, restScope, { from: "from_name", to: "to_name", fromNodeId: "from_node_id", toNodeId: "to_node_id" });
|
|
5432
5433
|
sql += " LIMIT 1";
|
|
5433
|
-
const
|
|
5434
|
-
if (!
|
|
5434
|
+
const rawTask = db.get(sql, ...params);
|
|
5435
|
+
if (!rawTask) {
|
|
5435
5436
|
return withCors(req, Response.json({ ok: false, error: "task_not_found", task_id: taskId }, { status: 404 }));
|
|
5436
5437
|
}
|
|
5438
|
+
// #2884 —— 共享会话忙:读取时补上「排队 / 会话等待」拆解(幂等,不改写入)。
|
|
5439
|
+
const task = withBusyElapsed(rawTask as any);
|
|
5437
5440
|
const taskNetworkId = typeof task.network_id === "string" ? task.network_id : null;
|
|
5438
5441
|
const canonicalTarget = task.to_node_id
|
|
5439
5442
|
? null
|
|
@@ -5535,7 +5538,7 @@ return Bun.serve({
|
|
|
5535
5538
|
sql += ` ORDER BY created_at DESC, task_id DESC LIMIT ?${params.length + 1}`;
|
|
5536
5539
|
params.push(limit);
|
|
5537
5540
|
|
|
5538
|
-
const rows = db.all<any>(sql, ...params);
|
|
5541
|
+
const rows = db.all<any>(sql, ...params).map((row: any) => withBusyElapsed(row));
|
|
5539
5542
|
// Existing desktop clients implement their one-task detail read as
|
|
5540
5543
|
// `/api/tasks?task_id=…&limit=1`. Keep broad list responses lean, but
|
|
5541
5544
|
// enrich that exact-detail shape as well as `/api/tasks/:id`.
|