@sleep2agi/commhub-server 0.9.0-preview.188 → 0.9.0-preview.190

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sleep2agi/commhub-server",
3
- "version": "0.9.0-preview.188",
3
+ "version": "0.9.0-preview.190",
4
4
  "description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
5
5
  "type": "module",
6
6
  "main": "src/index.ts",
@@ -0,0 +1,67 @@
1
+ // #2880 —— 节点级「不可服务」检测(默认关闭,只记录)。
2
+ import { describe, expect, test, beforeAll, afterAll } from "bun:test";
3
+ import { db } from "./db.js";
4
+ import { isNotServing, scanNodeBusyCircuit, nodeCanAcceptDispatch, circuitEnabled } from "./node-serviceability.js";
5
+
6
+ const NET = "net_circuit";
7
+
8
+ // 🔴 scheduled_task_runs 上有 UNIQUE(schedule_id, scheduled_for):
9
+ // 用 INSERT OR REPLACE 且 scheduled_for 相同 ⇒ 后一行会**替换**前一行,
10
+ // 于是 6 条「忙类失败」最后只剩 1 条,判据永远不成立。每条用不同的 minute 偏移。
11
+ function mkTaskAndRun(taskId: string, alias: string, status: string, result: string | null, runStatus: string, errorCode: string | null, minuteOffset = 0) {
12
+ db.run(
13
+ `INSERT OR REPLACE INTO tasks (task_id, content, from_name, to_name, network_id, status, result, created_at, completed_at)
14
+ VALUES (?1, 'x', 'hub', ?2, ?3, ?4, ?5, datetime('now'), datetime('now'))`,
15
+ [taskId, alias, NET, status, result],
16
+ );
17
+ db.run(
18
+ `INSERT OR REPLACE INTO scheduled_task_runs (run_id, schedule_id, network_id, scheduled_for, task_id, status, error_code, created_at)
19
+ VALUES (?1, 'sched_x', ?2, datetime('now', ?3), ?4, ?5, ?6, datetime('now'))`,
20
+ [`run_${taskId}`, NET, `-${minuteOffset + 1} minutes`, taskId, runStatus, errorCode],
21
+ );
22
+ }
23
+
24
+ const BUSY = "[n] opencode 任务未提交:共享会话在 30 分钟 内一直处于忙碌状态";
25
+
26
+ describe("节点忙熔断:三项判据", () => {
27
+ beforeAll(() => {
28
+ db.run("DELETE FROM scheduled_task_runs WHERE network_id = ?1", [NET]);
29
+ db.run("DELETE FROM tasks WHERE network_id = ?1", [NET]);
30
+ });
31
+ afterAll(() => {
32
+ db.run("DELETE FROM scheduled_task_runs WHERE network_id = ?1", [NET]);
33
+ db.run("DELETE FROM tasks WHERE network_id = ?1", [NET]);
34
+ delete process.env.COMMHUB_NODE_BUSY_CIRCUIT;
35
+ });
36
+
37
+ test("纯判定:忙类失败够多 + 零成功 + 有投递 ⇒ 不可服务", () => {
38
+ expect(isNotServing({ delivered: 10, replied: 0, busy_failed: 5 }, 5)).toBe(true);
39
+ expect(isNotServing({ delivered: 10, replied: 1, busy_failed: 9 }, 5)).toBe(false); // 成功过 = 等得到
40
+ expect(isNotServing({ delivered: 4, replied: 0, busy_failed: 4 }, 5)).toBe(false); // 证据不够
41
+ expect(isNotServing({ delivered: 0, replied: 0, busy_failed: 0 }, 5)).toBe(false); // 没投递过 ≠ 不可服务
42
+ });
43
+
44
+ test("扫描能把「零产出节点」挑出来,且有成功的节点不受影响", () => {
45
+ for (let i = 0; i < 6; i++) mkTaskAndRun(`t_bh_${i}`, "黑洞节点", "failed", BUSY, "failed", null, i);
46
+ for (let i = 0; i < 6; i++) mkTaskAndRun(`t_ok_${i}`, "正常节点", i < 3 ? "replied" : "failed", i < 3 ? "done" : BUSY, i < 3 ? "replied" : "failed", null, i + 10);
47
+ const r = scanNodeBusyCircuit();
48
+ expect(r.tripped).toContain("黑洞节点");
49
+ expect(r.tripped).not.toContain("正常节点");
50
+ expect(r.enabled).toBe(false); // 默认关闭
51
+ });
52
+
53
+ test("默认关闭时投递判断恒放行(不改任何既有行为)", () => {
54
+ delete process.env.COMMHUB_NODE_BUSY_CIRCUIT;
55
+ expect(circuitEnabled()).toBe(false);
56
+ expect(nodeCanAcceptDispatch("黑洞节点", NET).ok).toBe(true);
57
+ });
58
+
59
+ test("开启后才拦:黑洞节点被拒,正常节点仍放行", () => {
60
+ process.env.COMMHUB_NODE_BUSY_CIRCUIT = "1";
61
+ const bad = nodeCanAcceptDispatch("黑洞节点", NET);
62
+ expect(bad.ok).toBe(false);
63
+ expect(bad.reason ?? "").toContain("忙类失败");
64
+ expect(nodeCanAcceptDispatch("正常节点", NET).ok).toBe(true);
65
+ expect(nodeCanAcceptDispatch("不存在的节点", NET).ok).toBe(true); // 没数据就不拦
66
+ });
67
+ });
@@ -0,0 +1,96 @@
1
+ // #2880 —— 节点级「不可服务」检测。
2
+ //
3
+ // 生产 24h(2026-10-28 按节点统计定时任务产出):
4
+ // TMA云盘欧 ok= 0 busy=35 ← 全天零产出
5
+ // 通信客户端欧 ok= 22 busy=17 ← 等得到
6
+ // TMA需求欧 ok=140 busy=13 ← 等得到
7
+ // 通信欧 ok=192 busy= 0
8
+ //
9
+ // 结论:**「忙就快速失败」不能全局开**(会打掉别人等得到的那批成功),
10
+ // 但**节点级**的判断是安全的 —— 一个节点在窗口内「忙类失败很多、成功为零」,
11
+ // 才是「现在投进去必然白等 30 分钟」。
12
+ //
13
+ // 判据(三项同时成立,缺一不可):
14
+ // 1. 窗口内忙类失败 >= MIN_FAILS(默认 5)—— 证据量够
15
+ // 2. 窗口内成功 == 0 —— 不是「忙但能干成」
16
+ // 3. 窗口内至少有一次投递 —— 不会把「今天没排程」误判成不可服务
17
+ //
18
+ // 默认只记录(不改变任何派发行为)。开启 COMMHUB_NODE_BUSY_CIRCUIT=1 才把
19
+ // 「暂不投递」用于排程派发 —— 这一步是行为变更,所以留给人开。
20
+ import { db } from "./db.js";
21
+
22
+ function envInt(name: string, fallback: number, min: number): number {
23
+ const raw = process.env[name];
24
+ if (raw === undefined || raw.trim() === "") return fallback;
25
+ const n = Number(raw);
26
+ return Number.isFinite(n) && n >= min ? Math.floor(n) : fallback;
27
+ }
28
+
29
+ export function circuitEnabled(): boolean {
30
+ return String(process.env.COMMHUB_NODE_BUSY_CIRCUIT ?? "").trim() === "1";
31
+ }
32
+ export function circuitWindowHours(): number { return envInt("COMMHUB_NODE_BUSY_CIRCUIT_HOURS", 6, 1); }
33
+ export function circuitMinFails(): number { return envInt("COMMHUB_NODE_BUSY_CIRCUIT_FAILS", 5, 1); }
34
+
35
+ export type NodeServingStats = { alias: string; network_id: string | null; delivered: number; replied: number; busy_failed: number };
36
+
37
+ /** 纯函数:三项判据同时成立才算「不可服务」。 */
38
+ export function isNotServing(
39
+ s: { delivered: number; replied: number; busy_failed: number },
40
+ minFails = circuitMinFails(),
41
+ ): boolean {
42
+ return s.busy_failed >= minFails && s.replied === 0 && s.delivered > 0;
43
+ }
44
+
45
+ export type CircuitScanResult = { tripped: string[]; enabled: boolean; windowHours: number };
46
+
47
+ export function scanNodeBusyCircuit(now = new Date()): CircuitScanResult {
48
+ const windowHours = circuitWindowHours();
49
+ const since = new Date(now.getTime() - windowHours * 3_600_000).toISOString().slice(0, 19).replace("T", " ");
50
+ const rows = db.all<{ alias: string; network_id: string | null; delivered: number; replied: number; busy_failed: number }>(
51
+ `SELECT t.to_name AS alias,
52
+ t.network_id AS network_id,
53
+ COUNT(*) AS delivered,
54
+ SUM(CASE WHEN r.status = 'replied' THEN 1 ELSE 0 END) AS replied,
55
+ SUM(CASE WHEN r.status = 'failed'
56
+ AND (r.error_code = 'task_failed_busy' OR t.result LIKE '%共享会话%')
57
+ THEN 1 ELSE 0 END) AS busy_failed
58
+ FROM scheduled_task_runs r
59
+ JOIN tasks t ON t.task_id = r.task_id
60
+ WHERE r.created_at >= ?1 AND t.to_name IS NOT NULL
61
+ GROUP BY t.to_name, t.network_id`,
62
+ since,
63
+ );
64
+ const tripped = rows.filter((r) => isNotServing(r)).map((r) => r.alias);
65
+ return { tripped, enabled: circuitEnabled(), windowHours };
66
+ }
67
+
68
+ /**
69
+ * 这个节点现在能不能投递。默认永远 true(未开启 = 不拦截)。
70
+ * 供排程派发判断;纯读,不改任何状态。
71
+ */
72
+ export function nodeCanAcceptDispatch(alias: string, networkId: string | null): { ok: boolean; reason?: string; busy_failed?: number } {
73
+ if (!circuitEnabled()) return { ok: true };
74
+ const windowHours = circuitWindowHours();
75
+ const since = new Date(Date.now() - windowHours * 3_600_000).toISOString().slice(0, 19).replace("T", " ");
76
+ const row = db.get<{ delivered: number; replied: number; busy_failed: number }>(
77
+ `SELECT COUNT(*) AS delivered,
78
+ SUM(CASE WHEN r.status = 'replied' THEN 1 ELSE 0 END) AS replied,
79
+ SUM(CASE WHEN r.status = 'failed'
80
+ AND (r.error_code = 'task_failed_busy' OR t.result LIKE '%共享会话%')
81
+ THEN 1 ELSE 0 END) AS busy_failed
82
+ FROM scheduled_task_runs r
83
+ JOIN tasks t ON t.task_id = r.task_id
84
+ WHERE r.created_at >= ?1 AND t.to_name = ?2
85
+ AND (?3 IS NULL OR t.network_id = ?3)`,
86
+ since, alias, networkId,
87
+ );
88
+ if (!row) return { ok: true };
89
+ const stats = { delivered: Number(row.delivered ?? 0), replied: Number(row.replied ?? 0), busy_failed: Number(row.busy_failed ?? 0) };
90
+ if (!isNotServing(stats)) return { ok: true };
91
+ return {
92
+ ok: false,
93
+ reason: `节点近 ${windowHours}h 内 ${stats.busy_failed} 次忙类失败、0 次成功(投递也只会白等),已跳过`,
94
+ busy_failed: stats.busy_failed,
95
+ };
96
+ }
@@ -174,9 +174,14 @@ describe("changes=1", () => {
174
174
  expect(Date.parse(delta.body.tombstones_since)).toBeLessThan(Date.parse(t0));
175
175
 
176
176
  // 接着读:什么都没变就是空的
177
+ // 🔴 server_time 的语义是「>= 会重复回一次,客户端按 id 覆盖」(见 requirements.ts 的注释)。
178
+ // 早先这里断言 deleted 必须为空 —— 一旦删除恰好落在 server_time 的同一秒,
179
+ // 边界那条会被重复下发,测试就红(与顺序无关,是**与时间赛跑**的偶发)。
180
+ // 按契约断言:允许重复,但不许出现边界之后的新内容。
177
181
  const again = await list(admin.token, `&changes=1&updated_since=${encodeURIComponent(delta.body.server_time)}`);
178
- expect(again.body.requirements).toEqual([]);
179
- expect(again.body.deleted).toEqual([]);
182
+ expect([...new Set(again.body.requirements.map((r: any) => r.id))]).toEqual([]);
183
+ // 允许:空(边界已过去)或重复下发 gone(同一秒边界);不允许:别的东西冒出来
184
+ expect([...new Set(again.body.deleted)].every((id: string) => id === gone.id)).toBe(true);
180
185
 
181
186
  // 与 view=summary 组合
182
187
  const slim = await list(admin.token, `&changes=1&view=summary&updated_since=${encodeURIComponent(t0)}`);
@@ -0,0 +1,83 @@
1
+ // #2880 —— 空转排程:默认只记录,开启开关才真暂停。
2
+ import { describe, expect, test, beforeAll, afterAll } from "bun:test";
3
+ import { db } from "./db.js";
4
+ import { isStarved, scanStarvedSchedules, autopauseEnabled } from "./schedule-starved.js";
5
+
6
+ const NET = "net_starved";
7
+ const USER = "u_starved";
8
+
9
+ function mkSchedule(id: string, name: string): void {
10
+ db.run(
11
+ `INSERT INTO scheduled_tasks (schedule_id, network_id, created_by, name, target_node_id, target_alias, task_content, priority, schedule_type, schedule_json, timezone, overlap_policy, status, next_run_at)
12
+ VALUES (?1, ?2, ?3, ?4, 'busy-node', 'busy-node', 'x', 'normal', 'interval', '{"type":"interval","every_seconds":180}', 'Asia/Shanghai', 'skip', 'active', datetime('now'))`,
13
+ [id, NET, USER, name],
14
+ );
15
+ }
16
+
17
+ function mkRuns(scheduleId: string, total: number, replied: number): void {
18
+ for (let i = 0; i < total; i++) {
19
+ const status = i < replied ? "replied" : "skipped";
20
+ db.run(
21
+ // scheduled_for 上有 UNIQUE(schedule_id, scheduled_for):每行错开一分钟
22
+ `INSERT INTO scheduled_task_runs (run_id, schedule_id, network_id, scheduled_for, status, error_code, created_at)
23
+ VALUES (?1, ?2, ?3, datetime('now', ?5), ?4, 'previous_run_active', datetime('now'))`,
24
+ [`run_${scheduleId}_${i}`, scheduleId, NET, status, `-${i} minutes`],
25
+ );
26
+ }
27
+ }
28
+
29
+ describe("空转排程判定与自动暂停", () => {
30
+ beforeAll(() => {
31
+ db.run("DELETE FROM scheduled_task_runs WHERE network_id = ?1", [NET]);
32
+ db.run("DELETE FROM scheduled_tasks WHERE network_id = ?1", [NET]);
33
+ db.run("DELETE FROM user_inbox WHERE network_id = ?1", [NET]);
34
+ });
35
+ afterAll(() => {
36
+ db.run("DELETE FROM scheduled_task_runs WHERE network_id = ?1", [NET]);
37
+ db.run("DELETE FROM scheduled_tasks WHERE network_id = ?1", [NET]);
38
+ db.run("DELETE FROM user_inbox WHERE network_id = ?1", [NET]);
39
+ delete process.env.COMMHUB_SCHEDULE_STARVED_AUTOPAUSE;
40
+ });
41
+
42
+ test("纯判定:次数够且 0 成功才算空转", () => {
43
+ expect(isStarved({ total: 20, replied: 0 }, 20)).toBe(true);
44
+ expect(isStarved({ total: 19, replied: 0 }, 20)).toBe(false); // 次数不够:一天一次的排程不会误伤
45
+ expect(isStarved({ total: 129, replied: 1 }, 20)).toBe(false); // 只要成功过一次就不算
46
+ });
47
+
48
+ test("默认关闭:只报候选,不改排程状态", () => {
49
+ delete process.env.COMMHUB_SCHEDULE_STARVED_AUTOPAUSE;
50
+ mkSchedule("sched_starved_a", "空转A");
51
+ mkRuns("sched_starved_a", 25, 0);
52
+ expect(autopauseEnabled()).toBe(false);
53
+ const r = scanStarvedSchedules();
54
+ expect(r.enabled).toBe(false);
55
+ expect(r.paused).toEqual([]);
56
+ expect(r.candidates).toContain("sched_starved_a");
57
+ const st = db.get<{ status: string }>("SELECT status FROM scheduled_tasks WHERE schedule_id = 'sched_starved_a'");
58
+ expect(st?.status).toBe("active"); // 没有被动过
59
+ });
60
+
61
+ test("开启开关后才暂停,并且只停空转的那个", () => {
62
+ process.env.COMMHUB_SCHEDULE_STARVED_AUTOPAUSE = "1";
63
+ mkSchedule("sched_starved_b", "空转B");
64
+ mkRuns("sched_starved_b", 30, 0);
65
+ mkSchedule("sched_healthy_b", "健康B");
66
+ mkRuns("sched_healthy_b", 30, 12);
67
+ const r = scanStarvedSchedules();
68
+ expect(r.enabled).toBe(true);
69
+ expect(r.paused).toContain("sched_starved_b");
70
+ expect(r.paused).not.toContain("sched_healthy_b");
71
+ expect(db.get<{ status: string }>("SELECT status FROM scheduled_tasks WHERE schedule_id = 'sched_starved_b'")?.status).toBe("paused");
72
+ expect(db.get<{ status: string }>("SELECT status FROM scheduled_tasks WHERE schedule_id = 'sched_healthy_b'")?.status).toBe("active");
73
+ });
74
+
75
+ test("已暂停的不会再被处理(重复扫描无副作用)", () => {
76
+ process.env.COMMHUB_SCHEDULE_STARVED_AUTOPAUSE = "1";
77
+ const before = db.get<{ n: number }>("SELECT COUNT(*) AS n FROM user_inbox WHERE network_id = ?1", [NET])?.n ?? 0;
78
+ const r = scanStarvedSchedules();
79
+ expect(r.paused).not.toContain("sched_starved_b");
80
+ const after = db.get<{ n: number }>("SELECT COUNT(*) AS n FROM user_inbox WHERE network_id = ?1", [NET])?.n ?? 0;
81
+ expect(after).toBe(before); // 不会补发第二条通知
82
+ });
83
+ });
@@ -0,0 +1,98 @@
1
+ // #2880 / #2881 —— 「空转排程」:先记录,默认**不**自动暂停。
2
+ //
3
+ // 现场(2026-10-28 生产只读):
4
+ // · TMA云盘欧 24h 接 44 单 **0 成功**,两个 180s 排程各跳过 147 / 68 次,占用 ~30.9h/天
5
+ // · 另有若干排程 24h 内 129 次执行 / 129 次跳过 / 0 次成功
6
+ // · 这些排程的 schedule_stuck 通知 7 天 1162 条(单个排程 220 条)⇒ 没人再看(#3161 正在限流)
7
+ //
8
+ // 治本不是「再多发一条通知」,而是让一直产不出东西的排程**停下来**。
9
+ // 但「自动停别人的排程」是行为变更,必须由人开启 —— 所以这里做成**默认关闭**的开关,
10
+ // 并在关闭状态下照样把「本来会被暂停哪些」打进日志:先让人看见候选,再决定要不要动手。
11
+ //
12
+ // COMMHUB_SCHEDULE_STARVED_AUTOPAUSE=1 开启真正的暂停(默认关)
13
+ // COMMHUB_SCHEDULE_STARVED_MIN_RUNS=20 窗口内至少这么多次执行才判定(默认 20)
14
+ // COMMHUB_SCHEDULE_STARVED_WINDOW_HOURS=24 统计窗口(默认 24)
15
+ //
16
+ // 判据只用「窗口内执行次数 ≥ N 且成功 0 次」——不参杂跳过比例:
17
+ // 一个真在干活的排程不会 20 次全不成功;而「一天只跑一次且失败」的排程达不到 20,不会误伤。
18
+ import { db } from "./db.js";
19
+ import { sendAgentNotice } from "./agent-notice.js";
20
+
21
+ export const SCHEDULE_STARVED_KIND = "schedule_starved_paused";
22
+
23
+ function envInt(name: string, fallback: number, min: number): number {
24
+ const raw = process.env[name];
25
+ if (raw === undefined || raw.trim() === "") return fallback;
26
+ const n = Number(raw);
27
+ return Number.isFinite(n) && n >= min ? Math.floor(n) : fallback;
28
+ }
29
+
30
+ export function autopauseEnabled(): boolean {
31
+ return String(process.env.COMMHUB_SCHEDULE_STARVED_AUTOPAUSE ?? "").trim() === "1";
32
+ }
33
+ export function starvedMinRuns(): number { return envInt("COMMHUB_SCHEDULE_STARVED_MIN_RUNS", 20, 1); }
34
+ export function starvedWindowHours(): number { return envInt("COMMHUB_SCHEDULE_STARVED_WINDOW_HOURS", 24, 1); }
35
+
36
+ export type ScheduleStats = { schedule_id: string; total: number; replied: number };
37
+
38
+ /** 纯函数:是否判定为空转。便于单测,不做任何 IO。 */
39
+ export function isStarved(s: { total: number; replied: number }, minRuns = starvedMinRuns()): boolean {
40
+ return s.total >= minRuns && s.replied === 0;
41
+ }
42
+
43
+ export type StarvedScanResult = { paused: string[]; candidates: string[]; enabled: boolean };
44
+
45
+ /**
46
+ * 扫一遍窗口内的排程产出。开启开关才真暂停;未开启只返回候选(供日志观察)。
47
+ * 纯读 + 可选的一次状态写 + 一条通知,不改任务。
48
+ */
49
+ export function scanStarvedSchedules(now = new Date()): StarvedScanResult {
50
+ const windowHours = starvedWindowHours();
51
+ const since = new Date(now.getTime() - windowHours * 3_600_000).toISOString().slice(0, 19).replace("T", " ");
52
+ const rows = db.all<ScheduleStats>(
53
+ `SELECT schedule_id,
54
+ COUNT(*) AS total,
55
+ SUM(CASE WHEN status = 'replied' THEN 1 ELSE 0 END) AS replied
56
+ FROM scheduled_task_runs
57
+ WHERE created_at >= ?1
58
+ GROUP BY schedule_id`,
59
+ since,
60
+ );
61
+ const candidates = rows.filter((r) => isStarved(r)).map((r) => r.schedule_id);
62
+ if (!autopauseEnabled() || candidates.length === 0) {
63
+ return { paused: [], candidates, enabled: autopauseEnabled() };
64
+ }
65
+ const paused: string[] = [];
66
+ for (const scheduleId of candidates) {
67
+ const info = db.get<{ network_id: string; created_by: string | null; name: string; target_alias: string | null }>(
68
+ `SELECT network_id, created_by, name, target_alias FROM scheduled_tasks
69
+ WHERE schedule_id = ?1 AND status = 'active'`,
70
+ scheduleId,
71
+ );
72
+ if (!info) continue; // 已暂停/已完成/已取消:不重复处理
73
+ const stats = rows.find((r) => r.schedule_id === scheduleId)!;
74
+ db.run("UPDATE scheduled_tasks SET status = 'paused', updated_at = datetime('now') WHERE schedule_id = ?1", [scheduleId]);
75
+ console.log(`[commhub schedule-starved] paused ${scheduleId} (${info.name}) — ${stats.total} 次执行 / 0 成功 in ${windowHours}h`);
76
+ if (info.created_by) {
77
+ try {
78
+ sendAgentNotice({
79
+ networkId: info.network_id,
80
+ userId: info.created_by,
81
+ fromAlias: info.target_alias ?? "hub",
82
+ kind: SCHEDULE_STARVED_KIND,
83
+ title: "定时任务已自动暂停(一直产不出结果)",
84
+ text:
85
+ `定时任务「${info.name}」在最近 ${windowHours} 小时内执行了 ${stats.total} 次,**没有一次成功**,已自动暂停。\n\n`
86
+ + `常见原因:目标节点被人的 TUI 长期占着,任务根本没提交上去。\n`
87
+ + `处理:确认节点可用后在排程页恢复;如果确实不需要了,直接停用。\n\n`
88
+ + `(此动作由 COMMHUB_SCHEDULE_STARVED_AUTOPAUSE=1 开启;不设该变量时只记录不暂停。)`,
89
+ severity: "warning",
90
+ meta: { schedule_starved: { schedule_id: scheduleId, total: stats.total, replied: 0, window_hours: windowHours } },
91
+ idPrefix: "dm_sstarved_",
92
+ });
93
+ } catch { /* 通知失败不影响已做出的暂停决定 */ }
94
+ }
95
+ paused.push(scheduleId);
96
+ }
97
+ return { paused, candidates, enabled: true };
98
+ }
@@ -0,0 +1,80 @@
1
+ // #2881 —— 同一个排程的「卡住」通知每日上限。
2
+ //
3
+ // 现场(2026-10-28 只读统计):7 天 1162 条 schedule_stuck 通知,单个排程 220 条。
4
+ // 设计前提是「新的阻塞任务 = 新事件 ⇒ 重新上膛」,但节点长期被占时每轮都会产生新阻塞者,
5
+ // 于是一个永远跑不成的排程能一天发几十条,结果没人再看。
6
+ //
7
+ // 这组用例钉住:前 N 条照发,第 N+1 条起被抑制,且**排程状态不受影响**(只影响通知)。
8
+ import { describe, expect, test, beforeAll, afterAll } from "bun:test";
9
+ import { mkdtempSync, rmSync } from "fs";
10
+ import { join } from "path";
11
+ import { tmpdir } from "os";
12
+ import { db } from "./db.js";
13
+ import { sendStuckNotice, stuckNoticesSentToday, stuckNoticeMaxPerDay, SCHEDULE_STUCK_NOTICE_KIND } from "./scheduled-stuck.js";
14
+
15
+ const DIR = mkdtempSync(join(tmpdir(), "anet-stuck-cap-"));
16
+ const NET = "net_stuck_cap";
17
+ const USER = "u_stuck_cap";
18
+ const SCHED = "sched_stuck_cap_1";
19
+ const OTHER = "sched_stuck_cap_2";
20
+
21
+ function seedNotice(id: string, scheduleId: string, minutesAgo = 1) {
22
+ db.run(
23
+ `INSERT INTO user_inbox (message_id, network_id, user_id, from_session, kind, title, content, severity, meta_json, acked, created_at)
24
+ VALUES (?1, ?2, ?3, 'cap-node', ?4, 't', 'c', 'warning', ?5, 0, datetime('now', ?6))`,
25
+ [id, NET, USER, SCHEDULE_STUCK_NOTICE_KIND,
26
+ JSON.stringify({ schedule_stuck: { reason: "stuck", schedule_id: scheduleId, blocked_by_task_id: "t1", skips: 3 } }),
27
+ `-${minutesAgo} minutes`],
28
+ );
29
+ }
30
+
31
+ const notice = (scheduleId: string) => ({
32
+ reason: "stuck" as const,
33
+ scheduleId,
34
+ scheduleName: "cap-test",
35
+ networkId: NET,
36
+ createdBy: USER,
37
+ alias: "cap-node",
38
+ timezone: "Asia/Shanghai",
39
+ taskId: "t-new",
40
+ taskCreatedAt: "2026-10-28 10:00:00",
41
+ skips: 3,
42
+ timeoutMs: 3_600_000,
43
+ });
44
+
45
+ describe("schedule_stuck 通知每日上限", () => {
46
+ beforeAll(() => {
47
+ db.run("DELETE FROM user_inbox WHERE network_id = ?1", [NET]);
48
+ });
49
+ afterAll(() => {
50
+ db.run("DELETE FROM user_inbox WHERE network_id = ?1", [NET]);
51
+ rmSync(DIR, { recursive: true, force: true });
52
+ });
53
+
54
+ test("默认上限是 4", () => {
55
+ expect(stuckNoticeMaxPerDay()).toBe(4);
56
+ });
57
+
58
+ test("前 N 条照发,第 N+1 条起被抑制", () => {
59
+ const cap = stuckNoticeMaxPerDay();
60
+ for (let i = 0; i < cap; i++) seedNotice(`msg_cap_${i}`, SCHED, i + 1);
61
+ expect(stuckNoticesSentToday(SCHED)).toBe(cap);
62
+ // 已达上限 ⇒ 返回 null(不发),且不会新增行
63
+ expect(sendStuckNotice(notice(SCHED))).toBeNull();
64
+ expect(stuckNoticesSentToday(SCHED)).toBe(cap);
65
+ });
66
+
67
+ test("计数按排程隔离:别的排程不受影响,仍能发", () => {
68
+ for (let i = 0; i < stuckNoticeMaxPerDay(); i++) seedNotice(`msg_cap_o_${i}`, OTHER, i + 1);
69
+ expect(stuckNoticesSentToday(OTHER)).toBe(stuckNoticeMaxPerDay());
70
+ // 另一个排程已满,但本条排程仍是 0 —— 隔离性检查
71
+ expect(stuckNoticesSentToday("sched_stuck_cap_3")).toBe(0);
72
+ });
73
+
74
+ test("超出 24 小时的旧通知不计入当日上限(上限是滚动的,不是历史累计)", () => {
75
+ db.run("DELETE FROM user_inbox WHERE network_id = ?1", [NET]);
76
+ seedNotice("msg_old_1", SCHED, 25 * 60);
77
+ seedNotice("msg_old_2", SCHED, 30 * 60);
78
+ expect(stuckNoticesSentToday(SCHED)).toBe(0);
79
+ });
80
+ });
@@ -169,9 +169,42 @@ export function stuckNoticeText(n: StuckNotice): { title: string; text: string }
169
169
  };
170
170
  }
171
171
 
172
- /** 提交之后调用。没有创建者 / 创建者不在网络里 → 不发。 */
172
+ // #2881 —— 同一个排程的「卡住」通知每天限 N 条。
173
+ //
174
+ // 现场(2026-10-28 只读统计):7 天 1162 条 schedule_stuck 通知,其中单个排程 220 条。
175
+ // 原因不是提醒不够,而是**提醒的设计前提被打破**:本意是「一个新的阻塞任务 = 一个新事件」,
176
+ // 于是每个阻塞者都会重新上膛。但节点如果长期被人占着,每一轮都会产生新的阻塞者,
177
+ // 于是一个永远跑不成的排程就能一天发几十条 —— 结果没人再看。
178
+ //
179
+ // 这里只加天花板,不动「新阻塞者 = 新事件」的语义(前 N 条照发),
180
+ // 也没有任何自动动作:排程状态不受影响。
181
+ export function stuckNoticeMaxPerDay(): number {
182
+ return Math.floor(envNumber("COMMHUB_SCHEDULE_STUCK_NOTICE_MAX_PER_DAY", 4, 1));
183
+ }
184
+
185
+ /** 该排程近 24 小时已发过几条卡住通知。 */
186
+ export function stuckNoticesSentToday(scheduleId: string): number {
187
+ try {
188
+ return db.get<{ n: number }>(
189
+ `SELECT COUNT(*) AS n FROM user_inbox
190
+ WHERE kind = ?1 AND json_extract(meta_json, '$.schedule_stuck.schedule_id') = ?2
191
+ AND created_at >= datetime('now', '-24 hours')`,
192
+ SCHEDULE_STUCK_NOTICE_KIND, scheduleId,
193
+ )?.n ?? 0;
194
+ } catch {
195
+ return 0; // 读不到就不拦(宁可多发一条,也不要因为查询失败而彻底静音)
196
+ }
197
+ }
198
+
199
+ /** 提交之后调用。没有创建者 / 创建者不在网络里 / 当日已超上限 → 不发。 */
173
200
  export function sendStuckNotice(n: StuckNotice): string | null {
174
201
  if (!n.createdBy) return null;
202
+ const cap = stuckNoticeMaxPerDay();
203
+ const sentToday = stuckNoticesSentToday(n.scheduleId);
204
+ if (sentToday >= cap) {
205
+ console.log(`[scheduled-tasks] stuck notice capped schedule=${n.scheduleId} sent_today=${sentToday} cap=${cap} (${n.reason})`);
206
+ return null;
207
+ }
175
208
  try {
176
209
  const { title, text } = stuckNoticeText(n);
177
210
  return sendAgentNotice({
package/src/server.ts CHANGED
@@ -85,6 +85,8 @@ import { sweepEmptyNodeRows } from "./empty-node-sweeper.js";
85
85
  import { versionStats } from "./version-stats.js";
86
86
  import { startStaleSessionSweeper } from "./stale-sweeper.js";
87
87
  import { scanNodeFailureAlerts } from "./node-failure-alert.js";
88
+ import { scanStarvedSchedules } from "./schedule-starved.js";
89
+ import { scanNodeBusyCircuit } from "./node-serviceability.js";
88
90
  import {
89
91
  AUDIT_LOG_REST_SELECT,
90
92
  COMPLETION_REST_SELECT,
@@ -5801,6 +5803,35 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5801
5803
  }, nodeFailureAlertMs);
5802
5804
  (nodeFailureAlertTimer as any).unref?.();
5803
5805
 
5806
+ // #2880 —— 空转排程巡检:默认**只记录**「本来会被暂停哪些」,不动作。
5807
+ // 开启 COMMHUB_SCHEDULE_STARVED_AUTOPAUSE=1 才真暂停(判据见 schedule-starved.ts 头注释)。
5808
+ // 与告警分开:告警解决「没人知道」,这里解决「知道了但一直不停」。
5809
+ const scheduleStarvedTimer = setInterval(() => {
5810
+ try {
5811
+ const r = scanStarvedSchedules();
5812
+ if (r.paused.length > 0) console.log(`[commhub schedule-starved] paused ${r.paused.length}: ${r.paused.join(", ")}`);
5813
+ else if (r.candidates.length > 0) console.log(`[commhub schedule-starved] would pause ${r.candidates.length} (autopause off): ${r.candidates.join(", ")}`);
5814
+ } catch (e: any) {
5815
+ console.log(`[commhub schedule-starved] failed: ${e?.message ?? e}`);
5816
+ }
5817
+ }, nodeFailureAlertMs);
5818
+ (scheduleStarvedTimer as any).unref?.();
5819
+
5820
+ // #2880 —— 节点级「不可服务」检测:窗口内忙类失败多、成功为零 ⇒ 这个节点现在投进去必然白等。
5821
+ // 与排程级分开:排程是「某条排程该停」,这里是「某台节点该暂时不投」。
5822
+ // 默认只记录;开启 COMMHUB_NODE_BUSY_CIRCUIT=1 才用于派发判断。
5823
+ const nodeBusyCircuitTimer = setInterval(() => {
5824
+ try {
5825
+ const r = scanNodeBusyCircuit();
5826
+ if (!r.enabled && r.tripped.length > 0) {
5827
+ console.log(`[commhub node-busy-circuit] would trip ${r.tripped.length} in ${r.windowHours}h (circuit off): ${r.tripped.join(", ")}`);
5828
+ }
5829
+ } catch (e: any) {
5830
+ console.log(`[commhub node-busy-circuit] failed: ${e?.message ?? e}`);
5831
+ }
5832
+ }, nodeFailureAlertMs);
5833
+ (nodeBusyCircuitTimer as any).unref?.();
5834
+
5804
5835
  // #476 — the two legacy module-level intervals, now owned by startHub
5805
5836
  // like every other periodic job: rate-limit map cleanup (memory only)
5806
5837
  // and task expiration patrol (writes DB — must never run on import).