@sleep2agi/commhub-server 0.9.0-preview.190 → 0.9.0-preview.192
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/node-serviceability.test.ts +19 -0
- package/src/node-serviceability.ts +29 -1
- package/src/schedule-starved.test.ts +9 -0
- package/src/schedule-starved.ts +21 -5
- package/src/scheduled-tasks.ts +19 -0
- package/src/server.ts +23 -4
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sleep2agi/commhub-server",
|
|
3
|
-
"version": "0.9.0-preview.
|
|
3
|
+
"version": "0.9.0-preview.192",
|
|
4
4
|
"description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -50,6 +50,25 @@ describe("节点忙熔断:三项判据", () => {
|
|
|
50
50
|
expect(r.enabled).toBe(false); // 默认关闭
|
|
51
51
|
});
|
|
52
52
|
|
|
53
|
+
// 熔断后不能「永不投递」:否则节点拿不到成功记录 ⇒ 永远解不开(死锁)。
|
|
54
|
+
test("熔断期间按探针间隔放行一条(自愈通道)", () => {
|
|
55
|
+
process.env.COMMHUB_NODE_BUSY_CIRCUIT = "1";
|
|
56
|
+
// 黑洞节点:刚投递过 ⇒ 拦
|
|
57
|
+
db.run(`INSERT INTO tasks (task_id, content, from_name, to_name, network_id, status, created_at)
|
|
58
|
+
VALUES ('t_probe_now','x','hub','黑洞节点',?1,'failed',datetime('now'))`, [NET]);
|
|
59
|
+
const blocked = nodeCanAcceptDispatch("黑洞节点", NET);
|
|
60
|
+
expect(blocked.ok).toBe(false);
|
|
61
|
+
expect(blocked.probe).toBeUndefined();
|
|
62
|
+
// 把最近投递推到 2 小时前(默认探针间隔 1h)⇒ 放行,且标记 probe
|
|
63
|
+
// 注意:前面的用例也给这个节点插过 created_at=now 的任务,MAX(created_at) 会取到它们
|
|
64
|
+
db.run("UPDATE tasks SET created_at = datetime('now','-2 hours') WHERE to_name = '黑洞节点'");
|
|
65
|
+
const probe = nodeCanAcceptDispatch("黑洞节点", NET);
|
|
66
|
+
expect(probe.ok).toBe(true);
|
|
67
|
+
expect(probe.probe).toBe(true);
|
|
68
|
+
// 还原,避免影响后续用例(它们依赖「最近刚投递过」这个前态)
|
|
69
|
+
db.run("UPDATE tasks SET created_at = datetime('now') WHERE to_name = '黑洞节点'");
|
|
70
|
+
});
|
|
71
|
+
|
|
53
72
|
test("默认关闭时投递判断恒放行(不改任何既有行为)", () => {
|
|
54
73
|
delete process.env.COMMHUB_NODE_BUSY_CIRCUIT;
|
|
55
74
|
expect(circuitEnabled()).toBe(false);
|
|
@@ -31,6 +31,28 @@ export function circuitEnabled(): boolean {
|
|
|
31
31
|
}
|
|
32
32
|
export function circuitWindowHours(): number { return envInt("COMMHUB_NODE_BUSY_CIRCUIT_HOURS", 6, 1); }
|
|
33
33
|
export function circuitMinFails(): number { return envInt("COMMHUB_NODE_BUSY_CIRCUIT_FAILS", 5, 1); }
|
|
34
|
+
/**
|
|
35
|
+
* 熔断期间**放探针**的间隔(小时)。
|
|
36
|
+
*
|
|
37
|
+
* 🔴 为什么必须有:熔断后完全不再投递 ⇒ 节点永远拿不到成功记录 ⇒ 永远解不开(死锁)。
|
|
38
|
+
* 所以每 N 小时放一条探针任务:它若成功,replied 计数不再为 0,熔断自动解除。
|
|
39
|
+
*/
|
|
40
|
+
export function circuitProbeHours(): number { return envInt("COMMHUB_NODE_BUSY_CIRCUIT_PROBE_HOURS", 1, 1); }
|
|
41
|
+
|
|
42
|
+
/** 该节点最近一次**被投递**的时刻(ms)。没有投递过返回 null。 */
|
|
43
|
+
function lastDeliveryMs(alias: string, networkId: string | null): number | null {
|
|
44
|
+
try {
|
|
45
|
+
const row = db.get<{ t: string | null }>(
|
|
46
|
+
"SELECT MAX(created_at) AS t FROM tasks WHERE to_name = ?1 AND (?2 IS NULL OR network_id = ?2)",
|
|
47
|
+
alias, networkId,
|
|
48
|
+
);
|
|
49
|
+
if (!row?.t) return null;
|
|
50
|
+
const ms = Date.parse(String(row.t).replace(" ", "T") + "Z");
|
|
51
|
+
return Number.isFinite(ms) ? ms : null;
|
|
52
|
+
} catch {
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
34
56
|
|
|
35
57
|
export type NodeServingStats = { alias: string; network_id: string | null; delivered: number; replied: number; busy_failed: number };
|
|
36
58
|
|
|
@@ -69,7 +91,7 @@ export function scanNodeBusyCircuit(now = new Date()): CircuitScanResult {
|
|
|
69
91
|
* 这个节点现在能不能投递。默认永远 true(未开启 = 不拦截)。
|
|
70
92
|
* 供排程派发判断;纯读,不改任何状态。
|
|
71
93
|
*/
|
|
72
|
-
export function nodeCanAcceptDispatch(alias: string, networkId: string | null): { ok: boolean; reason?: string; busy_failed?: number } {
|
|
94
|
+
export function nodeCanAcceptDispatch(alias: string, networkId: string | null): { ok: boolean; reason?: string; busy_failed?: number; probe?: boolean } {
|
|
73
95
|
if (!circuitEnabled()) return { ok: true };
|
|
74
96
|
const windowHours = circuitWindowHours();
|
|
75
97
|
const since = new Date(Date.now() - windowHours * 3_600_000).toISOString().slice(0, 19).replace("T", " ");
|
|
@@ -88,6 +110,12 @@ export function nodeCanAcceptDispatch(alias: string, networkId: string | null):
|
|
|
88
110
|
if (!row) return { ok: true };
|
|
89
111
|
const stats = { delivered: Number(row.delivered ?? 0), replied: Number(row.replied ?? 0), busy_failed: Number(row.busy_failed ?? 0) };
|
|
90
112
|
if (!isNotServing(stats)) return { ok: true };
|
|
113
|
+
// 熔断期间每 N 小时放一条探针,否则节点永远拿不到成功记录、永远解不开。
|
|
114
|
+
const probeHours = circuitProbeHours();
|
|
115
|
+
const last = lastDeliveryMs(alias, networkId);
|
|
116
|
+
if (last === null || Date.now() - last >= probeHours * 3_600_000) {
|
|
117
|
+
return { ok: true, probe: true, busy_failed: stats.busy_failed };
|
|
118
|
+
}
|
|
91
119
|
return {
|
|
92
120
|
ok: false,
|
|
93
121
|
reason: `节点近 ${windowHours}h 内 ${stats.busy_failed} 次忙类失败、0 次成功(投递也只会白等),已跳过`,
|
|
@@ -45,6 +45,15 @@ describe("空转排程判定与自动暂停", () => {
|
|
|
45
45
|
expect(isStarved({ total: 129, replied: 1 }, 20)).toBe(false); // 只要成功过一次就不算
|
|
46
46
|
});
|
|
47
47
|
|
|
48
|
+
// 2026-10-28 生产实测:6 个候选里 4 个的 79 次运行全是 target_offline ——
|
|
49
|
+
// 它们在等节点回来,暂停它们反而要多一次人工恢复。
|
|
50
|
+
test("目标离线占比过高 ⇒ 不算空转(那是在等节点,不是排程有问题)", () => {
|
|
51
|
+
expect(isStarved({ total: 79, replied: 0, offline_skips: 79 }, 20)).toBe(false); // 100% 离线
|
|
52
|
+
expect(isStarved({ total: 100, replied: 0, offline_skips: 91 }, 20)).toBe(false); // >90%
|
|
53
|
+
expect(isStarved({ total: 100, replied: 0, offline_skips: 89 }, 20)).toBe(true); // <90% 仍算空转
|
|
54
|
+
expect(isStarved({ total: 94, replied: 0, offline_skips: 0 }, 20)).toBe(true); // 纯 busy/失败型
|
|
55
|
+
});
|
|
56
|
+
|
|
48
57
|
test("默认关闭:只报候选,不改排程状态", () => {
|
|
49
58
|
delete process.env.COMMHUB_SCHEDULE_STARVED_AUTOPAUSE;
|
|
50
59
|
mkSchedule("sched_starved_a", "空转A");
|
package/src/schedule-starved.ts
CHANGED
|
@@ -33,11 +33,26 @@ export function autopauseEnabled(): boolean {
|
|
|
33
33
|
export function starvedMinRuns(): number { return envInt("COMMHUB_SCHEDULE_STARVED_MIN_RUNS", 20, 1); }
|
|
34
34
|
export function starvedWindowHours(): number { return envInt("COMMHUB_SCHEDULE_STARVED_WINDOW_HOURS", 24, 1); }
|
|
35
35
|
|
|
36
|
-
export type ScheduleStats = { schedule_id: string; total: number; replied: number };
|
|
36
|
+
export type ScheduleStats = { schedule_id: string; total: number; replied: number; offline_skips: number };
|
|
37
37
|
|
|
38
|
-
/**
|
|
39
|
-
export
|
|
40
|
-
|
|
38
|
+
/** 目标不可达占比超过这个比例 ⇒ 不算空转(是在等节点回来,不是排程有问题)。 */
|
|
39
|
+
export const OFFLINE_SKIP_RATIO_MAX = 0.9;
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* 纯函数:是否判定为空转。便于单测,不做任何 IO。
|
|
43
|
+
*
|
|
44
|
+
* 🔴 2026-10-28 生产实测修正:只看「执行≥N 且成功=0」会把「等离线节点」误判成空转。
|
|
45
|
+
* 那天 6 个候选里 4 个的 79 次运行**全部是 skipped/target_offline** —— 它们没建过任务、
|
|
46
|
+
* 没浪费等待,节点一回来就会自己接着跑。把它们暂停,反而要多一次人工恢复。
|
|
47
|
+
* ⇒ target_offline 占比 > 90% 视为「目标不可达」,不算空转。
|
|
48
|
+
*/
|
|
49
|
+
export function isStarved(
|
|
50
|
+
s: { total: number; replied: number; offline_skips?: number },
|
|
51
|
+
minRuns = starvedMinRuns(),
|
|
52
|
+
): boolean {
|
|
53
|
+
if (s.total < minRuns || s.replied !== 0) return false;
|
|
54
|
+
const offline = s.offline_skips ?? 0;
|
|
55
|
+
return offline <= s.total * OFFLINE_SKIP_RATIO_MAX;
|
|
41
56
|
}
|
|
42
57
|
|
|
43
58
|
export type StarvedScanResult = { paused: string[]; candidates: string[]; enabled: boolean };
|
|
@@ -52,7 +67,8 @@ export function scanStarvedSchedules(now = new Date()): StarvedScanResult {
|
|
|
52
67
|
const rows = db.all<ScheduleStats>(
|
|
53
68
|
`SELECT schedule_id,
|
|
54
69
|
COUNT(*) AS total,
|
|
55
|
-
SUM(CASE WHEN status = 'replied' THEN 1 ELSE 0 END) AS replied
|
|
70
|
+
SUM(CASE WHEN status = 'replied' THEN 1 ELSE 0 END) AS replied,
|
|
71
|
+
SUM(CASE WHEN status = 'skipped' AND error_code = 'target_offline' THEN 1 ELSE 0 END) AS offline_skips
|
|
56
72
|
FROM scheduled_task_runs
|
|
57
73
|
WHERE created_at >= ?1
|
|
58
74
|
GROUP BY schedule_id`,
|
package/src/scheduled-tasks.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { sessionTaskOnDispatch } from "./session-task-on-dispatch.js";
|
|
|
6
6
|
import { TASK_CONTENT_MAX, sessionTaskPreview } from "./shared/task-content-limit.js";
|
|
7
7
|
import { assertNodeActive } from "./lifecycle-guard.js";
|
|
8
8
|
import { assertNodeHealthy } from "./node-health-guard.js";
|
|
9
|
+
import { nodeCanAcceptDispatch } from "./node-serviceability.js";
|
|
9
10
|
import { addNetworkScope, canRestWriteNetwork, singleNetworkId, type RestNetworkScope } from "./network-scope.js";
|
|
10
11
|
import { canMessageAgent, type AgentRef } from "./agent-access.js";
|
|
11
12
|
import { dispatchVerdict } from "./node-permissions.js";
|
|
@@ -587,6 +588,24 @@ export function dispatchScheduledOccurrence(row: ScheduledRow, scheduledFor: str
|
|
|
587
588
|
return;
|
|
588
589
|
}
|
|
589
590
|
|
|
591
|
+
// #2880 —— 节点级忙熔断(默认关闭;COMMHUB_NODE_BUSY_CIRCUIT=1 才生效)。
|
|
592
|
+
// 与 lifecycle/health 的区别:节点没坏、也没停,只是最近一直没有产出(忙类失败多且 0 成功),
|
|
593
|
+
// 投进去必然白等 ~30 分钟。所以记 **skipped** 而不是 failed —— 这不是故障,是「现在别投」。
|
|
594
|
+
// 熔断期间每 COMMHUB_NODE_BUSY_CIRCUIT_PROBE_HOURS 小时会放一条探针(见 node-serviceability.ts),
|
|
595
|
+
// 避免「不投递 ⇒ 拿不到成功 ⇒ 永远解不开」的死锁。
|
|
596
|
+
{
|
|
597
|
+
const circuit = nodeCanAcceptDispatch(node.alias, row.network_id);
|
|
598
|
+
if (!circuit.ok) {
|
|
599
|
+
finalStatus = "skipped";
|
|
600
|
+
db.run(
|
|
601
|
+
"UPDATE scheduled_task_runs SET status = 'skipped', error_code = 'target_busy_no_output', error_message = ?1, completed_at = datetime('now') WHERE run_id = ?2",
|
|
602
|
+
[String(circuit.reason ?? "target_busy_no_output").slice(0, 500), runId],
|
|
603
|
+
);
|
|
604
|
+
if (advanceSchedule) advance(row, scheduledFor, advanceAfter);
|
|
605
|
+
return;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
|
|
590
609
|
// #460 — the target reports a degraded layer (App Server down / TUI gone / login revoked): the run fails
|
|
591
610
|
// with that reason instead of queueing a task nobody will execute. Unknown / stale health runs as before.
|
|
592
611
|
const health = assertNodeHealthy(node.alias, row.network_id, { forceAllowed: false });
|
package/src/server.ts
CHANGED
|
@@ -5806,11 +5806,23 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
|
|
|
5806
5806
|
// #2880 —— 空转排程巡检:默认**只记录**「本来会被暂停哪些」,不动作。
|
|
5807
5807
|
// 开启 COMMHUB_SCHEDULE_STARVED_AUTOPAUSE=1 才真暂停(判据见 schedule-starved.ts 头注释)。
|
|
5808
5808
|
// 与告警分开:告警解决「没人知道」,这里解决「知道了但一直不停」。
|
|
5809
|
+
// 🔴 「只在候选集合变化时打一行」——每 5 分钟打一次等于 288 行/天,
|
|
5810
|
+
// 那正是这套机制要治的噪音(2026-10-28 上线 15 分钟就打了 8 行才被发现)。
|
|
5811
|
+
let lastStarvedKey = "";
|
|
5809
5812
|
const scheduleStarvedTimer = setInterval(() => {
|
|
5810
5813
|
try {
|
|
5811
5814
|
const r = scanStarvedSchedules();
|
|
5812
|
-
if (r.paused.length > 0)
|
|
5813
|
-
|
|
5815
|
+
if (r.paused.length > 0) {
|
|
5816
|
+
console.log(`[commhub schedule-starved] paused ${r.paused.length}: ${r.paused.join(", ")}`);
|
|
5817
|
+
} else {
|
|
5818
|
+
const key = r.candidates.slice().sort().join(",");
|
|
5819
|
+
if (r.candidates.length > 0 && key !== lastStarvedKey) {
|
|
5820
|
+
console.log(`[commhub schedule-starved] would pause ${r.candidates.length} (autopause off): ${r.candidates.join(", ")}`);
|
|
5821
|
+
} else if (r.candidates.length === 0 && lastStarvedKey !== "") {
|
|
5822
|
+
console.log(`[commhub schedule-starved] 候选已清空(之前 ${lastStarvedKey.split(",").length} 个)`);
|
|
5823
|
+
}
|
|
5824
|
+
lastStarvedKey = key;
|
|
5825
|
+
}
|
|
5814
5826
|
} catch (e: any) {
|
|
5815
5827
|
console.log(`[commhub schedule-starved] failed: ${e?.message ?? e}`);
|
|
5816
5828
|
}
|
|
@@ -5820,11 +5832,18 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
|
|
|
5820
5832
|
// #2880 —— 节点级「不可服务」检测:窗口内忙类失败多、成功为零 ⇒ 这个节点现在投进去必然白等。
|
|
5821
5833
|
// 与排程级分开:排程是「某条排程该停」,这里是「某台节点该暂时不投」。
|
|
5822
5834
|
// 默认只记录;开启 COMMHUB_NODE_BUSY_CIRCUIT=1 才用于派发判断。
|
|
5835
|
+
let lastCircuitKey = "";
|
|
5823
5836
|
const nodeBusyCircuitTimer = setInterval(() => {
|
|
5824
5837
|
try {
|
|
5825
5838
|
const r = scanNodeBusyCircuit();
|
|
5826
|
-
if (!r.enabled
|
|
5827
|
-
|
|
5839
|
+
if (!r.enabled) {
|
|
5840
|
+
const key = r.tripped.slice().sort().join(",");
|
|
5841
|
+
if (r.tripped.length > 0 && key !== lastCircuitKey) {
|
|
5842
|
+
console.log(`[commhub node-busy-circuit] would trip ${r.tripped.length} in ${r.windowHours}h (circuit off): ${r.tripped.join(", ")}`);
|
|
5843
|
+
} else if (r.tripped.length === 0 && lastCircuitKey !== "") {
|
|
5844
|
+
console.log(`[commhub node-busy-circuit] 候选已清空(之前 ${lastCircuitKey.split(",").length} 个)`);
|
|
5845
|
+
}
|
|
5846
|
+
lastCircuitKey = key;
|
|
5828
5847
|
}
|
|
5829
5848
|
} catch (e: any) {
|
|
5830
5849
|
console.log(`[commhub node-busy-circuit] failed: ${e?.message ?? e}`);
|