@sleep2agi/commhub-server 0.9.0-preview.161 → 0.9.0-preview.162

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sleep2agi/commhub-server",
3
- "version": "0.9.0-preview.161",
3
+ "version": "0.9.0-preview.162",
4
4
  "description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
5
5
  "type": "module",
6
6
  "main": "src/index.ts",
package/src/db-adapter.ts CHANGED
@@ -61,6 +61,36 @@ export const PG_EXPERIMENTAL_ENV = "COMMHUB_PG_EXPERIMENTAL";
61
61
  // SQLite Adapter (bun:sqlite, sync)
62
62
  // ════════════════════════════════════════════
63
63
 
64
+ // #2614 —— 慢 SQL 观测。SQLite 是**同步**的:一条慢查询会直接卡住整个事件循环
65
+ // (生产实测 2026-10-26 每小时一次的 8–11 秒停顿就是 retention 的全表扫描,
66
+ // 症状是日志静默 + 在飞数暴涨 + RSS 冲到 4.5 GB)。之前没有任何一条日志能指出
67
+ // “是这条 SQL 卡了 11 秒”,只能靠事后比对日志静默的时间窗手工推断。
68
+ //
69
+ // 这里只在**超过阈值**时打一行日志,不含参数值(避免把令牌/正文写进日志)。
70
+ // `COMMHUB_SLOW_SQL_MS` 控制阈值(默认 1000 ms,设 0 关闭)。
71
+ const DEFAULT_SLOW_SQL_MS = 1000;
72
+
73
+ export function slowSqlThresholdMs(): number {
74
+ const raw = Number(process.env.COMMHUB_SLOW_SQL_MS);
75
+ return Number.isFinite(raw) && raw >= 0 ? raw : DEFAULT_SLOW_SQL_MS;
76
+ }
77
+
78
+ /** 截断 + 压平 SQL,只保留足够定位的头部(不包含绑定参数)。 */
79
+ export function formatSlowSql(ms: number, sql: string): string {
80
+ const oneLine = sql.replace(/\s+/g, " ").trim().slice(0, 160);
81
+ return `[commhub slow-sql] ${ms.toFixed(0)}ms ${oneLine}`;
82
+ }
83
+
84
+ let sqlTimingDisabled: boolean | null = null;
85
+
86
+ function logIfSlow(started: number, sql: string): void {
87
+ if (sqlTimingDisabled === null) sqlTimingDisabled = slowSqlThresholdMs() <= 0;
88
+ if (sqlTimingDisabled) return;
89
+ const ms = performance.now() - started;
90
+ if (ms < slowSqlThresholdMs()) return;
91
+ console.log(formatSlowSql(ms, sql));
92
+ }
93
+
64
94
  export class SQLiteAdapter implements DbAdapter {
65
95
  readonly dialect = "sqlite" as const;
66
96
  readonly atomicTransactions = true;
@@ -70,19 +100,39 @@ export class SQLiteAdapter implements DbAdapter {
70
100
  constructor(private readonly rawDb: Database) {}
71
101
 
72
102
  run(sql: string, params?: any[]): QueryResult {
73
- return params ? this.rawDb.run(sql, params as any) : this.rawDb.run(sql);
103
+ const t0 = performance.now();
104
+ try {
105
+ return params ? this.rawDb.run(sql, params as any) : this.rawDb.run(sql);
106
+ } finally {
107
+ logIfSlow(t0, sql);
108
+ }
74
109
  }
75
110
 
76
111
  get<T = any>(sql: string, ...params: any[]): T | null {
77
- return this.rawDb.query<T, any[]>(sql).get(...params) ?? null;
112
+ const t0 = performance.now();
113
+ try {
114
+ return this.rawDb.query<T, any[]>(sql).get(...params) ?? null;
115
+ } finally {
116
+ logIfSlow(t0, sql);
117
+ }
78
118
  }
79
119
 
80
120
  all<T = any>(sql: string, ...params: any[]): T[] {
81
- return this.rawDb.query<T, any[]>(sql).all(...params);
121
+ const t0 = performance.now();
122
+ try {
123
+ return this.rawDb.query<T, any[]>(sql).all(...params);
124
+ } finally {
125
+ logIfSlow(t0, sql);
126
+ }
82
127
  }
83
128
 
84
129
  exec(sql: string): void {
85
- this.rawDb.exec(sql);
130
+ const t0 = performance.now();
131
+ try {
132
+ this.rawDb.exec(sql);
133
+ } finally {
134
+ logIfSlow(t0, sql);
135
+ }
86
136
  }
87
137
 
88
138
  transaction<T>(fn: () => T): T {
package/src/db.ts CHANGED
@@ -174,6 +174,13 @@ db.exec(`
174
174
 
175
175
  CREATE INDEX IF NOT EXISTS idx_agent_telemetry_alias_time
176
176
  ON agent_telemetry(network_id, alias, created_at);
177
+
178
+ -- #2614 —— retention 每小时跑一次 DELETE FROM agent_telemetry WHERE created_at < ?。
179
+ -- 上面三个复合索引都以 network_id 打头,**这个查询用不上**(没有 network_id 等值条件),
180
+ -- 于是每次扫全表 62 万行 → 生产实测每小时出现 **8–11 秒的事件循环阻塞**
181
+ -- ([commhub retention] swept in 8331ms / 9863ms / 10588ms,同时日志静默 11 秒)。
182
+ -- 单独给 created_at 建索引,让这个扫描变成范围 seek。
183
+ CREATE INDEX IF NOT EXISTS idx_agent_telemetry_created ON agent_telemetry(created_at);
177
184
  `);
178
185
 
179
186
  // Round-2/4 review ② — server-health index split.
@@ -0,0 +1,127 @@
1
+ // #2614 —— 事件循环停顿监测 + 慢 SQL 日志的单元测试。
2
+ //
3
+ // 这两个东西的价值不在“能测出停顿”,而在**停顿发生时留下可对齐的证据**:
4
+ // 2026-10-26 生产两次 RSS 3.6/4.5 GB 的尖峰,事后只能靠“日志静默几十秒”
5
+ // 手工推断,`/api/stats/routes` 只给出受害者请求(含排队时间)。
6
+ // 所以测试要钉住的是语义:怎么算一次停顿、留什么档、日志阈值怎么生效、关闭后还计数。
7
+
8
+ import { describe, expect, test } from "bun:test";
9
+ import {
10
+ eventLoopStats,
11
+ recordGap,
12
+ resetEventLoopLagForTests,
13
+ startEventLoopLagMonitor,
14
+ } from "./event-loop-lag.js";
15
+ import { formatSlowSql, slowSqlThresholdMs } from "./db-adapter.js";
16
+
17
+ describe("event-loop-lag: recordGap", () => {
18
+ test("正常 tick(≈interval)不算停顿", () => {
19
+ resetEventLoopLagForTests();
20
+ expect(recordGap(260, { intervalMs: 250, thresholdMs: 2000 })).toBe(10);
21
+ const s = eventLoopStats();
22
+ expect(s.stall_count).toBe(0);
23
+ expect(s.max_gap_ms).toBe(260);
24
+ expect(s.recent_stalls).toEqual([]);
25
+ });
26
+
27
+ test("超过阈值的 tick 记一条停顿,带 stall_ms / rss_mb / at", () => {
28
+ resetEventLoopLagForTests();
29
+ const now = Date.parse("2026-10-26T22:10:41Z");
30
+ const stall = recordGap(28_000, { intervalMs: 250, thresholdMs: 2000, rssMb: 4502, now });
31
+ expect(stall).toBe(27_750);
32
+ const s = eventLoopStats();
33
+ expect(s.stall_count).toBe(1);
34
+ expect(s.last_stall_ms).toBe(27_750);
35
+ expect(s.last_stall_at).toBe("2026-10-26T22:10:41.000Z");
36
+ expect(s.max_gap_ms).toBe(28_000);
37
+ expect(s.recent_stalls[0]).toMatchObject({ gap_ms: 28_000, stall_ms: 27_750, rss_mb: 4502 });
38
+ });
39
+
40
+ test("recent_stalls 最新在前,且只保留 20 条", () => {
41
+ resetEventLoopLagForTests();
42
+ for (let i = 0; i < 25; i++) {
43
+ recordGap(5_000 + i, { intervalMs: 250, thresholdMs: 2_000, rssMb: 300, now: 1_700_000_000_000 + i * 1_000 });
44
+ }
45
+ const s = eventLoopStats();
46
+ expect(s.stall_count).toBe(25);
47
+ expect(s.recent_stalls.length).toBe(20);
48
+ // 最新一条是 i=24
49
+ expect(s.recent_stalls[0].gap_ms).toBe(5_024);
50
+ expect(s.recent_stalls[19].gap_ms).toBe(5_005);
51
+ });
52
+
53
+ test("max_gap_ms 保留历史最大值(回落后不被覆盖)", () => {
54
+ resetEventLoopLagForTests();
55
+ recordGap(47_000, { intervalMs: 250, thresholdMs: 2_000 });
56
+ recordGap(260, { intervalMs: 250, thresholdMs: 2_000 });
57
+ const s = eventLoopStats();
58
+ expect(s.max_gap_ms).toBe(47_000);
59
+ expect(s.last_gap_ms).toBe(260);
60
+ });
61
+ });
62
+
63
+ describe("event-loop-lag: startEventLoopLagMonitor", () => {
64
+ test("阈值 0 时不打日志,但仍然计数", async () => {
65
+ resetEventLoopLagForTests();
66
+ const logs: string[] = [];
67
+ const timer = startEventLoopLagMonitor({
68
+ intervalMs: 20,
69
+ thresholdMs: 0,
70
+ log: (m) => logs.push(m),
71
+ rssMb: () => 123,
72
+ });
73
+ // 同步阻塞 80ms,让下一次 tick 一定超阈值
74
+ const until = performance.now() + 80;
75
+ while (performance.now() < until) { /* spin */ }
76
+ await Bun.sleep(60);
77
+ clearInterval(timer);
78
+ expect(logs).toEqual([]);
79
+ expect(eventLoopStats().max_gap_ms).toBeGreaterThan(0);
80
+ });
81
+
82
+ test("真实停顿会打一行 [commhub stall] 且带 rss", async () => {
83
+ resetEventLoopLagForTests();
84
+ const logs: string[] = [];
85
+ const timer = startEventLoopLagMonitor({
86
+ intervalMs: 20,
87
+ thresholdMs: 40,
88
+ log: (m) => logs.push(m),
89
+ rssMb: () => 4502,
90
+ });
91
+ const until = performance.now() + 120;
92
+ while (performance.now() < until) { /* spin 阻塞事件循环 */ }
93
+ await Bun.sleep(60);
94
+ clearInterval(timer);
95
+ expect(logs.length).toBeGreaterThan(0);
96
+ expect(logs[0]).toStartWith("[commhub stall] loop blocked ~");
97
+ expect(logs[0]).toContain("rss 4502 MB");
98
+ });
99
+ });
100
+
101
+ describe("db-adapter: 慢 SQL 日志", () => {
102
+ test("默认阈值 1000ms;环境变量可调,0 表示关闭", () => {
103
+ const prev = process.env.COMMHUB_SLOW_SQL_MS;
104
+ try {
105
+ delete process.env.COMMHUB_SLOW_SQL_MS;
106
+ expect(slowSqlThresholdMs()).toBe(1000);
107
+ process.env.COMMHUB_SLOW_SQL_MS = "250";
108
+ expect(slowSqlThresholdMs()).toBe(250);
109
+ process.env.COMMHUB_SLOW_SQL_MS = "0";
110
+ expect(slowSqlThresholdMs()).toBe(0);
111
+ process.env.COMMHUB_SLOW_SQL_MS = "not-a-number";
112
+ expect(slowSqlThresholdMs()).toBe(1000);
113
+ } finally {
114
+ if (prev === undefined) delete process.env.COMMHUB_SLOW_SQL_MS;
115
+ else process.env.COMMHUB_SLOW_SQL_MS = prev;
116
+ }
117
+ });
118
+
119
+ test("日志压平换行并截断,且不含绑定参数", () => {
120
+ const sql = "DELETE FROM agent_telemetry\n WHERE created_at < datetime('now', ?1)";
121
+ const line = formatSlowSql(10_588.4, sql);
122
+ expect(line).toStartWith("[commhub slow-sql] 10588ms DELETE FROM agent_telemetry WHERE created_at < datetime('now', ?1)");
123
+ expect(line).not.toContain("\n");
124
+ const long = formatSlowSql(1_200, "SELECT " + "x".repeat(500));
125
+ expect(long.length).toBeLessThanOrEqual("[commhub slow-sql] 1200ms ".length + 160);
126
+ });
127
+ });
@@ -0,0 +1,122 @@
1
+ // #2614 —— 事件循环停顿监测(只观测,不改任何行为)。
2
+ //
3
+ // 背景:2026-10-26 21:55–22:12 生产 Hub 出现两次内存尖峰(RSS 3.6 → 4.5 GB),
4
+ // 伴随机 `mcp_inflight` 387 → 807。事后看日志才发现**同一时刻日志静默 14–47 秒**
5
+ // ——Hub 是单线程(Bun + 同步 SQLite),日志静默就等于事件循环被阻塞。
6
+ //
7
+ // 现有 `/api/stats/routes` 只能告诉我们“哪个请求**受害**了”(它的耗时包含了排队时间),
8
+ // 不能告诉我们“当时**谁**在阻塞”。缺的就是停顿本身的时间序列。
9
+ //
10
+ // 这个模块补上这一格:每 `intervalMs` 醒一次,量 `now - lastTick`;
11
+ // 正常情况 ≈ interval,超出 `thresholdMs` 的部分就是一次真实停顿。
12
+ // 停顿会:① 计数/留档(`/health` 的 `event_loop` 字段可读);
13
+ // ② 打一行日志(带当时的 RSS,方便和内存曲线对齐)。
14
+ //
15
+ // 注意:`setInterval` 自己也会被阻塞 —— 这正是我们要测量的东西。
16
+ // 停顿结束后第一次 tick 的 gap 就是停顿长度,不需要外部进程配合。
17
+
18
+ export type EventLoopStall = {
19
+ /** ISO 时间(停顿**结束**的时刻,即这次 tick 的时间)。 */
20
+ at: string;
21
+ /** 距上次 tick 的实际间隔(毫秒)。等于 interval + 停顿。 */
22
+ gap_ms: number;
23
+ /** 停顿量 = gap - interval。 */
24
+ stall_ms: number;
25
+ /** 停顿结束时的进程 RSS(MB),用于和内存尖峰对齐。 */
26
+ rss_mb: number;
27
+ };
28
+
29
+ const KEEP_STALLS = 20;
30
+
31
+ let maxGapMs = 0;
32
+ let lastGapMs = 0;
33
+ let stallCount = 0;
34
+ let lastStallAt: string | null = null;
35
+ let lastStallMs = 0;
36
+ const stalls: EventLoopStall[] = [];
37
+
38
+ /** 记录一次 tick 间隔。返回这次 tick 的停顿量(毫秒,可为负→归零)。 */
39
+ export function recordGap(
40
+ gapMs: number,
41
+ opts: { intervalMs?: number; thresholdMs?: number; rssMb?: number; now?: number } = {},
42
+ ): number {
43
+ const interval = opts.intervalMs ?? 250;
44
+ const threshold = opts.thresholdMs ?? 2000;
45
+ lastGapMs = gapMs;
46
+ if (gapMs > maxGapMs) maxGapMs = gapMs;
47
+ const stallMs = Math.max(0, gapMs - interval);
48
+ if (stallMs < threshold) return stallMs;
49
+ const at = new Date(opts.now ?? Date.now()).toISOString();
50
+ stallCount++;
51
+ lastStallAt = at;
52
+ lastStallMs = stallMs;
53
+ stalls.push({ at, gap_ms: Math.round(gapMs), stall_ms: Math.round(stallMs), rss_mb: opts.rssMb ?? 0 });
54
+ while (stalls.length > KEEP_STALLS) stalls.shift();
55
+ return stallMs;
56
+ }
57
+
58
+ export function eventLoopStats(): {
59
+ max_gap_ms: number;
60
+ last_gap_ms: number;
61
+ stall_count: number;
62
+ last_stall_ms: number;
63
+ last_stall_at: string | null;
64
+ recent_stalls: EventLoopStall[];
65
+ } {
66
+ return {
67
+ max_gap_ms: Math.round(maxGapMs),
68
+ last_gap_ms: Math.round(lastGapMs),
69
+ stall_count: stallCount,
70
+ last_stall_ms: Math.round(lastStallMs),
71
+ last_stall_at: lastStallAt,
72
+ recent_stalls: [...stalls].reverse(),
73
+ };
74
+ }
75
+
76
+ /** 只给测试用:清空累计状态。 */
77
+ export function resetEventLoopLagForTests(): void {
78
+ maxGapMs = 0;
79
+ lastGapMs = 0;
80
+ stallCount = 0;
81
+ lastStallAt = null;
82
+ lastStallMs = 0;
83
+ stalls.length = 0;
84
+ }
85
+
86
+ function envMs(name: string, fallback: number): number {
87
+ const raw = Number(process.env[name]);
88
+ return Number.isFinite(raw) && raw >= 0 ? raw : fallback;
89
+ }
90
+
91
+ /**
92
+ * 启动监测。返回 timer(调用方负责 clearInterval)。
93
+ * - `COMMHUB_STALL_LOG_MS`:停顿日志阈值,默认 2000 ms,设 0 关闭日志(仍计数);
94
+ * - `COMMHUB_STALL_TICK_MS`:采样间隔,默认 250 ms。
95
+ */
96
+ export function startEventLoopLagMonitor(opts: {
97
+ intervalMs?: number;
98
+ thresholdMs?: number;
99
+ log?: (msg: string) => void;
100
+ rssMb?: () => number;
101
+ } = {}): ReturnType<typeof setInterval> {
102
+ const intervalMs = opts.intervalMs ?? Math.max(50, envMs("COMMHUB_STALL_TICK_MS", 250));
103
+ const thresholdMs = opts.thresholdMs ?? envMs("COMMHUB_STALL_LOG_MS", 2000);
104
+ const log = opts.log ?? ((m: string) => console.log(m));
105
+ const rssMb = opts.rssMb ?? (() => Math.round(process.memoryUsage().rss / 1048576));
106
+ let last = performance.now();
107
+ const timer = setInterval(() => {
108
+ const now = performance.now();
109
+ const gap = now - last;
110
+ last = now;
111
+ const rss = rssMb();
112
+ const stall = recordGap(gap, { intervalMs, thresholdMs, rssMb: rss });
113
+ if (thresholdMs > 0 && stall >= thresholdMs) {
114
+ // 和 retention/slow-sql 的日志同一前缀,方便 grep。
115
+ log(`[commhub stall] loop blocked ~${Math.round(stall)}ms (tick gap ${Math.round(gap)}ms, rss ${rss} MB)`);
116
+ }
117
+ }, intervalMs);
118
+ // 监测不该拖住进程退出。
119
+ (timer as unknown as { unref?: () => void }).unref?.();
120
+ last = performance.now();
121
+ return timer;
122
+ }
@@ -107,10 +107,16 @@ describe("#473 requirements_people", () => {
107
107
  test("compact rows: username, display_name, role, department, owned agents; unowned agents listed once", async () => {
108
108
  const r = await asBoss.call("requirements_people", { network_id: netA, q: "alice" });
109
109
  expect(r.ok).toBe(true);
110
- expect(r.people).toEqual([{
110
+ // #2596 —— Agent 行新增 updated_at / lifecycle_state(客户端用它做 alias 去重与“近 7 天”过滤)。
111
+ // 形状断言改为「关键字段匹配 + 允许新增键」,而不是整行严格相等。
112
+ expect(r.people).toHaveLength(1);
113
+ expect(r.people[0]).toMatchObject({
111
114
  user_id: alice.id, username: alice.username, display_name: "爱丽丝", role: "member",
112
- department: { id: deptId, name: `研发部${stamp % 1000}` }, agents: [{ node_id: node("alice"), alias: ALIAS.alice }],
113
- }]);
115
+ department: { id: deptId, name: `研发部${stamp % 1000}` },
116
+ });
117
+ expect(r.people[0].agents).toHaveLength(1);
118
+ expect(r.people[0].agents[0]).toMatchObject({ node_id: node("alice"), alias: ALIAS.alice });
119
+ expect(r.people[0].agents[0]).toHaveProperty("updated_at");
114
120
  const all = await asBoss.call("requirements_people", { network_id: netA, q: String(stamp) });
115
121
  expect(all.agents_without_owner.map((a: any) => a.alias).sort()).toEqual([ALIAS.dup, ALIAS.dup, ALIAS.free].sort());
116
122
  const bossRow = (await asBoss.call("requirements_people", { network_id: netA, q: boss.username })).people[0];
@@ -19,7 +19,22 @@ export const PEOPLE_MAX_LIMIT = 200;
19
19
 
20
20
  /** `department_id`: Agent 的**显式**部门归属(可空)。空 = 仍按「负责人在哪个部门」推断,
21
21
  * 客户端据此把 Agent 放对格子 —— 2026-10-19 前没有这一列,328 个 Agent 只能全算「未归属」。 */
22
- export type DirectoryAgent = { node_id: string; alias: string; department_id?: string | null };
22
+ export type DirectoryAgent = {
23
+ node_id: string;
24
+ alias: string;
25
+ department_id?: string | null;
26
+ /**
27
+ * #2596 —— 这一行“最后一次被 Hub 看到”的时间(`nodes.updated_at`),以及生命周期状态。
28
+ *
29
+ * 为什么加:组织架构页把 368 个 agent 全塞进「未归属」,其中 125 个是 5–8 月的历史残留
30
+ * (如 `海报设计师1–5号` 注册于 2026-05-17、早已 offline)。客户端要能“默认只显示近 7 天活跃”
31
+ * 与“同一 alias 只保留最新的那条”,就必须拿到时间轴——否则只能靠 node_id 猜。
32
+ *
33
+ * 只增不改:老客户端忽略未知键,形状断言已在同批测试里放宽。
34
+ */
35
+ updated_at?: string | null;
36
+ lifecycle_state?: string | null;
37
+ };
23
38
  export type DirectoryPerson = {
24
39
  user_id: string;
25
40
  username: string;
@@ -64,8 +79,8 @@ export function peopleDirectory(networkId: string, hidden: HiddenNode, query: {
64
79
  ORDER BY u.username, u.user_id`,
65
80
  networkId,
66
81
  );
67
- const nodes = db.all<{ node_id: string; alias: string | null; node_name: string; owner_user_id: string | null; department_id: string | null }>(
68
- "SELECT node_id, alias, node_name, owner_user_id, department_id FROM nodes WHERE network_id = ?1 ORDER BY COALESCE(alias, node_name), node_id",
82
+ const nodes = db.all<{ node_id: string; alias: string | null; node_name: string; owner_user_id: string | null; department_id: string | null; updated_at: string | null; lifecycle_state: string | null }>(
83
+ "SELECT node_id, alias, node_name, owner_user_id, department_id, updated_at, lifecycle_state FROM nodes WHERE network_id = ?1 ORDER BY COALESCE(alias, node_name), node_id",
69
84
  networkId,
70
85
  ).filter(n => !hidden?.(n.node_id));
71
86
  const memberIds = new Set(members.map(m => m.user_id));
@@ -75,7 +90,14 @@ export function peopleDirectory(networkId: string, hidden: HiddenNode, query: {
75
90
  // 只在**显式**归属时带这一格:没设 = 没有显式归属(顶层 agent_department_supported 告诉
76
91
  // 客户端这一点)。行形状保持不变 —— 旧客户端拿到的行还是老样子(多一个键会让严格要求
77
92
  // 形状的测试/客户端意外),只是新客户端不再把「负责人在哪个部门」当成 Agent 的部门。
78
- const agent = { node_id: n.node_id, alias: n.alias || n.node_name, ...(n.department_id ? { department_id: n.department_id } : {}) };
93
+ const agent = {
94
+ node_id: n.node_id,
95
+ alias: n.alias || n.node_name,
96
+ ...(n.department_id ? { department_id: n.department_id } : {}),
97
+ // #2596 —— 客户端据此做「按 alias 去重(保留最新)」与「默认只看近 7 天」。
98
+ updated_at: n.updated_at ?? null,
99
+ lifecycle_state: n.lifecycle_state ?? null,
100
+ };
79
101
  if (n.owner_user_id && memberIds.has(n.owner_user_id)) {
80
102
  const list = byOwner.get(n.owner_user_id) ?? [];
81
103
  list.push(agent);
package/src/server.ts CHANGED
@@ -29,6 +29,7 @@ import { deriveTeamRulesView } from "./team-rules-status.js";
29
29
  import { recentNodeIdentityConflicts } from "./node-identity-conflict.js";
30
30
  import { createSSEStream, createNetworkObserverStream, createUserEventStream, pushEvent, pushNetworkObserverEvent, getSSEStats, PRINTABLE_OBSERVER_KEY_PREFIX, closeUserStreamsInNetwork, getUserPresence, onUserPresenceChange, pushUserEventToNetwork } from "./push.js";
31
31
  import { enterMcpInflight, leaveMcpInflight, getMcpInflightStats } from "./mcp-inflight.js";
32
+ import { eventLoopStats, startEventLoopLagMonitor } from "./event-loop-lag.js";
32
33
  import { assertNodeActive } from "./lifecycle-guard.js";
33
34
  import { pendingInboxCount } from "./inbox-count.js";
34
35
  import { notifyExpiredTasks, type ExpiredTaskRow } from "./task-expiry-notice.js";
@@ -2788,6 +2789,11 @@ return Bun.serve({
2788
2789
  // 而 `sse_connections` 只反映长连接,看不到“同秒打进来的一批请求”。
2789
2790
  mcp_inflight: getMcpInflightStats().inflight,
2790
2791
  mcp_inflight_peak: getMcpInflightStats().peak,
2792
+ // #2614 —— 事件循环停顿。`mcp_inflight` 只能说明“有多少请求在排队”,
2793
+ // 说明不了“服务端为什么卡”。这里给出停顿本身:max_gap_ms 是进程内
2794
+ // 单次 tick 的最大间隔(正常 ≈250ms),recent_stalls 保留最近 20 次
2795
+ // 超过阈值的停顿(含当时 RSS),用于和内存曲线对齐。
2796
+ event_loop: eventLoopStats(),
2791
2797
  auth: DEV_OPEN ? "dev-open" : "user-token",
2792
2798
  security: DEV_OPEN ? "dev-open" : "secured",
2793
2799
  tmux: TMUX_ENABLED ? "enabled" : "disabled",
@@ -5631,6 +5637,12 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5631
5637
  : 60 * 60 * 1000;
5632
5638
  const retentionSweeperTimer = startRetentionSweeper(sweepIntervalMs);
5633
5639
 
5640
+ // #2614 —— 事件循环停顿监测。放这里是因为 startHub() 才是“真的在跑一个 Hub”
5641
+ // 的入口(测试用 bootServer() 不会背这个 timer)。停顿日志与
5642
+ // `[commhub retention]` / `[commhub slow-sql]` 共用前缀族,便于一次 grep 出
5643
+ // “同一秒里 retention 跑了 10 秒 + 有两百个请求在排队”。
5644
+ const eventLoopLagTimer = startEventLoopLagMonitor();
5645
+
5634
5646
  // Round-2/4 review ③ — stale session sweeper (coexists with the
5635
5647
  // retention sweeper above; different concerns + different cadences).
5636
5648
  // Replaces the per-request UPDATE in GET /api/status (+ /api/servers,
@@ -5664,6 +5676,7 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5664
5676
  // jobs must not — otherwise a test (or a future caller) that stops the
5665
5677
  // server would still hang on live intervals it cannot reach.
5666
5678
  (retentionSweeperTimer as any)?.unref?.();
5679
+ (eventLoopLagTimer as any)?.unref?.();
5667
5680
  (staleSweeperTimer as any)?.unref?.();
5668
5681
  (rateLimitSweepTimer as any)?.unref?.();
5669
5682
  (taskPatrolTimer as any)?.unref?.();
@@ -5675,6 +5688,7 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5675
5688
  function shutdown() {
5676
5689
  console.log("[commhub] shutting down...");
5677
5690
  clearInterval(retentionSweeperTimer);
5691
+ clearInterval(eventLoopLagTimer);
5678
5692
  clearInterval(staleSweeperTimer);
5679
5693
  clearInterval(rateLimitSweepTimer);
5680
5694
  clearInterval(taskPatrolTimer);
@@ -1,4 +1,4 @@
1
- // #2571 —— delivered_stale 的**分流**:目标在跑别的任务时不再报“可能离线”。
1
+ // #2571 —— delivered_stale 的**分流**(阈值 10m/30m,见 task-lifecycle-watcher.ts 顶部说明):目标在跑别的任务时不再报“可能离线”。
2
2
  //
3
3
  // 生产数据(24h):出现过 delivered_stale 的任务里 83% 最终成功,等待 p50 2.9 分钟、
4
4
  // p90 27 分钟 ⇒ 事件事实成立,但把“目标正忙”写成“可能离线”会制造告警疲劳。
@@ -11,7 +11,7 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test";
11
11
  import { db } from "./db.js";
12
12
  import { recordDeliveredStaleEvents } from "./task-lifecycle-watcher.js";
13
13
 
14
- const NOW = new Date("2026-08-10T00:01:01.000Z");
14
+ const NOW = new Date("2026-08-10T01:00:01.000Z");
15
15
  let networkId = "";
16
16
 
17
17
  function sqliteUtcTimestamp(date: Date): string {
@@ -48,15 +48,15 @@ beforeAll(async () => {
48
48
 
49
49
  // 目标在跑别的任务
50
50
  insertSession("target-busy", "working");
51
- insertTask("busy-60", "target-busy", "2026-08-10 00:00:01");
51
+ insertTask("busy-600", "target-busy", "2026-08-10 00:50:01");
52
52
  // 目标空闲却没开始(真异常)
53
53
  insertSession("target-idle", "idle");
54
- insertTask("idle-60", "target-idle", "2026-08-10 00:00:01");
54
+ insertTask("idle-600", "target-idle", "2026-08-10 00:50:01");
55
55
  // 目标掉线
56
56
  insertSession("target-offline", "offline");
57
- insertTask("offline-60", "target-offline", "2026-08-10 00:00:01");
57
+ insertTask("offline-600", "target-offline", "2026-08-10 00:50:01");
58
58
  // 完全没有 session 行(未知)
59
- insertTask("unknown-60", "target-without-session", "2026-08-10 00:00:01");
59
+ insertTask("unknown-600", "target-without-session", "2026-08-10 00:50:01");
60
60
  }, 30_000);
61
61
 
62
62
  afterAll(() => {
@@ -70,17 +70,17 @@ afterAll(() => {
70
70
  describe("#2571 stale vs queued-behind-busy", () => {
71
71
  test("目标 working ⇒ 只写 queued_behind_busy_*,不写 delivered_stale_*", () => {
72
72
  recordDeliveredStaleEvents(NOW);
73
- const events = eventsFor("busy-60");
74
- expect(events).toContain("task.warning.queued_behind_busy_60s");
75
- expect(events).not.toContain("task.warning.delivered_stale_60s");
73
+ const events = eventsFor("busy-600");
74
+ expect(events).toContain("task.warning.queued_behind_busy_600s");
75
+ expect(events).not.toContain("task.warning.delivered_stale_600s");
76
76
  });
77
77
 
78
78
  test("目标 idle / offline / 无 session ⇒ 只写 delivered_stale_*", () => {
79
79
  recordDeliveredStaleEvents(NOW);
80
- for (const taskId of ["idle-60", "offline-60", "unknown-60"]) {
80
+ for (const taskId of ["idle-600", "offline-600", "unknown-600"]) {
81
81
  const events = eventsFor(taskId);
82
- expect(events).toContain("task.warning.delivered_stale_60s");
83
- expect(events).not.toContain("task.warning.queued_behind_busy_60s");
82
+ expect(events).toContain("task.warning.delivered_stale_600s");
83
+ expect(events).not.toContain("task.warning.queued_behind_busy_600s");
84
84
  }
85
85
  });
86
86
 
@@ -100,10 +100,10 @@ describe("#2571 stale vs queued-behind-busy", () => {
100
100
 
101
101
  test("detail 文案能区分“排队”和“没在跑”", () => {
102
102
  const busy = db.get<{ detail: string }>(
103
- `SELECT detail FROM task_events WHERE task_id='busy-60' AND event_type LIKE 'task.warning.queued_behind_busy_%'`,
103
+ `SELECT detail FROM task_events WHERE task_id='busy-600' AND event_type LIKE 'task.warning.queued_behind_busy_%'`,
104
104
  );
105
105
  const stale = db.get<{ detail: string }>(
106
- `SELECT detail FROM task_events WHERE task_id='idle-60' AND event_type LIKE 'task.warning.delivered_stale_%'`,
106
+ `SELECT detail FROM task_events WHERE task_id='idle-600' AND event_type LIKE 'task.warning.delivered_stale_%'`,
107
107
  );
108
108
  expect(busy!.detail).toContain("queued after");
109
109
  expect(stale!.detail).toContain("still delivered");
@@ -4,7 +4,7 @@ import { db, logTaskEvent } from "./db.js";
4
4
  import { TASK_EVENT_REST_COLUMNS } from "./rest-projections.js";
5
5
  import { recordDeliveredStaleEvents } from "./task-lifecycle-watcher.js";
6
6
 
7
- const NOW = new Date("2026-08-10T00:01:01.000Z");
7
+ const NOW = new Date("2026-08-10T01:00:01.000Z");
8
8
  let server: ReturnType<typeof Bun.serve>;
9
9
  let base = "";
10
10
  let token = "";
@@ -34,9 +34,9 @@ beforeAll(async () => {
34
34
  token = auth.token!;
35
35
  networkId = auth.network_id!;
36
36
 
37
- insertTask("stale-29", "delivered", "2026-08-10 00:00:32");
38
- insertTask("stale-30", "delivered", "2026-08-10 00:00:31");
39
- insertTask("stale-60", "delivered", "2026-08-10 00:00:01");
37
+ insertTask("stale-599", "delivered", "2026-08-10 00:50:02");
38
+ insertTask("stale-600", "delivered", "2026-08-10 00:50:01");
39
+ insertTask("stale-1800", "delivered", "2026-08-10 00:30:01");
40
40
  insertTask("stale-acked", "acked", "2026-08-10 00:00:00");
41
41
  insertTask("stale-no-delivery", "delivered", null);
42
42
 
@@ -70,21 +70,21 @@ async function waitUntil(ready: () => boolean, timeoutMs: number, what: string |
70
70
  }
71
71
 
72
72
  describe("#167 Hub delivered-stale lifecycle watcher", () => {
73
- test("30s/60s thresholds are exact and non-delivered tasks stay silent", () => {
73
+ test("10m/30m thresholds are exact and non-delivered tasks stay silent", () => {
74
74
  const first = recordDeliveredStaleEvents(NOW);
75
75
  expect(first.inserted).toBe(3);
76
- expect(first.by_threshold_seconds).toEqual({ 30: 2, 60: 1 });
76
+ expect(first.by_threshold_seconds).toEqual({ 600: 2, 1800: 1 });
77
77
 
78
- expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-29'")!.count).toBe(0);
79
- expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-30'")!.count).toBe(1);
80
- expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-60'")!.count).toBe(2);
78
+ expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-599'")!.count).toBe(0);
79
+ expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-600'")!.count).toBe(1);
80
+ expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-1800'")!.count).toBe(2);
81
81
  expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-acked'")!.count).toBe(0);
82
82
  expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-no-delivery'")!.count).toBe(0);
83
83
 
84
- const sixty = db.all<any>("SELECT * FROM task_events WHERE task_id = 'stale-60' ORDER BY event_type");
84
+ const sixty = db.all<any>("SELECT * FROM task_events WHERE task_id = 'stale-1800' ORDER BY event_type");
85
85
  expect(sixty.map((row) => row.event_type)).toEqual([
86
- "task.warning.delivered_stale_30s",
87
- "task.warning.delivered_stale_60s",
86
+ "task.warning.delivered_stale_1800s",
87
+ "task.warning.delivered_stale_600s",
88
88
  ]);
89
89
  expect(sixty.map((row) => row.event_key)).toEqual(sixty.map((row) => row.event_type));
90
90
  expect(sixty.every((row) => row.from_status === "delivered" && row.to_status === "delivered")).toBe(true);
@@ -95,21 +95,21 @@ describe("#167 Hub delivered-stale lifecycle watcher", () => {
95
95
  test("a second patrol is write-once for each task and threshold", () => {
96
96
  const second = recordDeliveredStaleEvents(NOW);
97
97
  expect(second.inserted).toBe(0);
98
- expect(second.by_threshold_seconds).toEqual({ 30: 0, 60: 0 });
98
+ expect(second.by_threshold_seconds).toEqual({ 600: 0, 1800: 0 });
99
99
  expect(db.get<{ count: number }>(
100
100
  "SELECT COUNT(*) AS count FROM task_events WHERE event_type LIKE 'task.warning.delivered_stale_%'",
101
101
  )!.count).toBe(3);
102
102
  });
103
103
 
104
104
  test("ordinary lifecycle events get standard event names without a dedupe key", () => {
105
- logTaskEvent("stale-30", null, "delivered", "target");
106
- logTaskEvent("stale-30", "delivered", "acked", "target");
107
- logTaskEvent("stale-30", "acked", "running", "target");
108
- logTaskEvent("stale-30", "running", "replied", "target");
109
- logTaskEvent("stale-30", "running", "failed", "target");
110
- logTaskEvent("stale-30", "delivered", "expired", "target");
105
+ logTaskEvent("stale-600", null, "delivered", "target");
106
+ logTaskEvent("stale-600", "delivered", "acked", "target");
107
+ logTaskEvent("stale-600", "acked", "running", "target");
108
+ logTaskEvent("stale-600", "running", "replied", "target");
109
+ logTaskEvent("stale-600", "running", "failed", "target");
110
+ logTaskEvent("stale-600", "delivered", "expired", "target");
111
111
  const rows = db.all<any>(
112
- "SELECT event_type, event_key FROM task_events WHERE task_id = 'stale-30' AND actor = 'target' ORDER BY id",
112
+ "SELECT event_type, event_key FROM task_events WHERE task_id = 'stale-600' AND actor = 'target' ORDER BY id",
113
113
  );
114
114
  expect(rows).toEqual([
115
115
  { event_type: "task.send.delivered", event_key: null },
@@ -125,7 +125,7 @@ describe("#167 Hub delivered-stale lifecycle watcher", () => {
125
125
  expect(TASK_EVENT_REST_COLUMNS).toContain("event_type");
126
126
  expect(TASK_EVENT_REST_COLUMNS).not.toContain("event_key" as any);
127
127
  const response = await fetch(
128
- `${base}/api/task_events?network_id=${encodeURIComponent(networkId)}&task_id=stale-60`,
128
+ `${base}/api/task_events?network_id=${encodeURIComponent(networkId)}&task_id=stale-1800`,
129
129
  { headers: { Authorization: `Bearer ${token}` } },
130
130
  );
131
131
  expect(response.status).toBe(200);
@@ -177,7 +177,7 @@ describe("#167 Hub delivered-stale lifecycle watcher", () => {
177
177
  `INSERT INTO tasks
178
178
  (task_id, from_name, to_name, status, content, delivered_at, network_id)
179
179
  VALUES (?1, 'sender', 'target', 'delivered', 'live watcher fixture', ?2, ?3)`,
180
- [taskId, sqliteUtcTimestamp(new Date(Date.now() - 28_000)), networkId],
180
+ [taskId, sqliteUtcTimestamp(new Date(Date.now() - 601_000)), networkId],
181
181
  );
182
182
  expect(childDb.query<{ count: number }, [string]>(
183
183
  "SELECT COUNT(*) AS count FROM task_events WHERE task_id = ?1",
@@ -188,10 +188,10 @@ describe("#167 Hub delivered-stale lifecycle watcher", () => {
188
188
  // 实测在 CI 上被这一条打红过(#798 让这个文件第一次进 CI 才暴露)。
189
189
  // 改成「轮询到目标状态,或到期报错」:常态下快 ~60 倍,慢的时候等得起。
190
190
  const countStale = () => childDb.query<{ count: number }, [string]>(
191
- "SELECT COUNT(*) AS count FROM task_events WHERE task_id = ?1 AND event_type = 'task.warning.delivered_stale_30s'",
191
+ "SELECT COUNT(*) AS count FROM task_events WHERE task_id = ?1 AND event_type = 'task.warning.delivered_stale_600s'",
192
192
  ).get(taskId)!.count;
193
193
  await waitUntil(() => countStale() === 1, 20_000,
194
- "watcher did not write the delivered_stale_30s event");
194
+ "watcher did not write the delivered_stale_600s event");
195
195
  expect(countStale()).toBe(1);
196
196
  } finally {
197
197
  childDb.close();
@@ -1,6 +1,6 @@
1
1
  import { db } from "./db.js";
2
2
 
3
- export const DELIVERED_STALE_THRESHOLDS = [30, 60] as const;
3
+ export const DELIVERED_STALE_THRESHOLDS = [600, 1800] as const;
4
4
 
5
5
  export type DeliveredStalePatrolResult = {
6
6
  inserted: number;
@@ -11,6 +11,19 @@ function sqliteUtcTimestamp(date: Date): string {
11
11
  return date.toISOString().slice(0, 19).replace("T", " ");
12
12
  }
13
13
 
14
+ // #2571(第二轮)—— 阈值从 30s/60s 提到 10min/30min。
15
+ //
16
+ // 第一轮只做了“按目标忙/闲分流”,上线后(19:45–22:30)实测**告警量反而翻倍**
17
+ // (33/h → 73/h,33/h → 104/h),因为同一个任务会先在忙时写 busy、目标转闲后
18
+ // 再写一次 stale(46 个任务两类都出现)。而被标 stale 的任务里仍有 73.5%
19
+ // 最终成功。真正的病因不是“忙 vs 闲”,是**阈值本身离真实分布太远**:
20
+ // 7 天 11877 个成功任务,delivered → 首次活动 p50 = 1 秒、p90 = 6 秒、
21
+ // p95 = 57 秒、p99 = 12 分钟。
22
+ // ⇒ 30 秒的判据会命中 6.4% 的**正常**任务,60 秒命中 4.9%,等于把 p95–p99
23
+ // 的长尾全报警。改成 10 分钟(p99 之下)才有意义:超过它的比例 1.2%,
24
+ // 30 分钟是 0.2%,只留真异常。
25
+ //
26
+ // event_key 里带秒数 ⇒ 改阈值会产生新的 key,历史事件不受影响、可并存对比。
14
27
  /**
15
28
  * Persist Hub-authoritative warnings for tasks which are still delivered.
16
29
  *
@@ -27,13 +40,18 @@ function sqliteUtcTimestamp(date: Date): string {
27
40
  * 命中次数变成 0 或让 mutation 只命中一半 —— 那就把“可被变异测试覆盖”这条性质弄丢了。
28
41
  * 因此这里仍然只保留**一条** INSERT,用 CASE 选择事件名与文案。
29
42
  */
30
- function recordStale(thresholdSeconds: 30 | 60, cutoff: string): number {
43
+ function thresholdLabel(thresholdSeconds: 600 | 1800): string {
44
+ return thresholdSeconds === 600 ? "10m" : "30m";
45
+ }
46
+
47
+ function recordStale(thresholdSeconds: 600 | 1800, cutoff: string): number {
31
48
  const busyType = `task.warning.queued_behind_busy_${thresholdSeconds}s`;
32
49
  const staleType = `task.warning.delivered_stale_${thresholdSeconds}s`;
33
- const busyDetail = `queued after ${thresholdSeconds}s; target is working on another task`;
34
- const staleDetail = thresholdSeconds === 30
35
- ? "still delivered after 30s; target is not working on another task and has not started"
36
- : "still delivered after 60s; target is idle or offline and not consuming";
50
+ const label = thresholdLabel(thresholdSeconds);
51
+ const busyDetail = `queued after ${label}; target is working on another task`;
52
+ const staleDetail = thresholdSeconds === 600
53
+ ? `still delivered after ${label}; target is not working on another task and has not started`
54
+ : `still delivered after ${label}; target is idle or offline and not consuming`;
37
55
  const result = db.run(
38
56
  `INSERT INTO task_events
39
57
  (task_id, from_status, to_status, event_type, event_key, actor, detail, network_id)
@@ -62,7 +80,7 @@ function recordStale(thresholdSeconds: 30 | 60, cutoff: string): number {
62
80
  export function recordDeliveredStaleEvents(now = new Date()): DeliveredStalePatrolResult {
63
81
  if (!Number.isFinite(now.getTime())) throw new Error("invalid lifecycle patrol time");
64
82
 
65
- const byThreshold = { 30: 0, 60: 0 };
83
+ const byThreshold = { 600: 0, 1800: 0 };
66
84
  let inserted = 0;
67
85
 
68
86
  for (const thresholdSeconds of DELIVERED_STALE_THRESHOLDS) {