@sleep2agi/commhub-server 0.9.0-preview.161 → 0.9.0-preview.163
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/backfill-node-ids-2664.test.ts +128 -0
- package/src/db-adapter.ts +54 -4
- package/src/db.ts +108 -100
- package/src/event-loop-lag.test.ts +139 -0
- package/src/event-loop-lag.ts +150 -0
- package/src/requirements-people-mcp-http.test.ts +9 -3
- package/src/requirements-people.ts +26 -4
- package/src/route-timing.ts +4 -0
- package/src/scheduled-failure-backoff-2622.test.ts +145 -0
- package/src/scheduled-failures.ts +65 -0
- package/src/server.ts +14 -0
- package/src/task-lifecycle-watcher-busy.test.ts +14 -14
- package/src/task-lifecycle-watcher.test.ts +24 -24
- package/src/task-lifecycle-watcher.ts +25 -7
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sleep2agi/commhub-server",
|
|
3
|
-
"version": "0.9.0-preview.
|
|
3
|
+
"version": "0.9.0-preview.163",
|
|
4
4
|
"description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
// #2664 —— `backfillMessageNodeIds()` 从「逐行相关子查询」改成「物化的别名→node_id 表 + UPDATE … FROM」。
|
|
2
|
+
//
|
|
3
|
+
// 这次改写**只为了快**(生产副本实测单条 6.56s → 0.29s;boot 期六条约 10s → 约 0.75s),
|
|
4
|
+
// 而“快”最容易顺手改坏的是**选哪一行 / 谁不该被改**:
|
|
5
|
+
// · 人的用户名不能被映射成节点(`NOT EXISTS users.username`);
|
|
6
|
+
// · 改过名的节点走 rename_txn(只认 `committed`);
|
|
7
|
+
// · 已经有 node_id 的行**不能被覆盖**;
|
|
8
|
+
// · 重复调用幂等。
|
|
9
|
+
//
|
|
10
|
+
// 注:`sessions` 上有 `UNIQUE (network_id, alias)` 且 `network_id NOT NULL`,
|
|
11
|
+
// 所以“同一 alias 多条 session 时取 updated_at 最新”在当前 schema 下**不可构造**;
|
|
12
|
+
// 窗口函数只是为了在 PG / 老库形状下保持与旧 `ORDER BY … LIMIT 1` 等价,不是新语义。
|
|
13
|
+
|
|
14
|
+
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
15
|
+
import { db, backfillMessageNodeIds } from "./db.js";
|
|
16
|
+
|
|
17
|
+
let networkId = "";
|
|
18
|
+
const stamp = `${Date.now()}`;
|
|
19
|
+
|
|
20
|
+
function session(alias: string, nodeId: string | null, updatedAt: string, network = networkId) {
|
|
21
|
+
db.run(
|
|
22
|
+
`INSERT INTO sessions (resume_id, alias, node_id, status, network_id, registered_at, updated_at)
|
|
23
|
+
VALUES (?1, ?2, ?3, 'idle', ?4, ?5, ?5)`,
|
|
24
|
+
[`res_${alias}_${stamp}`, alias, nodeId, network, updatedAt],
|
|
25
|
+
);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function task(id: string, fromName: string, toName: string, network = networkId) {
|
|
29
|
+
db.run(
|
|
30
|
+
`INSERT INTO tasks (task_id, from_name, to_name, content, status, network_id)
|
|
31
|
+
VALUES (?1, ?2, ?3, 'fixture', 'delivered', ?4)`,
|
|
32
|
+
[id, fromName, toName, network],
|
|
33
|
+
);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function nodeIds(id: string) {
|
|
37
|
+
return db.get<{ from_node_id: string | null; to_node_id: string | null }>(
|
|
38
|
+
"SELECT from_node_id, to_node_id FROM tasks WHERE task_id = ?1",
|
|
39
|
+
[`${id}-${stamp}`],
|
|
40
|
+
)!;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
beforeAll(async () => {
|
|
44
|
+
const { register } = await import("./auth.js");
|
|
45
|
+
const auth = register(`bf_owner_${stamp}`, "Bf-Owner-Strong-1!", undefined, "tester");
|
|
46
|
+
expect(auth.ok).toBe(true);
|
|
47
|
+
networkId = auth.network_id!;
|
|
48
|
+
|
|
49
|
+
session(`bf-node-${stamp}`, `node-${stamp}`, "2026-10-20 00:00:00");
|
|
50
|
+
// 人类用户:即便 sessions 里出现过同名行,也不能把任务归到那个 node
|
|
51
|
+
session(`bf-human-${stamp}`, `node-should-not-win-${stamp}`, "2026-10-20 00:00:00");
|
|
52
|
+
db.run(
|
|
53
|
+
`INSERT INTO users (user_id, username, display_name, role, password_hash, created_at)
|
|
54
|
+
VALUES (?1, ?2, '人类', 'member', 'x', datetime('now'))`,
|
|
55
|
+
[`u_bf_${stamp}`, `bf-human-${stamp}`],
|
|
56
|
+
);
|
|
57
|
+
|
|
58
|
+
task(`bf-fresh-${stamp}`, `bf-node-${stamp}`, `bf-node-${stamp}`);
|
|
59
|
+
task(`bf-human-task-${stamp}`, `bf-human-${stamp}`, `bf-node-${stamp}`);
|
|
60
|
+
task(`bf-preset-${stamp}`, `bf-node-${stamp}`, `bf-node-${stamp}`);
|
|
61
|
+
db.run(`UPDATE tasks SET from_node_id = 'preset-node', to_node_id = 'preset-node' WHERE task_id = ?1`, [
|
|
62
|
+
`bf-preset-${stamp}`,
|
|
63
|
+
]);
|
|
64
|
+
|
|
65
|
+
// rename_txn:旧别名 → 新别名(committed 生效;pending 不生效)
|
|
66
|
+
session(`bf-new-alias-${stamp}`, `node-renamed-${stamp}`, "2026-10-20 00:00:00");
|
|
67
|
+
db.run(
|
|
68
|
+
`INSERT INTO rename_txn (txn_id, old_alias, new_alias, status, network_id, committed_at)
|
|
69
|
+
VALUES (?1, ?2, ?3, 'committed', ?4, '2026-10-21 00:00:00')`,
|
|
70
|
+
[`txn_bf_${stamp}`, `bf-old-alias-${stamp}`, `bf-new-alias-${stamp}`, networkId],
|
|
71
|
+
);
|
|
72
|
+
task(`bf-renamed-${stamp}`, `bf-old-alias-${stamp}`, `bf-old-alias-${stamp}`);
|
|
73
|
+
|
|
74
|
+
db.run(
|
|
75
|
+
`INSERT INTO rename_txn (txn_id, old_alias, new_alias, status, network_id, committed_at)
|
|
76
|
+
VALUES (?1, ?2, ?3, 'pending', ?4, NULL)`,
|
|
77
|
+
[`txn_bf_pending_${stamp}`, `bf-pending-alias-${stamp}`, `bf-new-alias-${stamp}`, networkId],
|
|
78
|
+
);
|
|
79
|
+
task(`bf-pending-${stamp}`, `bf-pending-alias-${stamp}`, `bf-pending-alias-${stamp}`);
|
|
80
|
+
|
|
81
|
+
backfillMessageNodeIds();
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
afterAll(() => {
|
|
85
|
+
try {
|
|
86
|
+
db.run(`DELETE FROM tasks WHERE task_id LIKE ?1`, [`bf-%-${stamp}`]);
|
|
87
|
+
db.run(`DELETE FROM sessions WHERE alias LIKE ?1`, [`bf-%-${stamp}`]);
|
|
88
|
+
db.run(`DELETE FROM rename_txn WHERE txn_id LIKE ?1`, [`txn_bf%${stamp}`]);
|
|
89
|
+
db.run(`DELETE FROM users WHERE username LIKE ?1`, [`bf-human-${stamp}`]);
|
|
90
|
+
} catch {}
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
describe("#2664 backfill semantics unchanged by the join rewrite", () => {
|
|
94
|
+
test("普通别名能补上 from/to 两侧的 node_id", () => {
|
|
95
|
+
expect(nodeIds("bf-fresh").from_node_id).toBe(`node-${stamp}`);
|
|
96
|
+
expect(nodeIds("bf-fresh").to_node_id).toBe(`node-${stamp}`);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test("人的用户名不会被映射成节点", () => {
|
|
100
|
+
expect(nodeIds("bf-human-task").from_node_id).toBeNull();
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test("已经有 node_id 的行不被覆盖", () => {
|
|
104
|
+
expect(nodeIds("bf-preset").from_node_id).toBe("preset-node");
|
|
105
|
+
expect(nodeIds("bf-preset").to_node_id).toBe("preset-node");
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
test("改过名的节点按 rename_txn 的旧别名补上", () => {
|
|
109
|
+
expect(nodeIds("bf-renamed").from_node_id).toBe(`node-renamed-${stamp}`);
|
|
110
|
+
expect(nodeIds("bf-renamed").to_node_id).toBe(`node-renamed-${stamp}`);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
test("未提交(pending)的改名不生效", () => {
|
|
114
|
+
expect(nodeIds("bf-pending").from_node_id).toBeNull();
|
|
115
|
+
expect(nodeIds("bf-pending").to_node_id).toBeNull();
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
test("第二次调用是幂等的", () => {
|
|
119
|
+
const count = () =>
|
|
120
|
+
db.get<{ n: number }>(
|
|
121
|
+
`SELECT COUNT(*) AS n FROM tasks WHERE network_id = ?1 AND (from_node_id IS NOT NULL OR to_node_id IS NOT NULL)`,
|
|
122
|
+
[networkId],
|
|
123
|
+
)!.n;
|
|
124
|
+
const before = count();
|
|
125
|
+
backfillMessageNodeIds();
|
|
126
|
+
expect(count()).toBe(before);
|
|
127
|
+
});
|
|
128
|
+
});
|
package/src/db-adapter.ts
CHANGED
|
@@ -61,6 +61,36 @@ export const PG_EXPERIMENTAL_ENV = "COMMHUB_PG_EXPERIMENTAL";
|
|
|
61
61
|
// SQLite Adapter (bun:sqlite, sync)
|
|
62
62
|
// ════════════════════════════════════════════
|
|
63
63
|
|
|
64
|
+
// #2614 —— 慢 SQL 观测。SQLite 是**同步**的:一条慢查询会直接卡住整个事件循环
|
|
65
|
+
// (生产实测 2026-10-26 每小时一次的 8–11 秒停顿就是 retention 的全表扫描,
|
|
66
|
+
// 症状是日志静默 + 在飞数暴涨 + RSS 冲到 4.5 GB)。之前没有任何一条日志能指出
|
|
67
|
+
// “是这条 SQL 卡了 11 秒”,只能靠事后比对日志静默的时间窗手工推断。
|
|
68
|
+
//
|
|
69
|
+
// 这里只在**超过阈值**时打一行日志,不含参数值(避免把令牌/正文写进日志)。
|
|
70
|
+
// `COMMHUB_SLOW_SQL_MS` 控制阈值(默认 1000 ms,设 0 关闭)。
|
|
71
|
+
const DEFAULT_SLOW_SQL_MS = 1000;
|
|
72
|
+
|
|
73
|
+
export function slowSqlThresholdMs(): number {
|
|
74
|
+
const raw = Number(process.env.COMMHUB_SLOW_SQL_MS);
|
|
75
|
+
return Number.isFinite(raw) && raw >= 0 ? raw : DEFAULT_SLOW_SQL_MS;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** 截断 + 压平 SQL,只保留足够定位的头部(不包含绑定参数)。 */
|
|
79
|
+
export function formatSlowSql(ms: number, sql: string): string {
|
|
80
|
+
const oneLine = sql.replace(/\s+/g, " ").trim().slice(0, 160);
|
|
81
|
+
return `[commhub slow-sql] ${ms.toFixed(0)}ms ${oneLine}`;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
let sqlTimingDisabled: boolean | null = null;
|
|
85
|
+
|
|
86
|
+
function logIfSlow(started: number, sql: string): void {
|
|
87
|
+
if (sqlTimingDisabled === null) sqlTimingDisabled = slowSqlThresholdMs() <= 0;
|
|
88
|
+
if (sqlTimingDisabled) return;
|
|
89
|
+
const ms = performance.now() - started;
|
|
90
|
+
if (ms < slowSqlThresholdMs()) return;
|
|
91
|
+
console.log(formatSlowSql(ms, sql));
|
|
92
|
+
}
|
|
93
|
+
|
|
64
94
|
export class SQLiteAdapter implements DbAdapter {
|
|
65
95
|
readonly dialect = "sqlite" as const;
|
|
66
96
|
readonly atomicTransactions = true;
|
|
@@ -70,19 +100,39 @@ export class SQLiteAdapter implements DbAdapter {
|
|
|
70
100
|
constructor(private readonly rawDb: Database) {}
|
|
71
101
|
|
|
72
102
|
run(sql: string, params?: any[]): QueryResult {
|
|
73
|
-
|
|
103
|
+
const t0 = performance.now();
|
|
104
|
+
try {
|
|
105
|
+
return params ? this.rawDb.run(sql, params as any) : this.rawDb.run(sql);
|
|
106
|
+
} finally {
|
|
107
|
+
logIfSlow(t0, sql);
|
|
108
|
+
}
|
|
74
109
|
}
|
|
75
110
|
|
|
76
111
|
get<T = any>(sql: string, ...params: any[]): T | null {
|
|
77
|
-
|
|
112
|
+
const t0 = performance.now();
|
|
113
|
+
try {
|
|
114
|
+
return this.rawDb.query<T, any[]>(sql).get(...params) ?? null;
|
|
115
|
+
} finally {
|
|
116
|
+
logIfSlow(t0, sql);
|
|
117
|
+
}
|
|
78
118
|
}
|
|
79
119
|
|
|
80
120
|
all<T = any>(sql: string, ...params: any[]): T[] {
|
|
81
|
-
|
|
121
|
+
const t0 = performance.now();
|
|
122
|
+
try {
|
|
123
|
+
return this.rawDb.query<T, any[]>(sql).all(...params);
|
|
124
|
+
} finally {
|
|
125
|
+
logIfSlow(t0, sql);
|
|
126
|
+
}
|
|
82
127
|
}
|
|
83
128
|
|
|
84
129
|
exec(sql: string): void {
|
|
85
|
-
|
|
130
|
+
const t0 = performance.now();
|
|
131
|
+
try {
|
|
132
|
+
this.rawDb.exec(sql);
|
|
133
|
+
} finally {
|
|
134
|
+
logIfSlow(t0, sql);
|
|
135
|
+
}
|
|
86
136
|
}
|
|
87
137
|
|
|
88
138
|
transaction<T>(fn: () => T): T {
|
package/src/db.ts
CHANGED
|
@@ -174,6 +174,13 @@ db.exec(`
|
|
|
174
174
|
|
|
175
175
|
CREATE INDEX IF NOT EXISTS idx_agent_telemetry_alias_time
|
|
176
176
|
ON agent_telemetry(network_id, alias, created_at);
|
|
177
|
+
|
|
178
|
+
-- #2614 —— retention 每小时跑一次 DELETE FROM agent_telemetry WHERE created_at < ?。
|
|
179
|
+
-- 上面三个复合索引都以 network_id 打头,**这个查询用不上**(没有 network_id 等值条件),
|
|
180
|
+
-- 于是每次扫全表 62 万行 → 生产实测每小时出现 **8–11 秒的事件循环阻塞**
|
|
181
|
+
-- ([commhub retention] swept in 8331ms / 9863ms / 10588ms,同时日志静默 11 秒)。
|
|
182
|
+
-- 单独给 created_at 建索引,让这个扫描变成范围 seek。
|
|
183
|
+
CREATE INDEX IF NOT EXISTS idx_agent_telemetry_created ON agent_telemetry(created_at);
|
|
177
184
|
`);
|
|
178
185
|
|
|
179
186
|
// Round-2/4 review ② — server-health index split.
|
|
@@ -1296,6 +1303,18 @@ try { db.exec("CREATE INDEX IF NOT EXISTS idx_tasks_to_node ON tasks(to_node_id)
|
|
|
1296
1303
|
try { db.exec("CREATE INDEX IF NOT EXISTS idx_inbox_node ON inbox(node_id)"); } catch {}
|
|
1297
1304
|
|
|
1298
1305
|
export function backfillMessageNodeIds() {
|
|
1306
|
+
// #2664 —— 旧写法是 6 条 `UPDATE … SET x = (SELECT … WHERE alias = t.name …)`:
|
|
1307
|
+
// 每条都对 **84k 行 tasks.from_node_id IS NULL / 36k 行 inbox.node_id IS NULL** 逐行
|
|
1308
|
+
// 跑一次相关子查询,而子查询里的 `COALESCE(network_id,'default')` 让
|
|
1309
|
+
// idx_sessions_network_alias_unique 用不上 ⇒ 每行扫一遍 369 行 sessions。
|
|
1310
|
+
// 生产实测(`.162` 慢 SQL 日志,2026-10-27 00:21 重启):5398ms / 2451ms / 1270ms,
|
|
1311
|
+
// 而且当次命中 0 行 —— **结果是 0,代价不是 0**。
|
|
1312
|
+
//
|
|
1313
|
+
// 改成 `UPDATE … FROM (物化后的别名→node_id 表)`:sessions/rename_txn 只扫一遍,
|
|
1314
|
+
// 每行只做一次自动索引探测。同一份生产库副本上 6.56s → 0.29s。
|
|
1315
|
+
// 语义不变:同一 (alias, network) 取 `updated_at` / `committed_at` 最新的那一行;
|
|
1316
|
+
// 仍然是「先按 session 别名直连,再补 rename_txn 的旧别名」;对 tasks 仍排除人是发送方的情况。
|
|
1317
|
+
const t0 = performance.now();
|
|
1299
1318
|
let inboxDirect = 0;
|
|
1300
1319
|
let inboxRenamed = 0;
|
|
1301
1320
|
let tasksToDirect = 0;
|
|
@@ -1306,157 +1325,146 @@ export function backfillMessageNodeIds() {
|
|
|
1306
1325
|
try {
|
|
1307
1326
|
inboxDirect = db.run(`
|
|
1308
1327
|
UPDATE inbox
|
|
1309
|
-
SET node_id =
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
)
|
|
1328
|
+
SET node_id = pick.node_id
|
|
1329
|
+
FROM (
|
|
1330
|
+
SELECT alias, COALESCE(network_id, 'default') AS net, node_id,
|
|
1331
|
+
ROW_NUMBER() OVER (
|
|
1332
|
+
PARTITION BY alias, COALESCE(network_id, 'default')
|
|
1333
|
+
ORDER BY updated_at DESC
|
|
1334
|
+
) AS rn
|
|
1335
|
+
FROM sessions
|
|
1336
|
+
WHERE node_id IS NOT NULL
|
|
1337
|
+
) pick
|
|
1338
|
+
WHERE pick.rn = 1
|
|
1339
|
+
AND pick.alias = inbox.session_name
|
|
1340
|
+
AND pick.net = COALESCE(inbox.network_id, 'default')
|
|
1341
|
+
AND inbox.node_id IS NULL
|
|
1324
1342
|
`).changes;
|
|
1325
1343
|
} catch {}
|
|
1326
1344
|
|
|
1327
1345
|
try {
|
|
1328
1346
|
inboxRenamed = db.run(`
|
|
1329
1347
|
UPDATE inbox
|
|
1330
|
-
SET node_id =
|
|
1331
|
-
|
|
1348
|
+
SET node_id = pick.node_id
|
|
1349
|
+
FROM (
|
|
1350
|
+
SELECT r.old_alias AS old_alias,
|
|
1351
|
+
COALESCE(r.network_id, 'default') AS net,
|
|
1352
|
+
s.node_id AS node_id,
|
|
1353
|
+
ROW_NUMBER() OVER (
|
|
1354
|
+
PARTITION BY r.old_alias, COALESCE(r.network_id, 'default')
|
|
1355
|
+
ORDER BY r.committed_at DESC
|
|
1356
|
+
) AS rn
|
|
1357
|
+
FROM rename_txn r
|
|
1332
1358
|
JOIN sessions s
|
|
1333
1359
|
ON s.alias = r.new_alias
|
|
1334
1360
|
AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
|
|
1335
1361
|
WHERE s.node_id IS NOT NULL
|
|
1336
1362
|
AND r.status = 'committed'
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
WHERE node_id IS NULL
|
|
1343
|
-
AND EXISTS (
|
|
1344
|
-
SELECT 1 FROM rename_txn r
|
|
1345
|
-
JOIN sessions s
|
|
1346
|
-
ON s.alias = r.new_alias
|
|
1347
|
-
AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
|
|
1348
|
-
WHERE s.node_id IS NOT NULL
|
|
1349
|
-
AND r.status = 'committed'
|
|
1350
|
-
AND r.old_alias = inbox.session_name
|
|
1351
|
-
AND COALESCE(r.network_id, 'default') = COALESCE(inbox.network_id, 'default')
|
|
1352
|
-
)
|
|
1363
|
+
) pick
|
|
1364
|
+
WHERE pick.rn = 1
|
|
1365
|
+
AND pick.old_alias = inbox.session_name
|
|
1366
|
+
AND pick.net = COALESCE(inbox.network_id, 'default')
|
|
1367
|
+
AND inbox.node_id IS NULL
|
|
1353
1368
|
`).changes;
|
|
1354
1369
|
} catch {}
|
|
1355
1370
|
|
|
1356
1371
|
try {
|
|
1357
1372
|
tasksToDirect = db.run(`
|
|
1358
1373
|
UPDATE tasks
|
|
1359
|
-
SET to_node_id =
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
)
|
|
1374
|
+
SET to_node_id = pick.node_id
|
|
1375
|
+
FROM (
|
|
1376
|
+
SELECT alias, COALESCE(network_id, 'default') AS net, node_id,
|
|
1377
|
+
ROW_NUMBER() OVER (
|
|
1378
|
+
PARTITION BY alias, COALESCE(network_id, 'default')
|
|
1379
|
+
ORDER BY updated_at DESC
|
|
1380
|
+
) AS rn
|
|
1381
|
+
FROM sessions
|
|
1382
|
+
WHERE node_id IS NOT NULL
|
|
1383
|
+
) pick
|
|
1384
|
+
WHERE pick.rn = 1
|
|
1385
|
+
AND pick.alias = tasks.to_name
|
|
1386
|
+
AND pick.net = COALESCE(tasks.network_id, 'default')
|
|
1387
|
+
AND tasks.to_node_id IS NULL
|
|
1374
1388
|
`).changes;
|
|
1375
1389
|
} catch {}
|
|
1376
1390
|
|
|
1377
1391
|
try {
|
|
1378
1392
|
tasksToRenamed = db.run(`
|
|
1379
1393
|
UPDATE tasks
|
|
1380
|
-
SET to_node_id =
|
|
1381
|
-
|
|
1394
|
+
SET to_node_id = pick.node_id
|
|
1395
|
+
FROM (
|
|
1396
|
+
SELECT r.old_alias AS old_alias,
|
|
1397
|
+
COALESCE(r.network_id, 'default') AS net,
|
|
1398
|
+
s.node_id AS node_id,
|
|
1399
|
+
ROW_NUMBER() OVER (
|
|
1400
|
+
PARTITION BY r.old_alias, COALESCE(r.network_id, 'default')
|
|
1401
|
+
ORDER BY r.committed_at DESC
|
|
1402
|
+
) AS rn
|
|
1403
|
+
FROM rename_txn r
|
|
1382
1404
|
JOIN sessions s
|
|
1383
1405
|
ON s.alias = r.new_alias
|
|
1384
1406
|
AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
|
|
1385
1407
|
WHERE s.node_id IS NOT NULL
|
|
1386
1408
|
AND r.status = 'committed'
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
WHERE to_node_id IS NULL
|
|
1393
|
-
AND EXISTS (
|
|
1394
|
-
SELECT 1 FROM rename_txn r
|
|
1395
|
-
JOIN sessions s
|
|
1396
|
-
ON s.alias = r.new_alias
|
|
1397
|
-
AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
|
|
1398
|
-
WHERE s.node_id IS NOT NULL
|
|
1399
|
-
AND r.status = 'committed'
|
|
1400
|
-
AND r.old_alias = tasks.to_name
|
|
1401
|
-
AND COALESCE(r.network_id, 'default') = COALESCE(tasks.network_id, 'default')
|
|
1402
|
-
)
|
|
1409
|
+
) pick
|
|
1410
|
+
WHERE pick.rn = 1
|
|
1411
|
+
AND pick.old_alias = tasks.to_name
|
|
1412
|
+
AND pick.net = COALESCE(tasks.network_id, 'default')
|
|
1413
|
+
AND tasks.to_node_id IS NULL
|
|
1403
1414
|
`).changes;
|
|
1404
1415
|
} catch {}
|
|
1405
1416
|
|
|
1406
1417
|
try {
|
|
1407
1418
|
tasksFromDirect = db.run(`
|
|
1408
1419
|
UPDATE tasks
|
|
1409
|
-
SET from_node_id =
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1420
|
+
SET from_node_id = pick.node_id
|
|
1421
|
+
FROM (
|
|
1422
|
+
SELECT alias, COALESCE(network_id, 'default') AS net, node_id,
|
|
1423
|
+
ROW_NUMBER() OVER (
|
|
1424
|
+
PARTITION BY alias, COALESCE(network_id, 'default')
|
|
1425
|
+
ORDER BY updated_at DESC
|
|
1426
|
+
) AS rn
|
|
1427
|
+
FROM sessions
|
|
1428
|
+
WHERE node_id IS NOT NULL
|
|
1429
|
+
) pick
|
|
1430
|
+
WHERE pick.rn = 1
|
|
1431
|
+
AND pick.alias = tasks.from_name
|
|
1432
|
+
AND pick.net = COALESCE(tasks.network_id, 'default')
|
|
1433
|
+
AND tasks.from_node_id IS NULL
|
|
1418
1434
|
AND NOT EXISTS (SELECT 1 FROM users u WHERE u.username = tasks.from_name)
|
|
1419
|
-
AND EXISTS (
|
|
1420
|
-
SELECT 1 FROM sessions s
|
|
1421
|
-
WHERE s.node_id IS NOT NULL
|
|
1422
|
-
AND s.alias = tasks.from_name
|
|
1423
|
-
AND COALESCE(s.network_id, 'default') = COALESCE(tasks.network_id, 'default')
|
|
1424
|
-
)
|
|
1425
1435
|
`).changes;
|
|
1426
1436
|
} catch {}
|
|
1427
1437
|
|
|
1428
1438
|
try {
|
|
1429
1439
|
tasksFromRenamed = db.run(`
|
|
1430
1440
|
UPDATE tasks
|
|
1431
|
-
SET from_node_id =
|
|
1432
|
-
|
|
1441
|
+
SET from_node_id = pick.node_id
|
|
1442
|
+
FROM (
|
|
1443
|
+
SELECT r.old_alias AS old_alias,
|
|
1444
|
+
COALESCE(r.network_id, 'default') AS net,
|
|
1445
|
+
s.node_id AS node_id,
|
|
1446
|
+
ROW_NUMBER() OVER (
|
|
1447
|
+
PARTITION BY r.old_alias, COALESCE(r.network_id, 'default')
|
|
1448
|
+
ORDER BY r.committed_at DESC
|
|
1449
|
+
) AS rn
|
|
1450
|
+
FROM rename_txn r
|
|
1433
1451
|
JOIN sessions s
|
|
1434
1452
|
ON s.alias = r.new_alias
|
|
1435
1453
|
AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
|
|
1436
1454
|
WHERE s.node_id IS NOT NULL
|
|
1437
1455
|
AND r.status = 'committed'
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
WHERE from_node_id IS NULL
|
|
1456
|
+
) pick
|
|
1457
|
+
WHERE pick.rn = 1
|
|
1458
|
+
AND pick.old_alias = tasks.from_name
|
|
1459
|
+
AND pick.net = COALESCE(tasks.network_id, 'default')
|
|
1460
|
+
AND tasks.from_node_id IS NULL
|
|
1444
1461
|
AND NOT EXISTS (SELECT 1 FROM users u WHERE u.username = tasks.from_name)
|
|
1445
|
-
AND EXISTS (
|
|
1446
|
-
SELECT 1 FROM rename_txn r
|
|
1447
|
-
JOIN sessions s
|
|
1448
|
-
ON s.alias = r.new_alias
|
|
1449
|
-
AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
|
|
1450
|
-
WHERE s.node_id IS NOT NULL
|
|
1451
|
-
AND r.status = 'committed'
|
|
1452
|
-
AND r.old_alias = tasks.from_name
|
|
1453
|
-
AND COALESCE(r.network_id, 'default') = COALESCE(tasks.network_id, 'default')
|
|
1454
|
-
)
|
|
1455
1462
|
`).changes;
|
|
1456
1463
|
} catch {}
|
|
1457
1464
|
|
|
1458
1465
|
const total = inboxDirect + inboxRenamed + tasksToDirect + tasksToRenamed + tasksFromDirect + tasksFromRenamed;
|
|
1459
|
-
|
|
1466
|
+
// #2664 —— 旧日志只有条数,于是「total=0 却花了 10 秒」在生产里完全看不见。
|
|
1467
|
+
console.log(`[commhub] node_id backfill: inbox=${inboxDirect + inboxRenamed}, tasks.to=${tasksToDirect + tasksToRenamed}, tasks.from=${tasksFromDirect + tasksFromRenamed}, total=${total}, took=${Math.round(performance.now() - t0)}ms`);
|
|
1460
1468
|
}
|
|
1461
1469
|
|
|
1462
1470
|
backfillMessageNodeIds();
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
// #2614 —— 事件循环停顿监测 + 慢 SQL 日志的单元测试。
|
|
2
|
+
//
|
|
3
|
+
// 这两个东西的价值不在“能测出停顿”,而在**停顿发生时留下可对齐的证据**:
|
|
4
|
+
// 2026-10-26 生产两次 RSS 3.6/4.5 GB 的尖峰,事后只能靠“日志静默几十秒”
|
|
5
|
+
// 手工推断,`/api/stats/routes` 只给出受害者请求(含排队时间)。
|
|
6
|
+
// 所以测试要钉住的是语义:怎么算一次停顿、留什么档、日志阈值怎么生效、关闭后还计数。
|
|
7
|
+
|
|
8
|
+
import { describe, expect, test } from "bun:test";
|
|
9
|
+
import {
|
|
10
|
+
eventLoopStats,
|
|
11
|
+
noteRequestServed,
|
|
12
|
+
recordGap,
|
|
13
|
+
resetEventLoopLagForTests,
|
|
14
|
+
startEventLoopLagMonitor,
|
|
15
|
+
} from "./event-loop-lag.js";
|
|
16
|
+
import { formatSlowSql, slowSqlThresholdMs } from "./db-adapter.js";
|
|
17
|
+
|
|
18
|
+
describe("event-loop-lag: recordGap", () => {
|
|
19
|
+
test("正常 tick(≈interval)不算停顿", () => {
|
|
20
|
+
resetEventLoopLagForTests();
|
|
21
|
+
expect(recordGap(260, { intervalMs: 250, thresholdMs: 2000 })).toBe(10);
|
|
22
|
+
const s = eventLoopStats();
|
|
23
|
+
expect(s.stall_count).toBe(0);
|
|
24
|
+
expect(s.max_gap_ms).toBe(260);
|
|
25
|
+
expect(s.recent_stalls).toEqual([]);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test("超过阈值的 tick 记一条停顿,带 stall_ms / rss_mb / at", () => {
|
|
29
|
+
resetEventLoopLagForTests();
|
|
30
|
+
const now = Date.parse("2026-10-26T22:10:41Z");
|
|
31
|
+
const stall = recordGap(28_000, { intervalMs: 250, thresholdMs: 2000, rssMb: 4502, now });
|
|
32
|
+
expect(stall).toBe(27_750);
|
|
33
|
+
const s = eventLoopStats();
|
|
34
|
+
expect(s.stall_count).toBe(1);
|
|
35
|
+
expect(s.last_stall_ms).toBe(27_750);
|
|
36
|
+
expect(s.last_stall_at).toBe("2026-10-26T22:10:41.000Z");
|
|
37
|
+
expect(s.max_gap_ms).toBe(28_000);
|
|
38
|
+
expect(s.recent_stalls[0]).toMatchObject({ gap_ms: 28_000, stall_ms: 27_750, rss_mb: 4502 });
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("recent_stalls 最新在前,且只保留 20 条", () => {
|
|
42
|
+
resetEventLoopLagForTests();
|
|
43
|
+
for (let i = 0; i < 25; i++) {
|
|
44
|
+
recordGap(5_000 + i, { intervalMs: 250, thresholdMs: 2_000, rssMb: 300, now: 1_700_000_000_000 + i * 1_000 });
|
|
45
|
+
}
|
|
46
|
+
const s = eventLoopStats();
|
|
47
|
+
expect(s.stall_count).toBe(25);
|
|
48
|
+
expect(s.recent_stalls.length).toBe(20);
|
|
49
|
+
// 最新一条是 i=24
|
|
50
|
+
expect(s.recent_stalls[0].gap_ms).toBe(5_024);
|
|
51
|
+
expect(s.recent_stalls[19].gap_ms).toBe(5_005);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test("记录这段间隔里服务了多少请求(区分“阻塞”与“timer 饥饿”)", () => {
|
|
55
|
+
resetEventLoopLagForTests();
|
|
56
|
+
recordGap(85_000, { intervalMs: 250, thresholdMs: 2_000, rssMb: 2076, served: 137 });
|
|
57
|
+
const s = eventLoopStats();
|
|
58
|
+
expect(s.recent_stalls[0].requests_since_last_tick).toBe(137);
|
|
59
|
+
// 0 服务数 = 这段时间真的什么都没推进
|
|
60
|
+
recordGap(30_000, { intervalMs: 250, thresholdMs: 2_000, rssMb: 500, served: 0 });
|
|
61
|
+
expect(eventLoopStats().recent_stalls[0].requests_since_last_tick).toBe(0);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("max_gap_ms 保留历史最大值(回落后不被覆盖)", () => {
|
|
65
|
+
resetEventLoopLagForTests();
|
|
66
|
+
recordGap(47_000, { intervalMs: 250, thresholdMs: 2_000 });
|
|
67
|
+
recordGap(260, { intervalMs: 250, thresholdMs: 2_000 });
|
|
68
|
+
const s = eventLoopStats();
|
|
69
|
+
expect(s.max_gap_ms).toBe(47_000);
|
|
70
|
+
expect(s.last_gap_ms).toBe(260);
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
describe("event-loop-lag: startEventLoopLagMonitor", () => {
|
|
75
|
+
test("阈值 0 时不打日志,但仍然计数", async () => {
|
|
76
|
+
resetEventLoopLagForTests();
|
|
77
|
+
const logs: string[] = [];
|
|
78
|
+
const timer = startEventLoopLagMonitor({
|
|
79
|
+
intervalMs: 20,
|
|
80
|
+
thresholdMs: 0,
|
|
81
|
+
log: (m) => logs.push(m),
|
|
82
|
+
rssMb: () => 123,
|
|
83
|
+
});
|
|
84
|
+
// 同步阻塞 80ms,让下一次 tick 一定超阈值
|
|
85
|
+
const until = performance.now() + 80;
|
|
86
|
+
while (performance.now() < until) { /* spin */ }
|
|
87
|
+
await Bun.sleep(60);
|
|
88
|
+
clearInterval(timer);
|
|
89
|
+
expect(logs).toEqual([]);
|
|
90
|
+
expect(eventLoopStats().max_gap_ms).toBeGreaterThan(0);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test("真实停顿会打一行 [commhub stall] 且带 rss", async () => {
|
|
94
|
+
resetEventLoopLagForTests();
|
|
95
|
+
const logs: string[] = [];
|
|
96
|
+
const timer = startEventLoopLagMonitor({
|
|
97
|
+
intervalMs: 20,
|
|
98
|
+
thresholdMs: 40,
|
|
99
|
+
log: (m) => logs.push(m),
|
|
100
|
+
rssMb: () => 4502,
|
|
101
|
+
});
|
|
102
|
+
const until = performance.now() + 120;
|
|
103
|
+
while (performance.now() < until) { /* spin 阻塞事件循环 */ }
|
|
104
|
+
await Bun.sleep(60);
|
|
105
|
+
clearInterval(timer);
|
|
106
|
+
expect(logs.length).toBeGreaterThan(0);
|
|
107
|
+
expect(logs[0]).toStartWith("[commhub stall] loop idle ~");
|
|
108
|
+
expect(logs[0]).toContain("rss 4502 MB");
|
|
109
|
+
expect(logs[0]).toContain("served 0 req");
|
|
110
|
+
});
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
describe("db-adapter: 慢 SQL 日志", () => {
|
|
114
|
+
test("默认阈值 1000ms;环境变量可调,0 表示关闭", () => {
|
|
115
|
+
const prev = process.env.COMMHUB_SLOW_SQL_MS;
|
|
116
|
+
try {
|
|
117
|
+
delete process.env.COMMHUB_SLOW_SQL_MS;
|
|
118
|
+
expect(slowSqlThresholdMs()).toBe(1000);
|
|
119
|
+
process.env.COMMHUB_SLOW_SQL_MS = "250";
|
|
120
|
+
expect(slowSqlThresholdMs()).toBe(250);
|
|
121
|
+
process.env.COMMHUB_SLOW_SQL_MS = "0";
|
|
122
|
+
expect(slowSqlThresholdMs()).toBe(0);
|
|
123
|
+
process.env.COMMHUB_SLOW_SQL_MS = "not-a-number";
|
|
124
|
+
expect(slowSqlThresholdMs()).toBe(1000);
|
|
125
|
+
} finally {
|
|
126
|
+
if (prev === undefined) delete process.env.COMMHUB_SLOW_SQL_MS;
|
|
127
|
+
else process.env.COMMHUB_SLOW_SQL_MS = prev;
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
test("日志压平换行并截断,且不含绑定参数", () => {
|
|
132
|
+
const sql = "DELETE FROM agent_telemetry\n WHERE created_at < datetime('now', ?1)";
|
|
133
|
+
const line = formatSlowSql(10_588.4, sql);
|
|
134
|
+
expect(line).toStartWith("[commhub slow-sql] 10588ms DELETE FROM agent_telemetry WHERE created_at < datetime('now', ?1)");
|
|
135
|
+
expect(line).not.toContain("\n");
|
|
136
|
+
const long = formatSlowSql(1_200, "SELECT " + "x".repeat(500));
|
|
137
|
+
expect(long.length).toBeLessThanOrEqual("[commhub slow-sql] 1200ms ".length + 160);
|
|
138
|
+
});
|
|
139
|
+
});
|