@sleep2agi/commhub-server 0.9.0-preview.162 → 0.9.0-preview.163

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sleep2agi/commhub-server",
3
- "version": "0.9.0-preview.162",
3
+ "version": "0.9.0-preview.163",
4
4
  "description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
5
5
  "type": "module",
6
6
  "main": "src/index.ts",
@@ -0,0 +1,128 @@
1
+ // #2664 —— `backfillMessageNodeIds()` 从「逐行相关子查询」改成「物化的别名→node_id 表 + UPDATE … FROM」。
2
+ //
3
+ // 这次改写**只为了快**(生产副本实测单条 6.56s → 0.29s;boot 期六条约 10s → 约 0.75s),
4
+ // 而“快”最容易顺手改坏的是**选哪一行 / 谁不该被改**:
5
+ // · 人的用户名不能被映射成节点(`NOT EXISTS users.username`);
6
+ // · 改过名的节点走 rename_txn(只认 `committed`);
7
+ // · 已经有 node_id 的行**不能被覆盖**;
8
+ // · 重复调用幂等。
9
+ //
10
+ // 注:`sessions` 上有 `UNIQUE (network_id, alias)` 且 `network_id NOT NULL`,
11
+ // 所以“同一 alias 多条 session 时取 updated_at 最新”在当前 schema 下**不可构造**;
12
+ // 窗口函数只是为了在 PG / 老库形状下保持与旧 `ORDER BY … LIMIT 1` 等价,不是新语义。
13
+
14
+ import { afterAll, beforeAll, describe, expect, test } from "bun:test";
15
+ import { db, backfillMessageNodeIds } from "./db.js";
16
+
17
+ let networkId = "";
18
+ const stamp = `${Date.now()}`;
19
+
20
+ function session(alias: string, nodeId: string | null, updatedAt: string, network = networkId) {
21
+ db.run(
22
+ `INSERT INTO sessions (resume_id, alias, node_id, status, network_id, registered_at, updated_at)
23
+ VALUES (?1, ?2, ?3, 'idle', ?4, ?5, ?5)`,
24
+ [`res_${alias}_${stamp}`, alias, nodeId, network, updatedAt],
25
+ );
26
+ }
27
+
28
+ function task(id: string, fromName: string, toName: string, network = networkId) {
29
+ db.run(
30
+ `INSERT INTO tasks (task_id, from_name, to_name, content, status, network_id)
31
+ VALUES (?1, ?2, ?3, 'fixture', 'delivered', ?4)`,
32
+ [id, fromName, toName, network],
33
+ );
34
+ }
35
+
36
+ function nodeIds(id: string) {
37
+ return db.get<{ from_node_id: string | null; to_node_id: string | null }>(
38
+ "SELECT from_node_id, to_node_id FROM tasks WHERE task_id = ?1",
39
+ [`${id}-${stamp}`],
40
+ )!;
41
+ }
42
+
43
+ beforeAll(async () => {
44
+ const { register } = await import("./auth.js");
45
+ const auth = register(`bf_owner_${stamp}`, "Bf-Owner-Strong-1!", undefined, "tester");
46
+ expect(auth.ok).toBe(true);
47
+ networkId = auth.network_id!;
48
+
49
+ session(`bf-node-${stamp}`, `node-${stamp}`, "2026-10-20 00:00:00");
50
+ // 人类用户:即便 sessions 里出现过同名行,也不能把任务归到那个 node
51
+ session(`bf-human-${stamp}`, `node-should-not-win-${stamp}`, "2026-10-20 00:00:00");
52
+ db.run(
53
+ `INSERT INTO users (user_id, username, display_name, role, password_hash, created_at)
54
+ VALUES (?1, ?2, '人类', 'member', 'x', datetime('now'))`,
55
+ [`u_bf_${stamp}`, `bf-human-${stamp}`],
56
+ );
57
+
58
+ task(`bf-fresh-${stamp}`, `bf-node-${stamp}`, `bf-node-${stamp}`);
59
+ task(`bf-human-task-${stamp}`, `bf-human-${stamp}`, `bf-node-${stamp}`);
60
+ task(`bf-preset-${stamp}`, `bf-node-${stamp}`, `bf-node-${stamp}`);
61
+ db.run(`UPDATE tasks SET from_node_id = 'preset-node', to_node_id = 'preset-node' WHERE task_id = ?1`, [
62
+ `bf-preset-${stamp}`,
63
+ ]);
64
+
65
+ // rename_txn:旧别名 → 新别名(committed 生效;pending 不生效)
66
+ session(`bf-new-alias-${stamp}`, `node-renamed-${stamp}`, "2026-10-20 00:00:00");
67
+ db.run(
68
+ `INSERT INTO rename_txn (txn_id, old_alias, new_alias, status, network_id, committed_at)
69
+ VALUES (?1, ?2, ?3, 'committed', ?4, '2026-10-21 00:00:00')`,
70
+ [`txn_bf_${stamp}`, `bf-old-alias-${stamp}`, `bf-new-alias-${stamp}`, networkId],
71
+ );
72
+ task(`bf-renamed-${stamp}`, `bf-old-alias-${stamp}`, `bf-old-alias-${stamp}`);
73
+
74
+ db.run(
75
+ `INSERT INTO rename_txn (txn_id, old_alias, new_alias, status, network_id, committed_at)
76
+ VALUES (?1, ?2, ?3, 'pending', ?4, NULL)`,
77
+ [`txn_bf_pending_${stamp}`, `bf-pending-alias-${stamp}`, `bf-new-alias-${stamp}`, networkId],
78
+ );
79
+ task(`bf-pending-${stamp}`, `bf-pending-alias-${stamp}`, `bf-pending-alias-${stamp}`);
80
+
81
+ backfillMessageNodeIds();
82
+ });
83
+
84
+ afterAll(() => {
85
+ try {
86
+ db.run(`DELETE FROM tasks WHERE task_id LIKE ?1`, [`bf-%-${stamp}`]);
87
+ db.run(`DELETE FROM sessions WHERE alias LIKE ?1`, [`bf-%-${stamp}`]);
88
+ db.run(`DELETE FROM rename_txn WHERE txn_id LIKE ?1`, [`txn_bf%${stamp}`]);
89
+ db.run(`DELETE FROM users WHERE username LIKE ?1`, [`bf-human-${stamp}`]);
90
+ } catch {}
91
+ });
92
+
93
+ describe("#2664 backfill semantics unchanged by the join rewrite", () => {
94
+ test("普通别名能补上 from/to 两侧的 node_id", () => {
95
+ expect(nodeIds("bf-fresh").from_node_id).toBe(`node-${stamp}`);
96
+ expect(nodeIds("bf-fresh").to_node_id).toBe(`node-${stamp}`);
97
+ });
98
+
99
+ test("人的用户名不会被映射成节点", () => {
100
+ expect(nodeIds("bf-human-task").from_node_id).toBeNull();
101
+ });
102
+
103
+ test("已经有 node_id 的行不被覆盖", () => {
104
+ expect(nodeIds("bf-preset").from_node_id).toBe("preset-node");
105
+ expect(nodeIds("bf-preset").to_node_id).toBe("preset-node");
106
+ });
107
+
108
+ test("改过名的节点按 rename_txn 的旧别名补上", () => {
109
+ expect(nodeIds("bf-renamed").from_node_id).toBe(`node-renamed-${stamp}`);
110
+ expect(nodeIds("bf-renamed").to_node_id).toBe(`node-renamed-${stamp}`);
111
+ });
112
+
113
+ test("未提交(pending)的改名不生效", () => {
114
+ expect(nodeIds("bf-pending").from_node_id).toBeNull();
115
+ expect(nodeIds("bf-pending").to_node_id).toBeNull();
116
+ });
117
+
118
+ test("第二次调用是幂等的", () => {
119
+ const count = () =>
120
+ db.get<{ n: number }>(
121
+ `SELECT COUNT(*) AS n FROM tasks WHERE network_id = ?1 AND (from_node_id IS NOT NULL OR to_node_id IS NOT NULL)`,
122
+ [networkId],
123
+ )!.n;
124
+ const before = count();
125
+ backfillMessageNodeIds();
126
+ expect(count()).toBe(before);
127
+ });
128
+ });
package/src/db.ts CHANGED
@@ -1303,6 +1303,18 @@ try { db.exec("CREATE INDEX IF NOT EXISTS idx_tasks_to_node ON tasks(to_node_id)
1303
1303
  try { db.exec("CREATE INDEX IF NOT EXISTS idx_inbox_node ON inbox(node_id)"); } catch {}
1304
1304
 
1305
1305
  export function backfillMessageNodeIds() {
1306
+ // #2664 —— 旧写法是 6 条 `UPDATE … SET x = (SELECT … WHERE alias = t.name …)`:
1307
+ // 每条都对 **84k 行 tasks.from_node_id IS NULL / 36k 行 inbox.node_id IS NULL** 逐行
1308
+ // 跑一次相关子查询,而子查询里的 `COALESCE(network_id,'default')` 让
1309
+ // idx_sessions_network_alias_unique 用不上 ⇒ 每行扫一遍 369 行 sessions。
1310
+ // 生产实测(`.162` 慢 SQL 日志,2026-10-27 00:21 重启):5398ms / 2451ms / 1270ms,
1311
+ // 而且当次命中 0 行 —— **结果是 0,代价不是 0**。
1312
+ //
1313
+ // 改成 `UPDATE … FROM (物化后的别名→node_id 表)`:sessions/rename_txn 只扫一遍,
1314
+ // 每行只做一次自动索引探测。同一份生产库副本上 6.56s → 0.29s。
1315
+ // 语义不变:同一 (alias, network) 取 `updated_at` / `committed_at` 最新的那一行;
1316
+ // 仍然是「先按 session 别名直连,再补 rename_txn 的旧别名」;对 tasks 仍排除人是发送方的情况。
1317
+ const t0 = performance.now();
1306
1318
  let inboxDirect = 0;
1307
1319
  let inboxRenamed = 0;
1308
1320
  let tasksToDirect = 0;
@@ -1313,157 +1325,146 @@ export function backfillMessageNodeIds() {
1313
1325
  try {
1314
1326
  inboxDirect = db.run(`
1315
1327
  UPDATE inbox
1316
- SET node_id = (
1317
- SELECT s.node_id FROM sessions s
1318
- WHERE s.node_id IS NOT NULL
1319
- AND s.alias = inbox.session_name
1320
- AND COALESCE(s.network_id, 'default') = COALESCE(inbox.network_id, 'default')
1321
- ORDER BY s.updated_at DESC
1322
- LIMIT 1
1323
- )
1324
- WHERE node_id IS NULL
1325
- AND EXISTS (
1326
- SELECT 1 FROM sessions s
1327
- WHERE s.node_id IS NOT NULL
1328
- AND s.alias = inbox.session_name
1329
- AND COALESCE(s.network_id, 'default') = COALESCE(inbox.network_id, 'default')
1330
- )
1328
+ SET node_id = pick.node_id
1329
+ FROM (
1330
+ SELECT alias, COALESCE(network_id, 'default') AS net, node_id,
1331
+ ROW_NUMBER() OVER (
1332
+ PARTITION BY alias, COALESCE(network_id, 'default')
1333
+ ORDER BY updated_at DESC
1334
+ ) AS rn
1335
+ FROM sessions
1336
+ WHERE node_id IS NOT NULL
1337
+ ) pick
1338
+ WHERE pick.rn = 1
1339
+ AND pick.alias = inbox.session_name
1340
+ AND pick.net = COALESCE(inbox.network_id, 'default')
1341
+ AND inbox.node_id IS NULL
1331
1342
  `).changes;
1332
1343
  } catch {}
1333
1344
 
1334
1345
  try {
1335
1346
  inboxRenamed = db.run(`
1336
1347
  UPDATE inbox
1337
- SET node_id = (
1338
- SELECT s.node_id FROM rename_txn r
1348
+ SET node_id = pick.node_id
1349
+ FROM (
1350
+ SELECT r.old_alias AS old_alias,
1351
+ COALESCE(r.network_id, 'default') AS net,
1352
+ s.node_id AS node_id,
1353
+ ROW_NUMBER() OVER (
1354
+ PARTITION BY r.old_alias, COALESCE(r.network_id, 'default')
1355
+ ORDER BY r.committed_at DESC
1356
+ ) AS rn
1357
+ FROM rename_txn r
1339
1358
  JOIN sessions s
1340
1359
  ON s.alias = r.new_alias
1341
1360
  AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
1342
1361
  WHERE s.node_id IS NOT NULL
1343
1362
  AND r.status = 'committed'
1344
- AND r.old_alias = inbox.session_name
1345
- AND COALESCE(r.network_id, 'default') = COALESCE(inbox.network_id, 'default')
1346
- ORDER BY r.committed_at DESC
1347
- LIMIT 1
1348
- )
1349
- WHERE node_id IS NULL
1350
- AND EXISTS (
1351
- SELECT 1 FROM rename_txn r
1352
- JOIN sessions s
1353
- ON s.alias = r.new_alias
1354
- AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
1355
- WHERE s.node_id IS NOT NULL
1356
- AND r.status = 'committed'
1357
- AND r.old_alias = inbox.session_name
1358
- AND COALESCE(r.network_id, 'default') = COALESCE(inbox.network_id, 'default')
1359
- )
1363
+ ) pick
1364
+ WHERE pick.rn = 1
1365
+ AND pick.old_alias = inbox.session_name
1366
+ AND pick.net = COALESCE(inbox.network_id, 'default')
1367
+ AND inbox.node_id IS NULL
1360
1368
  `).changes;
1361
1369
  } catch {}
1362
1370
 
1363
1371
  try {
1364
1372
  tasksToDirect = db.run(`
1365
1373
  UPDATE tasks
1366
- SET to_node_id = (
1367
- SELECT s.node_id FROM sessions s
1368
- WHERE s.node_id IS NOT NULL
1369
- AND s.alias = tasks.to_name
1370
- AND COALESCE(s.network_id, 'default') = COALESCE(tasks.network_id, 'default')
1371
- ORDER BY s.updated_at DESC
1372
- LIMIT 1
1373
- )
1374
- WHERE to_node_id IS NULL
1375
- AND EXISTS (
1376
- SELECT 1 FROM sessions s
1377
- WHERE s.node_id IS NOT NULL
1378
- AND s.alias = tasks.to_name
1379
- AND COALESCE(s.network_id, 'default') = COALESCE(tasks.network_id, 'default')
1380
- )
1374
+ SET to_node_id = pick.node_id
1375
+ FROM (
1376
+ SELECT alias, COALESCE(network_id, 'default') AS net, node_id,
1377
+ ROW_NUMBER() OVER (
1378
+ PARTITION BY alias, COALESCE(network_id, 'default')
1379
+ ORDER BY updated_at DESC
1380
+ ) AS rn
1381
+ FROM sessions
1382
+ WHERE node_id IS NOT NULL
1383
+ ) pick
1384
+ WHERE pick.rn = 1
1385
+ AND pick.alias = tasks.to_name
1386
+ AND pick.net = COALESCE(tasks.network_id, 'default')
1387
+ AND tasks.to_node_id IS NULL
1381
1388
  `).changes;
1382
1389
  } catch {}
1383
1390
 
1384
1391
  try {
1385
1392
  tasksToRenamed = db.run(`
1386
1393
  UPDATE tasks
1387
- SET to_node_id = (
1388
- SELECT s.node_id FROM rename_txn r
1394
+ SET to_node_id = pick.node_id
1395
+ FROM (
1396
+ SELECT r.old_alias AS old_alias,
1397
+ COALESCE(r.network_id, 'default') AS net,
1398
+ s.node_id AS node_id,
1399
+ ROW_NUMBER() OVER (
1400
+ PARTITION BY r.old_alias, COALESCE(r.network_id, 'default')
1401
+ ORDER BY r.committed_at DESC
1402
+ ) AS rn
1403
+ FROM rename_txn r
1389
1404
  JOIN sessions s
1390
1405
  ON s.alias = r.new_alias
1391
1406
  AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
1392
1407
  WHERE s.node_id IS NOT NULL
1393
1408
  AND r.status = 'committed'
1394
- AND r.old_alias = tasks.to_name
1395
- AND COALESCE(r.network_id, 'default') = COALESCE(tasks.network_id, 'default')
1396
- ORDER BY r.committed_at DESC
1397
- LIMIT 1
1398
- )
1399
- WHERE to_node_id IS NULL
1400
- AND EXISTS (
1401
- SELECT 1 FROM rename_txn r
1402
- JOIN sessions s
1403
- ON s.alias = r.new_alias
1404
- AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
1405
- WHERE s.node_id IS NOT NULL
1406
- AND r.status = 'committed'
1407
- AND r.old_alias = tasks.to_name
1408
- AND COALESCE(r.network_id, 'default') = COALESCE(tasks.network_id, 'default')
1409
- )
1409
+ ) pick
1410
+ WHERE pick.rn = 1
1411
+ AND pick.old_alias = tasks.to_name
1412
+ AND pick.net = COALESCE(tasks.network_id, 'default')
1413
+ AND tasks.to_node_id IS NULL
1410
1414
  `).changes;
1411
1415
  } catch {}
1412
1416
 
1413
1417
  try {
1414
1418
  tasksFromDirect = db.run(`
1415
1419
  UPDATE tasks
1416
- SET from_node_id = (
1417
- SELECT s.node_id FROM sessions s
1418
- WHERE s.node_id IS NOT NULL
1419
- AND s.alias = tasks.from_name
1420
- AND COALESCE(s.network_id, 'default') = COALESCE(tasks.network_id, 'default')
1421
- ORDER BY s.updated_at DESC
1422
- LIMIT 1
1423
- )
1424
- WHERE from_node_id IS NULL
1420
+ SET from_node_id = pick.node_id
1421
+ FROM (
1422
+ SELECT alias, COALESCE(network_id, 'default') AS net, node_id,
1423
+ ROW_NUMBER() OVER (
1424
+ PARTITION BY alias, COALESCE(network_id, 'default')
1425
+ ORDER BY updated_at DESC
1426
+ ) AS rn
1427
+ FROM sessions
1428
+ WHERE node_id IS NOT NULL
1429
+ ) pick
1430
+ WHERE pick.rn = 1
1431
+ AND pick.alias = tasks.from_name
1432
+ AND pick.net = COALESCE(tasks.network_id, 'default')
1433
+ AND tasks.from_node_id IS NULL
1425
1434
  AND NOT EXISTS (SELECT 1 FROM users u WHERE u.username = tasks.from_name)
1426
- AND EXISTS (
1427
- SELECT 1 FROM sessions s
1428
- WHERE s.node_id IS NOT NULL
1429
- AND s.alias = tasks.from_name
1430
- AND COALESCE(s.network_id, 'default') = COALESCE(tasks.network_id, 'default')
1431
- )
1432
1435
  `).changes;
1433
1436
  } catch {}
1434
1437
 
1435
1438
  try {
1436
1439
  tasksFromRenamed = db.run(`
1437
1440
  UPDATE tasks
1438
- SET from_node_id = (
1439
- SELECT s.node_id FROM rename_txn r
1441
+ SET from_node_id = pick.node_id
1442
+ FROM (
1443
+ SELECT r.old_alias AS old_alias,
1444
+ COALESCE(r.network_id, 'default') AS net,
1445
+ s.node_id AS node_id,
1446
+ ROW_NUMBER() OVER (
1447
+ PARTITION BY r.old_alias, COALESCE(r.network_id, 'default')
1448
+ ORDER BY r.committed_at DESC
1449
+ ) AS rn
1450
+ FROM rename_txn r
1440
1451
  JOIN sessions s
1441
1452
  ON s.alias = r.new_alias
1442
1453
  AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
1443
1454
  WHERE s.node_id IS NOT NULL
1444
1455
  AND r.status = 'committed'
1445
- AND r.old_alias = tasks.from_name
1446
- AND COALESCE(r.network_id, 'default') = COALESCE(tasks.network_id, 'default')
1447
- ORDER BY r.committed_at DESC
1448
- LIMIT 1
1449
- )
1450
- WHERE from_node_id IS NULL
1456
+ ) pick
1457
+ WHERE pick.rn = 1
1458
+ AND pick.old_alias = tasks.from_name
1459
+ AND pick.net = COALESCE(tasks.network_id, 'default')
1460
+ AND tasks.from_node_id IS NULL
1451
1461
  AND NOT EXISTS (SELECT 1 FROM users u WHERE u.username = tasks.from_name)
1452
- AND EXISTS (
1453
- SELECT 1 FROM rename_txn r
1454
- JOIN sessions s
1455
- ON s.alias = r.new_alias
1456
- AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
1457
- WHERE s.node_id IS NOT NULL
1458
- AND r.status = 'committed'
1459
- AND r.old_alias = tasks.from_name
1460
- AND COALESCE(r.network_id, 'default') = COALESCE(tasks.network_id, 'default')
1461
- )
1462
1462
  `).changes;
1463
1463
  } catch {}
1464
1464
 
1465
1465
  const total = inboxDirect + inboxRenamed + tasksToDirect + tasksToRenamed + tasksFromDirect + tasksFromRenamed;
1466
- console.log(`[commhub] node_id backfill: inbox=${inboxDirect + inboxRenamed}, tasks.to=${tasksToDirect + tasksToRenamed}, tasks.from=${tasksFromDirect + tasksFromRenamed}, total=${total}`);
1466
+ // #2664 —— 旧日志只有条数,于是「total=0 却花了 10 秒」在生产里完全看不见。
1467
+ console.log(`[commhub] node_id backfill: inbox=${inboxDirect + inboxRenamed}, tasks.to=${tasksToDirect + tasksToRenamed}, tasks.from=${tasksFromDirect + tasksFromRenamed}, total=${total}, took=${Math.round(performance.now() - t0)}ms`);
1467
1468
  }
1468
1469
 
1469
1470
  backfillMessageNodeIds();
@@ -8,6 +8,7 @@
8
8
  import { describe, expect, test } from "bun:test";
9
9
  import {
10
10
  eventLoopStats,
11
+ noteRequestServed,
11
12
  recordGap,
12
13
  resetEventLoopLagForTests,
13
14
  startEventLoopLagMonitor,
@@ -50,6 +51,16 @@ describe("event-loop-lag: recordGap", () => {
50
51
  expect(s.recent_stalls[19].gap_ms).toBe(5_005);
51
52
  });
52
53
 
54
+ test("记录这段间隔里服务了多少请求(区分“阻塞”与“timer 饥饿”)", () => {
55
+ resetEventLoopLagForTests();
56
+ recordGap(85_000, { intervalMs: 250, thresholdMs: 2_000, rssMb: 2076, served: 137 });
57
+ const s = eventLoopStats();
58
+ expect(s.recent_stalls[0].requests_since_last_tick).toBe(137);
59
+ // 0 服务数 = 这段时间真的什么都没推进
60
+ recordGap(30_000, { intervalMs: 250, thresholdMs: 2_000, rssMb: 500, served: 0 });
61
+ expect(eventLoopStats().recent_stalls[0].requests_since_last_tick).toBe(0);
62
+ });
63
+
53
64
  test("max_gap_ms 保留历史最大值(回落后不被覆盖)", () => {
54
65
  resetEventLoopLagForTests();
55
66
  recordGap(47_000, { intervalMs: 250, thresholdMs: 2_000 });
@@ -93,8 +104,9 @@ describe("event-loop-lag: startEventLoopLagMonitor", () => {
93
104
  await Bun.sleep(60);
94
105
  clearInterval(timer);
95
106
  expect(logs.length).toBeGreaterThan(0);
96
- expect(logs[0]).toStartWith("[commhub stall] loop blocked ~");
107
+ expect(logs[0]).toStartWith("[commhub stall] loop idle ~");
97
108
  expect(logs[0]).toContain("rss 4502 MB");
109
+ expect(logs[0]).toContain("served 0 req");
98
110
  });
99
111
  });
100
112
 
@@ -16,6 +16,7 @@
16
16
  // 停顿结束后第一次 tick 的 gap 就是停顿长度,不需要外部进程配合。
17
17
 
18
18
  export type EventLoopStall = {
19
+ /** 这个间隔里处理完的请求数:0 = 真的没推进;>0 = 循环在干活,只是 timer 被饿住。 */
19
20
  /** ISO 时间(停顿**结束**的时刻,即这次 tick 的时间)。 */
20
21
  at: string;
21
22
  /** 距上次 tick 的实际间隔(毫秒)。等于 interval + 停顿。 */
@@ -24,10 +25,28 @@ export type EventLoopStall = {
24
25
  stall_ms: number;
25
26
  /** 停顿结束时的进程 RSS(MB),用于和内存尖峰对齐。 */
26
27
  rss_mb: number;
28
+ requests_since_last_tick?: number;
27
29
  };
28
30
 
29
31
  const KEEP_STALLS = 20;
30
32
 
33
+ /**
34
+ * #2614(第二轮)—— 2026-10-27 00:39 线上出现 gap 84887ms,但**同一时段日志在正常写**
35
+ * (00:38:52 / 00:38:58 / 00:39:08 …),而且 `/api/stats/routes` 里最慢的请求只有 2.4s。
36
+ * 也就是说:事件循环**没有被阻塞 84 秒**,是 **timer 回调被 I/O 队列饿住**了。
37
+ * 只看 tick 间隔会把这两种情况混成一条“卡了 84 秒”的假结论。
38
+ *
39
+ * 区分办法:数一下这个间隔里**服务了多少请求**。
40
+ * requests_since_last_tick > 0 ⇒ 循环在干活 ⇒ 这是 timer 饥饿(高负载/大批回调)
41
+ * requests_since_last_tick = 0 ⇒ 真的没有推进任何请求 ⇒ 才可能是同步阻塞
42
+ */
43
+ let servedSinceTick = 0;
44
+
45
+ /** 请求处理完成时调用(route-timing 里)。只计数,不参与任何决策。 */
46
+ export function noteRequestServed(): void {
47
+ servedSinceTick++;
48
+ }
49
+
31
50
  let maxGapMs = 0;
32
51
  let lastGapMs = 0;
33
52
  let stallCount = 0;
@@ -38,7 +57,7 @@ const stalls: EventLoopStall[] = [];
38
57
  /** 记录一次 tick 间隔。返回这次 tick 的停顿量(毫秒,可为负→归零)。 */
39
58
  export function recordGap(
40
59
  gapMs: number,
41
- opts: { intervalMs?: number; thresholdMs?: number; rssMb?: number; now?: number } = {},
60
+ opts: { intervalMs?: number; thresholdMs?: number; rssMb?: number; now?: number; served?: number } = {},
42
61
  ): number {
43
62
  const interval = opts.intervalMs ?? 250;
44
63
  const threshold = opts.thresholdMs ?? 2000;
@@ -50,7 +69,13 @@ export function recordGap(
50
69
  stallCount++;
51
70
  lastStallAt = at;
52
71
  lastStallMs = stallMs;
53
- stalls.push({ at, gap_ms: Math.round(gapMs), stall_ms: Math.round(stallMs), rss_mb: opts.rssMb ?? 0 });
72
+ stalls.push({
73
+ at,
74
+ gap_ms: Math.round(gapMs),
75
+ stall_ms: Math.round(stallMs),
76
+ rss_mb: opts.rssMb ?? 0,
77
+ requests_since_last_tick: opts.served ?? 0,
78
+ });
54
79
  while (stalls.length > KEEP_STALLS) stalls.shift();
55
80
  return stallMs;
56
81
  }
@@ -109,10 +134,13 @@ export function startEventLoopLagMonitor(opts: {
109
134
  const gap = now - last;
110
135
  last = now;
111
136
  const rss = rssMb();
112
- const stall = recordGap(gap, { intervalMs, thresholdMs, rssMb: rss });
137
+ const served = servedSinceTick;
138
+ servedSinceTick = 0;
139
+ const stall = recordGap(gap, { intervalMs, thresholdMs, rssMb: rss, served });
113
140
  if (thresholdMs > 0 && stall >= thresholdMs) {
114
141
  // 和 retention/slow-sql 的日志同一前缀,方便 grep。
115
- log(`[commhub stall] loop blocked ~${Math.round(stall)}ms (tick gap ${Math.round(gap)}ms, rss ${rss} MB)`);
142
+ // `served` 是判据的关键:它非零就说明循环在干活,这段“gap”是 timer 被饿住,不是阻塞。
143
+ log(`[commhub stall] loop idle ~${Math.round(stall)}ms (tick gap ${Math.round(gap)}ms, rss ${rss} MB, served ${served} req)`);
116
144
  }
117
145
  }, intervalMs);
118
146
  // 监测不该拖住进程退出。
@@ -5,6 +5,8 @@
5
5
  // 2026-10-01 —— 原来是 4096 条样本的环:生产上 212 s 就写满,?minutes=10 实际只覆盖三分半钟。
6
6
  // 改成按分钟分桶的累计器:每个桶里每个路由一行(次数 / 总耗时 / 最大值 / 字节 / 5xx / 耗时直方图),
7
7
  // 保留 ROUTE_BUCKET_RETENTION 个桶(24 小时),窗口多长就真覆盖多长。p95 从直方图取(桶上沿,不超过 max)。
8
+ import { noteRequestServed } from "./event-loop-lag.js";
9
+
8
10
  export const ROUTE_BUCKET_MS = 60_000;
9
11
  export const ROUTE_BUCKET_RETENTION = 1440;
10
12
  /** 每个分钟桶里最多这么多个不同的路由;再多的记到 ROUTE_OVERFLOW_KEY,防止怪路径 / 怪工具名把内存撑大。 */
@@ -142,6 +144,8 @@ export function recordRouteTiming<T extends Response | undefined>(req: Request,
142
144
  const length = Number(res.headers.get("content-length"));
143
145
  const bytes = Number.isFinite(length) && res.headers.has("content-length") ? length : 0;
144
146
  addSample(route, performance.now() - started, res.status, bytes, now, callerClass(req));
147
+ // #2614 —— 让停顿监测能区分「循环被阻塞」与「timer 被饿住」:见 event-loop-lag.ts 顶部说明。
148
+ noteRequestServed();
145
149
  return res;
146
150
  }
147
151
 
@@ -0,0 +1,145 @@
1
+ // #2622 —— 定时任务「连续失败 → 间隔翻倍 → 成功即恢复」。
2
+ //
3
+ // 为什么需要它(生产 24h,2026-10-27):
4
+ // 一个 `every_seconds=180` 的排程 351 次失败 / 24 次成功 —— 失败与成功交替,
5
+ // 所以「连续失败」永远攒不到 #523 的自动暂停阈值,而**每一次失败都按 3 分钟原速重试**。
6
+ // `#523` 的机制(连续 5 次通知、可选自动暂停)都在,缺的只是「放慢」这一档。
7
+ //
8
+ // 这个文件钉住四件事:
9
+ // 1. 连续失败达到阈值 ⇒ `backoff_effective_seconds` 按 2 倍递增,且有上限;
10
+ // 2. `next_run_at` 只往后推,不会被“放大后又缩短”的抖动来回拉;
11
+ // 3. 成功一次 ⇒ 立刻清空退避,`next_run_at` 拉回正常节奏;
12
+ // 4. 与 `no_change_backoff` 互斥(那套逻辑自己拥有这一列),关掉开关时完全不介入。
13
+
14
+ import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
15
+ import { register } from "./auth.js";
16
+ import { db } from "./db.js";
17
+ import { applyFailureBackoff, failureBackoffRuns } from "./scheduled-failures.js";
18
+
19
+ const stamp = Date.now();
20
+ let NET = "";
21
+ let creatorId = "";
22
+ const ENV_KEY = "COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS";
23
+ const savedEnv = process.env[ENV_KEY];
24
+ let slot = 0;
25
+
26
+ function seedSchedule(everySeconds = 180, patch: Record<string, unknown> = {}) {
27
+ const scheduleId = `sched_bo_${crypto.randomUUID()}`;
28
+ db.run(
29
+ `INSERT INTO scheduled_tasks
30
+ (schedule_id, network_id, created_by, name, target_node_id, target_alias, task_content, priority,
31
+ schedule_type, schedule_json, timezone, status, next_run_at, overlap_policy)
32
+ VALUES (?1, ?2, ?3, 'backoff fixture', 'n_bo', 'bo-node', 'x', 'normal', 'interval', ?4,
33
+ 'Asia/Shanghai', 'active', ?5, 'skip')`,
34
+ [scheduleId, NET, creatorId, JSON.stringify({ type: "interval", every_seconds: everySeconds }),
35
+ new Date(Date.now() + everySeconds * 1000).toISOString()],
36
+ );
37
+ for (const [k, v] of Object.entries(patch)) db.run(`UPDATE scheduled_tasks SET ${k} = ?1 WHERE schedule_id = ?2`, [v as any, scheduleId]);
38
+ return scheduleId;
39
+ }
40
+
41
+ function addRun(scheduleId: string, status: string, minutesFromNow: number) {
42
+ const runId = `srun_bo_${crypto.randomUUID()}`;
43
+ db.run(
44
+ `INSERT INTO scheduled_task_runs (run_id, schedule_id, network_id, scheduled_for, status, created_at)
45
+ VALUES (?1, ?2, ?3, ?4, ?5, ?4)`,
46
+ [runId, scheduleId, NET, new Date(Date.now() + minutesFromNow * 60_000).toISOString(), status],
47
+ );
48
+ return runId;
49
+ }
50
+
51
+ const reload = (id: string) => db.get<any>("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", id)!;
52
+
53
+ beforeAll(async () => {
54
+ const auth = register(`bo_owner_${stamp}`, "Bo-Owner-Strong-1!", undefined, "tester");
55
+ expect(auth.ok).toBe(true);
56
+ NET = auth.network_id!;
57
+ creatorId = auth.user_id!;
58
+ process.env[ENV_KEY] = "6";
59
+ });
60
+
61
+ afterEach(() => {
62
+ process.env[ENV_KEY] = "6";
63
+ });
64
+
65
+ afterAll(() => {
66
+ try { db.run(`DELETE FROM scheduled_task_runs WHERE network_id = ?1`, [NET]); } catch {}
67
+ try { db.run(`DELETE FROM scheduled_tasks WHERE network_id = ?1`, [NET]); } catch {}
68
+ if (savedEnv === undefined) delete process.env[ENV_KEY]; else process.env[ENV_KEY] = savedEnv;
69
+ });
70
+
71
+ describe("#2622 failure backoff", () => {
72
+ test("默认阈值是 6,且可以关掉", () => {
73
+ delete process.env[ENV_KEY];
74
+ expect(failureBackoffRuns()).toBe(6);
75
+ process.env[ENV_KEY] = "0";
76
+ expect(failureBackoffRuns()).toBe(0);
77
+ });
78
+
79
+ test("失败次数未到阈值:不写退避", () => {
80
+ const id = seedSchedule();
81
+ for (let i = 0; i < 5; i++) addRun(id, "failed", -60 + i);
82
+ const out = applyFailureBackoff(id);
83
+ expect(out).toEqual({ streak: 5, effective_seconds: null });
84
+ expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
85
+ });
86
+
87
+ test("达到阈值后间隔翻倍,且 next_run_at 只会往后推", () => {
88
+ const id = seedSchedule(180);
89
+ for (let i = 0; i < 6; i++) addRun(id, "failed", -60 + i);
90
+ const before = reload(id).next_run_at;
91
+ const out = applyFailureBackoff(id);
92
+ expect(out!.effective_seconds).toBe(360); // 180 × 2
93
+ const after = reload(id);
94
+ expect(after.backoff_effective_seconds).toBe(360);
95
+ expect(Date.parse(after.next_run_at)).toBeGreaterThanOrEqual(Date.parse(before));
96
+ });
97
+
98
+ test("失败继续累积:指数放大且有 1 小时上限", () => {
99
+ const id = seedSchedule(180);
100
+ for (let i = 0; i < 12; i++) addRun(id, "failed", -120 + i);
101
+ const out = applyFailureBackoff(id);
102
+ // 180 × 2^k 会被 3600 截断
103
+ expect(out!.effective_seconds).toBeLessThanOrEqual(3600);
104
+ expect(out!.streak).toBe(12);
105
+ });
106
+
107
+ test("成功一次 ⇒ 立刻清空退避并把 next_run_at 拉回正常节奏", () => {
108
+ const id = seedSchedule(180);
109
+ for (let i = 0; i < 8; i++) addRun(id, "failed", -120 + i);
110
+ applyFailureBackoff(id);
111
+ expect(reload(id).backoff_effective_seconds).toBeGreaterThan(180);
112
+ addRun(id, "replied", -1);
113
+ const out = applyFailureBackoff(id);
114
+ expect(out!.streak).toBe(0);
115
+ expect(out!.effective_seconds).toBeNull();
116
+ expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
117
+ });
118
+
119
+ test("no_change_backoff 的排程不被这条逻辑改写", () => {
120
+ const id = seedSchedule(180, { no_change_backoff: 1 });
121
+ for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
122
+ expect(applyFailureBackoff(id)).toBeNull();
123
+ expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
124
+ });
125
+
126
+ test("非 interval 排程不参与", () => {
127
+ const id = seedSchedule(180, { schedule_type: "daily", schedule_json: JSON.stringify({ type: "daily", time: "09:00" }) });
128
+ for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
129
+ expect(applyFailureBackoff(id)).toBeNull();
130
+ });
131
+
132
+ test("paused 的排程不参与(暂停已经把它停住了)", () => {
133
+ const id = seedSchedule(180, { status: "paused", next_run_at: null });
134
+ for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
135
+ expect(applyFailureBackoff(id)).toBeNull();
136
+ });
137
+
138
+ test("开关设为 0 时完全不介入", () => {
139
+ const id = seedSchedule(180);
140
+ for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
141
+ process.env[ENV_KEY] = "0";
142
+ expect(applyFailureBackoff(id)).toBeNull();
143
+ expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
144
+ });
145
+ });
@@ -167,6 +167,65 @@ export function evaluateScheduleFailures(scheduleId: string, now = new Date()):
167
167
  return notice;
168
168
  }
169
169
 
170
+ /**
171
+ * #2622 —— 失败退避:连续失败达到 N 次后把**间隔翻倍**(上限 1 小时),成功一次立刻恢复原节奏。
172
+ *
173
+ * 为什么不是「暂停」:`#523` 已经有 `COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS`(默认关),
174
+ * 但它会把排程**停下来**,需要人来恢复。对“上游额度暂时打满”这类外部原因,暂停太久同样是损失;
175
+ * 退避只是**放慢重试**,外部原因一恢复就会自己跑回原节奏。
176
+ *
177
+ * 现场(2026-10-27 生产 24h):一个 `every_seconds=180` 的排程失败 351 次、成功 24 次 ——
178
+ * 失败与成功交替出现,所以“连续失败”很少攒到暂停阈值,而**每次失败都在按 3 分钟原速重试**。
179
+ *
180
+ * 存储复用 `backoff_effective_seconds`(派发时按 `max(every_seconds, backoff_effective_seconds)` 生效),
181
+ * 与 `no_change_backoff` 互斥:那套逻辑自己会写这一列,这里只在 `no_change_backoff = 0` 时介入。
182
+ *
183
+ * 环境变量:`COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS`(默认 6;0 = 关闭)。
184
+ */
185
+ export function failureBackoffRuns(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS", 6, 0); }
186
+
187
+ const FAILURE_BACKOFF_CAP_SECONDS = 3600;
188
+
189
+ export function applyFailureBackoff(scheduleId: string, now = new Date()): { streak: number; effective_seconds: number | null } | null {
190
+ const from = failureBackoffRuns();
191
+ if (from <= 0) return null;
192
+ const row = db.get<any>(
193
+ `SELECT schedule_id, revision, status, schedule_type, schedule_json, no_change_backoff,
194
+ backoff_effective_seconds, next_run_at
195
+ FROM scheduled_tasks WHERE schedule_id = ?1`,
196
+ scheduleId,
197
+ );
198
+ if (!row || row.status !== "active" || row.no_change_backoff === 1) return null;
199
+ if (row.schedule_type !== "interval") return null;
200
+ let spec: any = null;
201
+ try { spec = JSON.parse(row.schedule_json || "{}"); } catch { spec = null; }
202
+ const base = spec && spec.type === "interval" ? Number(spec.every_seconds) : NaN;
203
+ if (!Number.isFinite(base) || base <= 0) return null;
204
+
205
+ const streak = failureStreak(scheduleId).consecutiveFailures;
206
+ const effective = streak >= from
207
+ ? Math.min(FAILURE_BACKOFF_CAP_SECONDS, base * 2 ** Math.min(streak - from + 1, 6))
208
+ : null;
209
+ const target = effective ?? base;
210
+ const previous = row.backoff_effective_seconds ?? base;
211
+ let next = row.next_run_at;
212
+ if (target !== previous && next) {
213
+ // 放大:至少推到 now + target(不把已经更远的 next 拉近);
214
+ // 恢复:至少拉回 now + target(不把已经更近的 next 推远)。
215
+ const candidate = new Date(now.getTime() + target * 1000).toISOString();
216
+ const existing = Date.parse(next);
217
+ if (Number.isFinite(existing)) {
218
+ next = new Date(target > previous ? Math.max(existing, Date.parse(candidate)) : Math.min(existing, Date.parse(candidate))).toISOString();
219
+ }
220
+ }
221
+ db.run(
222
+ `UPDATE scheduled_tasks SET backoff_streak = ?1, backoff_effective_seconds = ?2, next_run_at = ?3,
223
+ updated_at = datetime('now') WHERE schedule_id = ?4 AND revision = ?5 AND status = 'active'`,
224
+ [streak, effective, next, scheduleId, row.revision],
225
+ );
226
+ return { streak, effective_seconds: effective };
227
+ }
228
+
170
229
  /** GET /runs 的附加字段。 */
171
230
  export function failureSummary(scheduleId: string): { consecutive_failures: number; failure_alert_threshold: number; last_failure_alert_at: string | null } {
172
231
  const streak = failureStreak(scheduleId);
@@ -176,4 +235,10 @@ export function failureSummary(scheduleId: string): { consecutive_failures: numb
176
235
 
177
236
  setScheduledRunTerminalHook((scheduleId, status) => {
178
237
  if (status === "failed") evaluateScheduleFailures(scheduleId);
238
+ // #2622 —— 成功一次就把间隔恢复原样;失败则按连续失败次数放大间隔。
239
+ try {
240
+ applyFailureBackoff(scheduleId);
241
+ } catch (e: any) {
242
+ console.error(`[scheduled-tasks] failure backoff failed schedule=${scheduleId}: ${e?.message || e}`);
243
+ }
179
244
  });