@sleep2agi/commhub-server 0.9.0-preview.162 → 0.9.0-preview.163
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/backfill-node-ids-2664.test.ts +128 -0
- package/src/db.ts +101 -100
- package/src/event-loop-lag.test.ts +13 -1
- package/src/event-loop-lag.ts +32 -4
- package/src/route-timing.ts +4 -0
- package/src/scheduled-failure-backoff-2622.test.ts +145 -0
- package/src/scheduled-failures.ts +65 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sleep2agi/commhub-server",
|
|
3
|
-
"version": "0.9.0-preview.
|
|
3
|
+
"version": "0.9.0-preview.163",
|
|
4
4
|
"description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
// #2664 —— `backfillMessageNodeIds()` 从「逐行相关子查询」改成「物化的别名→node_id 表 + UPDATE … FROM」。
|
|
2
|
+
//
|
|
3
|
+
// 这次改写**只为了快**(生产副本实测单条 6.56s → 0.29s;boot 期六条约 10s → 约 0.75s),
|
|
4
|
+
// 而“快”最容易顺手改坏的是**选哪一行 / 谁不该被改**:
|
|
5
|
+
// · 人的用户名不能被映射成节点(`NOT EXISTS users.username`);
|
|
6
|
+
// · 改过名的节点走 rename_txn(只认 `committed`);
|
|
7
|
+
// · 已经有 node_id 的行**不能被覆盖**;
|
|
8
|
+
// · 重复调用幂等。
|
|
9
|
+
//
|
|
10
|
+
// 注:`sessions` 上有 `UNIQUE (network_id, alias)` 且 `network_id NOT NULL`,
|
|
11
|
+
// 所以“同一 alias 多条 session 时取 updated_at 最新”在当前 schema 下**不可构造**;
|
|
12
|
+
// 窗口函数只是为了在 PG / 老库形状下保持与旧 `ORDER BY … LIMIT 1` 等价,不是新语义。
|
|
13
|
+
|
|
14
|
+
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
15
|
+
import { db, backfillMessageNodeIds } from "./db.js";
|
|
16
|
+
|
|
17
|
+
let networkId = "";
|
|
18
|
+
const stamp = `${Date.now()}`;
|
|
19
|
+
|
|
20
|
+
function session(alias: string, nodeId: string | null, updatedAt: string, network = networkId) {
|
|
21
|
+
db.run(
|
|
22
|
+
`INSERT INTO sessions (resume_id, alias, node_id, status, network_id, registered_at, updated_at)
|
|
23
|
+
VALUES (?1, ?2, ?3, 'idle', ?4, ?5, ?5)`,
|
|
24
|
+
[`res_${alias}_${stamp}`, alias, nodeId, network, updatedAt],
|
|
25
|
+
);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function task(id: string, fromName: string, toName: string, network = networkId) {
|
|
29
|
+
db.run(
|
|
30
|
+
`INSERT INTO tasks (task_id, from_name, to_name, content, status, network_id)
|
|
31
|
+
VALUES (?1, ?2, ?3, 'fixture', 'delivered', ?4)`,
|
|
32
|
+
[id, fromName, toName, network],
|
|
33
|
+
);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function nodeIds(id: string) {
|
|
37
|
+
return db.get<{ from_node_id: string | null; to_node_id: string | null }>(
|
|
38
|
+
"SELECT from_node_id, to_node_id FROM tasks WHERE task_id = ?1",
|
|
39
|
+
[`${id}-${stamp}`],
|
|
40
|
+
)!;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
beforeAll(async () => {
|
|
44
|
+
const { register } = await import("./auth.js");
|
|
45
|
+
const auth = register(`bf_owner_${stamp}`, "Bf-Owner-Strong-1!", undefined, "tester");
|
|
46
|
+
expect(auth.ok).toBe(true);
|
|
47
|
+
networkId = auth.network_id!;
|
|
48
|
+
|
|
49
|
+
session(`bf-node-${stamp}`, `node-${stamp}`, "2026-10-20 00:00:00");
|
|
50
|
+
// 人类用户:即便 sessions 里出现过同名行,也不能把任务归到那个 node
|
|
51
|
+
session(`bf-human-${stamp}`, `node-should-not-win-${stamp}`, "2026-10-20 00:00:00");
|
|
52
|
+
db.run(
|
|
53
|
+
`INSERT INTO users (user_id, username, display_name, role, password_hash, created_at)
|
|
54
|
+
VALUES (?1, ?2, '人类', 'member', 'x', datetime('now'))`,
|
|
55
|
+
[`u_bf_${stamp}`, `bf-human-${stamp}`],
|
|
56
|
+
);
|
|
57
|
+
|
|
58
|
+
task(`bf-fresh-${stamp}`, `bf-node-${stamp}`, `bf-node-${stamp}`);
|
|
59
|
+
task(`bf-human-task-${stamp}`, `bf-human-${stamp}`, `bf-node-${stamp}`);
|
|
60
|
+
task(`bf-preset-${stamp}`, `bf-node-${stamp}`, `bf-node-${stamp}`);
|
|
61
|
+
db.run(`UPDATE tasks SET from_node_id = 'preset-node', to_node_id = 'preset-node' WHERE task_id = ?1`, [
|
|
62
|
+
`bf-preset-${stamp}`,
|
|
63
|
+
]);
|
|
64
|
+
|
|
65
|
+
// rename_txn:旧别名 → 新别名(committed 生效;pending 不生效)
|
|
66
|
+
session(`bf-new-alias-${stamp}`, `node-renamed-${stamp}`, "2026-10-20 00:00:00");
|
|
67
|
+
db.run(
|
|
68
|
+
`INSERT INTO rename_txn (txn_id, old_alias, new_alias, status, network_id, committed_at)
|
|
69
|
+
VALUES (?1, ?2, ?3, 'committed', ?4, '2026-10-21 00:00:00')`,
|
|
70
|
+
[`txn_bf_${stamp}`, `bf-old-alias-${stamp}`, `bf-new-alias-${stamp}`, networkId],
|
|
71
|
+
);
|
|
72
|
+
task(`bf-renamed-${stamp}`, `bf-old-alias-${stamp}`, `bf-old-alias-${stamp}`);
|
|
73
|
+
|
|
74
|
+
db.run(
|
|
75
|
+
`INSERT INTO rename_txn (txn_id, old_alias, new_alias, status, network_id, committed_at)
|
|
76
|
+
VALUES (?1, ?2, ?3, 'pending', ?4, NULL)`,
|
|
77
|
+
[`txn_bf_pending_${stamp}`, `bf-pending-alias-${stamp}`, `bf-new-alias-${stamp}`, networkId],
|
|
78
|
+
);
|
|
79
|
+
task(`bf-pending-${stamp}`, `bf-pending-alias-${stamp}`, `bf-pending-alias-${stamp}`);
|
|
80
|
+
|
|
81
|
+
backfillMessageNodeIds();
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
afterAll(() => {
|
|
85
|
+
try {
|
|
86
|
+
db.run(`DELETE FROM tasks WHERE task_id LIKE ?1`, [`bf-%-${stamp}`]);
|
|
87
|
+
db.run(`DELETE FROM sessions WHERE alias LIKE ?1`, [`bf-%-${stamp}`]);
|
|
88
|
+
db.run(`DELETE FROM rename_txn WHERE txn_id LIKE ?1`, [`txn_bf%${stamp}`]);
|
|
89
|
+
db.run(`DELETE FROM users WHERE username LIKE ?1`, [`bf-human-${stamp}`]);
|
|
90
|
+
} catch {}
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
describe("#2664 backfill semantics unchanged by the join rewrite", () => {
|
|
94
|
+
test("普通别名能补上 from/to 两侧的 node_id", () => {
|
|
95
|
+
expect(nodeIds("bf-fresh").from_node_id).toBe(`node-${stamp}`);
|
|
96
|
+
expect(nodeIds("bf-fresh").to_node_id).toBe(`node-${stamp}`);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test("人的用户名不会被映射成节点", () => {
|
|
100
|
+
expect(nodeIds("bf-human-task").from_node_id).toBeNull();
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test("已经有 node_id 的行不被覆盖", () => {
|
|
104
|
+
expect(nodeIds("bf-preset").from_node_id).toBe("preset-node");
|
|
105
|
+
expect(nodeIds("bf-preset").to_node_id).toBe("preset-node");
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
test("改过名的节点按 rename_txn 的旧别名补上", () => {
|
|
109
|
+
expect(nodeIds("bf-renamed").from_node_id).toBe(`node-renamed-${stamp}`);
|
|
110
|
+
expect(nodeIds("bf-renamed").to_node_id).toBe(`node-renamed-${stamp}`);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
test("未提交(pending)的改名不生效", () => {
|
|
114
|
+
expect(nodeIds("bf-pending").from_node_id).toBeNull();
|
|
115
|
+
expect(nodeIds("bf-pending").to_node_id).toBeNull();
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
test("第二次调用是幂等的", () => {
|
|
119
|
+
const count = () =>
|
|
120
|
+
db.get<{ n: number }>(
|
|
121
|
+
`SELECT COUNT(*) AS n FROM tasks WHERE network_id = ?1 AND (from_node_id IS NOT NULL OR to_node_id IS NOT NULL)`,
|
|
122
|
+
[networkId],
|
|
123
|
+
)!.n;
|
|
124
|
+
const before = count();
|
|
125
|
+
backfillMessageNodeIds();
|
|
126
|
+
expect(count()).toBe(before);
|
|
127
|
+
});
|
|
128
|
+
});
|
package/src/db.ts
CHANGED
|
@@ -1303,6 +1303,18 @@ try { db.exec("CREATE INDEX IF NOT EXISTS idx_tasks_to_node ON tasks(to_node_id)
|
|
|
1303
1303
|
try { db.exec("CREATE INDEX IF NOT EXISTS idx_inbox_node ON inbox(node_id)"); } catch {}
|
|
1304
1304
|
|
|
1305
1305
|
export function backfillMessageNodeIds() {
|
|
1306
|
+
// #2664 —— 旧写法是 6 条 `UPDATE … SET x = (SELECT … WHERE alias = t.name …)`:
|
|
1307
|
+
// 每条都对 **84k 行 tasks.from_node_id IS NULL / 36k 行 inbox.node_id IS NULL** 逐行
|
|
1308
|
+
// 跑一次相关子查询,而子查询里的 `COALESCE(network_id,'default')` 让
|
|
1309
|
+
// idx_sessions_network_alias_unique 用不上 ⇒ 每行扫一遍 369 行 sessions。
|
|
1310
|
+
// 生产实测(`.162` 慢 SQL 日志,2026-10-27 00:21 重启):5398ms / 2451ms / 1270ms,
|
|
1311
|
+
// 而且当次命中 0 行 —— **结果是 0,代价不是 0**。
|
|
1312
|
+
//
|
|
1313
|
+
// 改成 `UPDATE … FROM (物化后的别名→node_id 表)`:sessions/rename_txn 只扫一遍,
|
|
1314
|
+
// 每行只做一次自动索引探测。同一份生产库副本上 6.56s → 0.29s。
|
|
1315
|
+
// 语义不变:同一 (alias, network) 取 `updated_at` / `committed_at` 最新的那一行;
|
|
1316
|
+
// 仍然是「先按 session 别名直连,再补 rename_txn 的旧别名」;对 tasks 仍排除人是发送方的情况。
|
|
1317
|
+
const t0 = performance.now();
|
|
1306
1318
|
let inboxDirect = 0;
|
|
1307
1319
|
let inboxRenamed = 0;
|
|
1308
1320
|
let tasksToDirect = 0;
|
|
@@ -1313,157 +1325,146 @@ export function backfillMessageNodeIds() {
|
|
|
1313
1325
|
try {
|
|
1314
1326
|
inboxDirect = db.run(`
|
|
1315
1327
|
UPDATE inbox
|
|
1316
|
-
SET node_id =
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
)
|
|
1328
|
+
SET node_id = pick.node_id
|
|
1329
|
+
FROM (
|
|
1330
|
+
SELECT alias, COALESCE(network_id, 'default') AS net, node_id,
|
|
1331
|
+
ROW_NUMBER() OVER (
|
|
1332
|
+
PARTITION BY alias, COALESCE(network_id, 'default')
|
|
1333
|
+
ORDER BY updated_at DESC
|
|
1334
|
+
) AS rn
|
|
1335
|
+
FROM sessions
|
|
1336
|
+
WHERE node_id IS NOT NULL
|
|
1337
|
+
) pick
|
|
1338
|
+
WHERE pick.rn = 1
|
|
1339
|
+
AND pick.alias = inbox.session_name
|
|
1340
|
+
AND pick.net = COALESCE(inbox.network_id, 'default')
|
|
1341
|
+
AND inbox.node_id IS NULL
|
|
1331
1342
|
`).changes;
|
|
1332
1343
|
} catch {}
|
|
1333
1344
|
|
|
1334
1345
|
try {
|
|
1335
1346
|
inboxRenamed = db.run(`
|
|
1336
1347
|
UPDATE inbox
|
|
1337
|
-
SET node_id =
|
|
1338
|
-
|
|
1348
|
+
SET node_id = pick.node_id
|
|
1349
|
+
FROM (
|
|
1350
|
+
SELECT r.old_alias AS old_alias,
|
|
1351
|
+
COALESCE(r.network_id, 'default') AS net,
|
|
1352
|
+
s.node_id AS node_id,
|
|
1353
|
+
ROW_NUMBER() OVER (
|
|
1354
|
+
PARTITION BY r.old_alias, COALESCE(r.network_id, 'default')
|
|
1355
|
+
ORDER BY r.committed_at DESC
|
|
1356
|
+
) AS rn
|
|
1357
|
+
FROM rename_txn r
|
|
1339
1358
|
JOIN sessions s
|
|
1340
1359
|
ON s.alias = r.new_alias
|
|
1341
1360
|
AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
|
|
1342
1361
|
WHERE s.node_id IS NOT NULL
|
|
1343
1362
|
AND r.status = 'committed'
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
WHERE node_id IS NULL
|
|
1350
|
-
AND EXISTS (
|
|
1351
|
-
SELECT 1 FROM rename_txn r
|
|
1352
|
-
JOIN sessions s
|
|
1353
|
-
ON s.alias = r.new_alias
|
|
1354
|
-
AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
|
|
1355
|
-
WHERE s.node_id IS NOT NULL
|
|
1356
|
-
AND r.status = 'committed'
|
|
1357
|
-
AND r.old_alias = inbox.session_name
|
|
1358
|
-
AND COALESCE(r.network_id, 'default') = COALESCE(inbox.network_id, 'default')
|
|
1359
|
-
)
|
|
1363
|
+
) pick
|
|
1364
|
+
WHERE pick.rn = 1
|
|
1365
|
+
AND pick.old_alias = inbox.session_name
|
|
1366
|
+
AND pick.net = COALESCE(inbox.network_id, 'default')
|
|
1367
|
+
AND inbox.node_id IS NULL
|
|
1360
1368
|
`).changes;
|
|
1361
1369
|
} catch {}
|
|
1362
1370
|
|
|
1363
1371
|
try {
|
|
1364
1372
|
tasksToDirect = db.run(`
|
|
1365
1373
|
UPDATE tasks
|
|
1366
|
-
SET to_node_id =
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
)
|
|
1374
|
+
SET to_node_id = pick.node_id
|
|
1375
|
+
FROM (
|
|
1376
|
+
SELECT alias, COALESCE(network_id, 'default') AS net, node_id,
|
|
1377
|
+
ROW_NUMBER() OVER (
|
|
1378
|
+
PARTITION BY alias, COALESCE(network_id, 'default')
|
|
1379
|
+
ORDER BY updated_at DESC
|
|
1380
|
+
) AS rn
|
|
1381
|
+
FROM sessions
|
|
1382
|
+
WHERE node_id IS NOT NULL
|
|
1383
|
+
) pick
|
|
1384
|
+
WHERE pick.rn = 1
|
|
1385
|
+
AND pick.alias = tasks.to_name
|
|
1386
|
+
AND pick.net = COALESCE(tasks.network_id, 'default')
|
|
1387
|
+
AND tasks.to_node_id IS NULL
|
|
1381
1388
|
`).changes;
|
|
1382
1389
|
} catch {}
|
|
1383
1390
|
|
|
1384
1391
|
try {
|
|
1385
1392
|
tasksToRenamed = db.run(`
|
|
1386
1393
|
UPDATE tasks
|
|
1387
|
-
SET to_node_id =
|
|
1388
|
-
|
|
1394
|
+
SET to_node_id = pick.node_id
|
|
1395
|
+
FROM (
|
|
1396
|
+
SELECT r.old_alias AS old_alias,
|
|
1397
|
+
COALESCE(r.network_id, 'default') AS net,
|
|
1398
|
+
s.node_id AS node_id,
|
|
1399
|
+
ROW_NUMBER() OVER (
|
|
1400
|
+
PARTITION BY r.old_alias, COALESCE(r.network_id, 'default')
|
|
1401
|
+
ORDER BY r.committed_at DESC
|
|
1402
|
+
) AS rn
|
|
1403
|
+
FROM rename_txn r
|
|
1389
1404
|
JOIN sessions s
|
|
1390
1405
|
ON s.alias = r.new_alias
|
|
1391
1406
|
AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
|
|
1392
1407
|
WHERE s.node_id IS NOT NULL
|
|
1393
1408
|
AND r.status = 'committed'
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
WHERE to_node_id IS NULL
|
|
1400
|
-
AND EXISTS (
|
|
1401
|
-
SELECT 1 FROM rename_txn r
|
|
1402
|
-
JOIN sessions s
|
|
1403
|
-
ON s.alias = r.new_alias
|
|
1404
|
-
AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
|
|
1405
|
-
WHERE s.node_id IS NOT NULL
|
|
1406
|
-
AND r.status = 'committed'
|
|
1407
|
-
AND r.old_alias = tasks.to_name
|
|
1408
|
-
AND COALESCE(r.network_id, 'default') = COALESCE(tasks.network_id, 'default')
|
|
1409
|
-
)
|
|
1409
|
+
) pick
|
|
1410
|
+
WHERE pick.rn = 1
|
|
1411
|
+
AND pick.old_alias = tasks.to_name
|
|
1412
|
+
AND pick.net = COALESCE(tasks.network_id, 'default')
|
|
1413
|
+
AND tasks.to_node_id IS NULL
|
|
1410
1414
|
`).changes;
|
|
1411
1415
|
} catch {}
|
|
1412
1416
|
|
|
1413
1417
|
try {
|
|
1414
1418
|
tasksFromDirect = db.run(`
|
|
1415
1419
|
UPDATE tasks
|
|
1416
|
-
SET from_node_id =
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1420
|
+
SET from_node_id = pick.node_id
|
|
1421
|
+
FROM (
|
|
1422
|
+
SELECT alias, COALESCE(network_id, 'default') AS net, node_id,
|
|
1423
|
+
ROW_NUMBER() OVER (
|
|
1424
|
+
PARTITION BY alias, COALESCE(network_id, 'default')
|
|
1425
|
+
ORDER BY updated_at DESC
|
|
1426
|
+
) AS rn
|
|
1427
|
+
FROM sessions
|
|
1428
|
+
WHERE node_id IS NOT NULL
|
|
1429
|
+
) pick
|
|
1430
|
+
WHERE pick.rn = 1
|
|
1431
|
+
AND pick.alias = tasks.from_name
|
|
1432
|
+
AND pick.net = COALESCE(tasks.network_id, 'default')
|
|
1433
|
+
AND tasks.from_node_id IS NULL
|
|
1425
1434
|
AND NOT EXISTS (SELECT 1 FROM users u WHERE u.username = tasks.from_name)
|
|
1426
|
-
AND EXISTS (
|
|
1427
|
-
SELECT 1 FROM sessions s
|
|
1428
|
-
WHERE s.node_id IS NOT NULL
|
|
1429
|
-
AND s.alias = tasks.from_name
|
|
1430
|
-
AND COALESCE(s.network_id, 'default') = COALESCE(tasks.network_id, 'default')
|
|
1431
|
-
)
|
|
1432
1435
|
`).changes;
|
|
1433
1436
|
} catch {}
|
|
1434
1437
|
|
|
1435
1438
|
try {
|
|
1436
1439
|
tasksFromRenamed = db.run(`
|
|
1437
1440
|
UPDATE tasks
|
|
1438
|
-
SET from_node_id =
|
|
1439
|
-
|
|
1441
|
+
SET from_node_id = pick.node_id
|
|
1442
|
+
FROM (
|
|
1443
|
+
SELECT r.old_alias AS old_alias,
|
|
1444
|
+
COALESCE(r.network_id, 'default') AS net,
|
|
1445
|
+
s.node_id AS node_id,
|
|
1446
|
+
ROW_NUMBER() OVER (
|
|
1447
|
+
PARTITION BY r.old_alias, COALESCE(r.network_id, 'default')
|
|
1448
|
+
ORDER BY r.committed_at DESC
|
|
1449
|
+
) AS rn
|
|
1450
|
+
FROM rename_txn r
|
|
1440
1451
|
JOIN sessions s
|
|
1441
1452
|
ON s.alias = r.new_alias
|
|
1442
1453
|
AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
|
|
1443
1454
|
WHERE s.node_id IS NOT NULL
|
|
1444
1455
|
AND r.status = 'committed'
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
WHERE from_node_id IS NULL
|
|
1456
|
+
) pick
|
|
1457
|
+
WHERE pick.rn = 1
|
|
1458
|
+
AND pick.old_alias = tasks.from_name
|
|
1459
|
+
AND pick.net = COALESCE(tasks.network_id, 'default')
|
|
1460
|
+
AND tasks.from_node_id IS NULL
|
|
1451
1461
|
AND NOT EXISTS (SELECT 1 FROM users u WHERE u.username = tasks.from_name)
|
|
1452
|
-
AND EXISTS (
|
|
1453
|
-
SELECT 1 FROM rename_txn r
|
|
1454
|
-
JOIN sessions s
|
|
1455
|
-
ON s.alias = r.new_alias
|
|
1456
|
-
AND COALESCE(s.network_id, 'default') = COALESCE(r.network_id, 'default')
|
|
1457
|
-
WHERE s.node_id IS NOT NULL
|
|
1458
|
-
AND r.status = 'committed'
|
|
1459
|
-
AND r.old_alias = tasks.from_name
|
|
1460
|
-
AND COALESCE(r.network_id, 'default') = COALESCE(tasks.network_id, 'default')
|
|
1461
|
-
)
|
|
1462
1462
|
`).changes;
|
|
1463
1463
|
} catch {}
|
|
1464
1464
|
|
|
1465
1465
|
const total = inboxDirect + inboxRenamed + tasksToDirect + tasksToRenamed + tasksFromDirect + tasksFromRenamed;
|
|
1466
|
-
|
|
1466
|
+
// #2664 —— 旧日志只有条数,于是「total=0 却花了 10 秒」在生产里完全看不见。
|
|
1467
|
+
console.log(`[commhub] node_id backfill: inbox=${inboxDirect + inboxRenamed}, tasks.to=${tasksToDirect + tasksToRenamed}, tasks.from=${tasksFromDirect + tasksFromRenamed}, total=${total}, took=${Math.round(performance.now() - t0)}ms`);
|
|
1467
1468
|
}
|
|
1468
1469
|
|
|
1469
1470
|
backfillMessageNodeIds();
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { describe, expect, test } from "bun:test";
|
|
9
9
|
import {
|
|
10
10
|
eventLoopStats,
|
|
11
|
+
noteRequestServed,
|
|
11
12
|
recordGap,
|
|
12
13
|
resetEventLoopLagForTests,
|
|
13
14
|
startEventLoopLagMonitor,
|
|
@@ -50,6 +51,16 @@ describe("event-loop-lag: recordGap", () => {
|
|
|
50
51
|
expect(s.recent_stalls[19].gap_ms).toBe(5_005);
|
|
51
52
|
});
|
|
52
53
|
|
|
54
|
+
test("记录这段间隔里服务了多少请求(区分“阻塞”与“timer 饥饿”)", () => {
|
|
55
|
+
resetEventLoopLagForTests();
|
|
56
|
+
recordGap(85_000, { intervalMs: 250, thresholdMs: 2_000, rssMb: 2076, served: 137 });
|
|
57
|
+
const s = eventLoopStats();
|
|
58
|
+
expect(s.recent_stalls[0].requests_since_last_tick).toBe(137);
|
|
59
|
+
// 0 服务数 = 这段时间真的什么都没推进
|
|
60
|
+
recordGap(30_000, { intervalMs: 250, thresholdMs: 2_000, rssMb: 500, served: 0 });
|
|
61
|
+
expect(eventLoopStats().recent_stalls[0].requests_since_last_tick).toBe(0);
|
|
62
|
+
});
|
|
63
|
+
|
|
53
64
|
test("max_gap_ms 保留历史最大值(回落后不被覆盖)", () => {
|
|
54
65
|
resetEventLoopLagForTests();
|
|
55
66
|
recordGap(47_000, { intervalMs: 250, thresholdMs: 2_000 });
|
|
@@ -93,8 +104,9 @@ describe("event-loop-lag: startEventLoopLagMonitor", () => {
|
|
|
93
104
|
await Bun.sleep(60);
|
|
94
105
|
clearInterval(timer);
|
|
95
106
|
expect(logs.length).toBeGreaterThan(0);
|
|
96
|
-
expect(logs[0]).toStartWith("[commhub stall] loop
|
|
107
|
+
expect(logs[0]).toStartWith("[commhub stall] loop idle ~");
|
|
97
108
|
expect(logs[0]).toContain("rss 4502 MB");
|
|
109
|
+
expect(logs[0]).toContain("served 0 req");
|
|
98
110
|
});
|
|
99
111
|
});
|
|
100
112
|
|
package/src/event-loop-lag.ts
CHANGED
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
// 停顿结束后第一次 tick 的 gap 就是停顿长度,不需要外部进程配合。
|
|
17
17
|
|
|
18
18
|
export type EventLoopStall = {
|
|
19
|
+
/** 这个间隔里处理完的请求数:0 = 真的没推进;>0 = 循环在干活,只是 timer 被饿住。 */
|
|
19
20
|
/** ISO 时间(停顿**结束**的时刻,即这次 tick 的时间)。 */
|
|
20
21
|
at: string;
|
|
21
22
|
/** 距上次 tick 的实际间隔(毫秒)。等于 interval + 停顿。 */
|
|
@@ -24,10 +25,28 @@ export type EventLoopStall = {
|
|
|
24
25
|
stall_ms: number;
|
|
25
26
|
/** 停顿结束时的进程 RSS(MB),用于和内存尖峰对齐。 */
|
|
26
27
|
rss_mb: number;
|
|
28
|
+
requests_since_last_tick?: number;
|
|
27
29
|
};
|
|
28
30
|
|
|
29
31
|
const KEEP_STALLS = 20;
|
|
30
32
|
|
|
33
|
+
/**
|
|
34
|
+
* #2614(第二轮)—— 2026-10-27 00:39 线上出现 gap 84887ms,但**同一时段日志在正常写**
|
|
35
|
+
* (00:38:52 / 00:38:58 / 00:39:08 …),而且 `/api/stats/routes` 里最慢的请求只有 2.4s。
|
|
36
|
+
* 也就是说:事件循环**没有被阻塞 84 秒**,是 **timer 回调被 I/O 队列饿住**了。
|
|
37
|
+
* 只看 tick 间隔会把这两种情况混成一条“卡了 84 秒”的假结论。
|
|
38
|
+
*
|
|
39
|
+
* 区分办法:数一下这个间隔里**服务了多少请求**。
|
|
40
|
+
* requests_since_last_tick > 0 ⇒ 循环在干活 ⇒ 这是 timer 饥饿(高负载/大批回调)
|
|
41
|
+
* requests_since_last_tick = 0 ⇒ 真的没有推进任何请求 ⇒ 才可能是同步阻塞
|
|
42
|
+
*/
|
|
43
|
+
let servedSinceTick = 0;
|
|
44
|
+
|
|
45
|
+
/** 请求处理完成时调用(route-timing 里)。只计数,不参与任何决策。 */
|
|
46
|
+
export function noteRequestServed(): void {
|
|
47
|
+
servedSinceTick++;
|
|
48
|
+
}
|
|
49
|
+
|
|
31
50
|
let maxGapMs = 0;
|
|
32
51
|
let lastGapMs = 0;
|
|
33
52
|
let stallCount = 0;
|
|
@@ -38,7 +57,7 @@ const stalls: EventLoopStall[] = [];
|
|
|
38
57
|
/** 记录一次 tick 间隔。返回这次 tick 的停顿量(毫秒,可为负→归零)。 */
|
|
39
58
|
export function recordGap(
|
|
40
59
|
gapMs: number,
|
|
41
|
-
opts: { intervalMs?: number; thresholdMs?: number; rssMb?: number; now?: number } = {},
|
|
60
|
+
opts: { intervalMs?: number; thresholdMs?: number; rssMb?: number; now?: number; served?: number } = {},
|
|
42
61
|
): number {
|
|
43
62
|
const interval = opts.intervalMs ?? 250;
|
|
44
63
|
const threshold = opts.thresholdMs ?? 2000;
|
|
@@ -50,7 +69,13 @@ export function recordGap(
|
|
|
50
69
|
stallCount++;
|
|
51
70
|
lastStallAt = at;
|
|
52
71
|
lastStallMs = stallMs;
|
|
53
|
-
stalls.push({
|
|
72
|
+
stalls.push({
|
|
73
|
+
at,
|
|
74
|
+
gap_ms: Math.round(gapMs),
|
|
75
|
+
stall_ms: Math.round(stallMs),
|
|
76
|
+
rss_mb: opts.rssMb ?? 0,
|
|
77
|
+
requests_since_last_tick: opts.served ?? 0,
|
|
78
|
+
});
|
|
54
79
|
while (stalls.length > KEEP_STALLS) stalls.shift();
|
|
55
80
|
return stallMs;
|
|
56
81
|
}
|
|
@@ -109,10 +134,13 @@ export function startEventLoopLagMonitor(opts: {
|
|
|
109
134
|
const gap = now - last;
|
|
110
135
|
last = now;
|
|
111
136
|
const rss = rssMb();
|
|
112
|
-
const
|
|
137
|
+
const served = servedSinceTick;
|
|
138
|
+
servedSinceTick = 0;
|
|
139
|
+
const stall = recordGap(gap, { intervalMs, thresholdMs, rssMb: rss, served });
|
|
113
140
|
if (thresholdMs > 0 && stall >= thresholdMs) {
|
|
114
141
|
// 和 retention/slow-sql 的日志同一前缀,方便 grep。
|
|
115
|
-
|
|
142
|
+
// `served` 是判据的关键:它非零就说明循环在干活,这段“gap”是 timer 被饿住,不是阻塞。
|
|
143
|
+
log(`[commhub stall] loop idle ~${Math.round(stall)}ms (tick gap ${Math.round(gap)}ms, rss ${rss} MB, served ${served} req)`);
|
|
116
144
|
}
|
|
117
145
|
}, intervalMs);
|
|
118
146
|
// 监测不该拖住进程退出。
|
package/src/route-timing.ts
CHANGED
|
@@ -5,6 +5,8 @@
|
|
|
5
5
|
// 2026-10-01 —— 原来是 4096 条样本的环:生产上 212 s 就写满,?minutes=10 实际只覆盖三分半钟。
|
|
6
6
|
// 改成按分钟分桶的累计器:每个桶里每个路由一行(次数 / 总耗时 / 最大值 / 字节 / 5xx / 耗时直方图),
|
|
7
7
|
// 保留 ROUTE_BUCKET_RETENTION 个桶(24 小时),窗口多长就真覆盖多长。p95 从直方图取(桶上沿,不超过 max)。
|
|
8
|
+
import { noteRequestServed } from "./event-loop-lag.js";
|
|
9
|
+
|
|
8
10
|
export const ROUTE_BUCKET_MS = 60_000;
|
|
9
11
|
export const ROUTE_BUCKET_RETENTION = 1440;
|
|
10
12
|
/** 每个分钟桶里最多这么多个不同的路由;再多的记到 ROUTE_OVERFLOW_KEY,防止怪路径 / 怪工具名把内存撑大。 */
|
|
@@ -142,6 +144,8 @@ export function recordRouteTiming<T extends Response | undefined>(req: Request,
|
|
|
142
144
|
const length = Number(res.headers.get("content-length"));
|
|
143
145
|
const bytes = Number.isFinite(length) && res.headers.has("content-length") ? length : 0;
|
|
144
146
|
addSample(route, performance.now() - started, res.status, bytes, now, callerClass(req));
|
|
147
|
+
// #2614 —— 让停顿监测能区分「循环被阻塞」与「timer 被饿住」:见 event-loop-lag.ts 顶部说明。
|
|
148
|
+
noteRequestServed();
|
|
145
149
|
return res;
|
|
146
150
|
}
|
|
147
151
|
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
// #2622 —— 定时任务「连续失败 → 间隔翻倍 → 成功即恢复」。
|
|
2
|
+
//
|
|
3
|
+
// 为什么需要它(生产 24h,2026-10-27):
|
|
4
|
+
// 一个 `every_seconds=180` 的排程 351 次失败 / 24 次成功 —— 失败与成功交替,
|
|
5
|
+
// 所以「连续失败」永远攒不到 #523 的自动暂停阈值,而**每一次失败都按 3 分钟原速重试**。
|
|
6
|
+
// `#523` 的机制(连续 5 次通知、可选自动暂停)都在,缺的只是「放慢」这一档。
|
|
7
|
+
//
|
|
8
|
+
// 这个文件钉住四件事:
|
|
9
|
+
// 1. 连续失败达到阈值 ⇒ `backoff_effective_seconds` 按 2 倍递增,且有上限;
|
|
10
|
+
// 2. `next_run_at` 只往后推,不会被“放大后又缩短”的抖动来回拉;
|
|
11
|
+
// 3. 成功一次 ⇒ 立刻清空退避,`next_run_at` 拉回正常节奏;
|
|
12
|
+
// 4. 与 `no_change_backoff` 互斥(那套逻辑自己拥有这一列),关掉开关时完全不介入。
|
|
13
|
+
|
|
14
|
+
import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
|
|
15
|
+
import { register } from "./auth.js";
|
|
16
|
+
import { db } from "./db.js";
|
|
17
|
+
import { applyFailureBackoff, failureBackoffRuns } from "./scheduled-failures.js";
|
|
18
|
+
|
|
19
|
+
const stamp = Date.now();
|
|
20
|
+
let NET = "";
|
|
21
|
+
let creatorId = "";
|
|
22
|
+
const ENV_KEY = "COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS";
|
|
23
|
+
const savedEnv = process.env[ENV_KEY];
|
|
24
|
+
let slot = 0;
|
|
25
|
+
|
|
26
|
+
function seedSchedule(everySeconds = 180, patch: Record<string, unknown> = {}) {
|
|
27
|
+
const scheduleId = `sched_bo_${crypto.randomUUID()}`;
|
|
28
|
+
db.run(
|
|
29
|
+
`INSERT INTO scheduled_tasks
|
|
30
|
+
(schedule_id, network_id, created_by, name, target_node_id, target_alias, task_content, priority,
|
|
31
|
+
schedule_type, schedule_json, timezone, status, next_run_at, overlap_policy)
|
|
32
|
+
VALUES (?1, ?2, ?3, 'backoff fixture', 'n_bo', 'bo-node', 'x', 'normal', 'interval', ?4,
|
|
33
|
+
'Asia/Shanghai', 'active', ?5, 'skip')`,
|
|
34
|
+
[scheduleId, NET, creatorId, JSON.stringify({ type: "interval", every_seconds: everySeconds }),
|
|
35
|
+
new Date(Date.now() + everySeconds * 1000).toISOString()],
|
|
36
|
+
);
|
|
37
|
+
for (const [k, v] of Object.entries(patch)) db.run(`UPDATE scheduled_tasks SET ${k} = ?1 WHERE schedule_id = ?2`, [v as any, scheduleId]);
|
|
38
|
+
return scheduleId;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function addRun(scheduleId: string, status: string, minutesFromNow: number) {
|
|
42
|
+
const runId = `srun_bo_${crypto.randomUUID()}`;
|
|
43
|
+
db.run(
|
|
44
|
+
`INSERT INTO scheduled_task_runs (run_id, schedule_id, network_id, scheduled_for, status, created_at)
|
|
45
|
+
VALUES (?1, ?2, ?3, ?4, ?5, ?4)`,
|
|
46
|
+
[runId, scheduleId, NET, new Date(Date.now() + minutesFromNow * 60_000).toISOString(), status],
|
|
47
|
+
);
|
|
48
|
+
return runId;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const reload = (id: string) => db.get<any>("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", id)!;
|
|
52
|
+
|
|
53
|
+
beforeAll(async () => {
|
|
54
|
+
const auth = register(`bo_owner_${stamp}`, "Bo-Owner-Strong-1!", undefined, "tester");
|
|
55
|
+
expect(auth.ok).toBe(true);
|
|
56
|
+
NET = auth.network_id!;
|
|
57
|
+
creatorId = auth.user_id!;
|
|
58
|
+
process.env[ENV_KEY] = "6";
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
afterEach(() => {
|
|
62
|
+
process.env[ENV_KEY] = "6";
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
afterAll(() => {
|
|
66
|
+
try { db.run(`DELETE FROM scheduled_task_runs WHERE network_id = ?1`, [NET]); } catch {}
|
|
67
|
+
try { db.run(`DELETE FROM scheduled_tasks WHERE network_id = ?1`, [NET]); } catch {}
|
|
68
|
+
if (savedEnv === undefined) delete process.env[ENV_KEY]; else process.env[ENV_KEY] = savedEnv;
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
describe("#2622 failure backoff", () => {
|
|
72
|
+
test("默认阈值是 6,且可以关掉", () => {
|
|
73
|
+
delete process.env[ENV_KEY];
|
|
74
|
+
expect(failureBackoffRuns()).toBe(6);
|
|
75
|
+
process.env[ENV_KEY] = "0";
|
|
76
|
+
expect(failureBackoffRuns()).toBe(0);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
test("失败次数未到阈值:不写退避", () => {
|
|
80
|
+
const id = seedSchedule();
|
|
81
|
+
for (let i = 0; i < 5; i++) addRun(id, "failed", -60 + i);
|
|
82
|
+
const out = applyFailureBackoff(id);
|
|
83
|
+
expect(out).toEqual({ streak: 5, effective_seconds: null });
|
|
84
|
+
expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("达到阈值后间隔翻倍,且 next_run_at 只会往后推", () => {
|
|
88
|
+
const id = seedSchedule(180);
|
|
89
|
+
for (let i = 0; i < 6; i++) addRun(id, "failed", -60 + i);
|
|
90
|
+
const before = reload(id).next_run_at;
|
|
91
|
+
const out = applyFailureBackoff(id);
|
|
92
|
+
expect(out!.effective_seconds).toBe(360); // 180 × 2
|
|
93
|
+
const after = reload(id);
|
|
94
|
+
expect(after.backoff_effective_seconds).toBe(360);
|
|
95
|
+
expect(Date.parse(after.next_run_at)).toBeGreaterThanOrEqual(Date.parse(before));
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
test("失败继续累积:指数放大且有 1 小时上限", () => {
|
|
99
|
+
const id = seedSchedule(180);
|
|
100
|
+
for (let i = 0; i < 12; i++) addRun(id, "failed", -120 + i);
|
|
101
|
+
const out = applyFailureBackoff(id);
|
|
102
|
+
// 180 × 2^k 会被 3600 截断
|
|
103
|
+
expect(out!.effective_seconds).toBeLessThanOrEqual(3600);
|
|
104
|
+
expect(out!.streak).toBe(12);
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
test("成功一次 ⇒ 立刻清空退避并把 next_run_at 拉回正常节奏", () => {
|
|
108
|
+
const id = seedSchedule(180);
|
|
109
|
+
for (let i = 0; i < 8; i++) addRun(id, "failed", -120 + i);
|
|
110
|
+
applyFailureBackoff(id);
|
|
111
|
+
expect(reload(id).backoff_effective_seconds).toBeGreaterThan(180);
|
|
112
|
+
addRun(id, "replied", -1);
|
|
113
|
+
const out = applyFailureBackoff(id);
|
|
114
|
+
expect(out!.streak).toBe(0);
|
|
115
|
+
expect(out!.effective_seconds).toBeNull();
|
|
116
|
+
expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
test("no_change_backoff 的排程不被这条逻辑改写", () => {
|
|
120
|
+
const id = seedSchedule(180, { no_change_backoff: 1 });
|
|
121
|
+
for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
|
|
122
|
+
expect(applyFailureBackoff(id)).toBeNull();
|
|
123
|
+
expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
test("非 interval 排程不参与", () => {
|
|
127
|
+
const id = seedSchedule(180, { schedule_type: "daily", schedule_json: JSON.stringify({ type: "daily", time: "09:00" }) });
|
|
128
|
+
for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
|
|
129
|
+
expect(applyFailureBackoff(id)).toBeNull();
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
test("paused 的排程不参与(暂停已经把它停住了)", () => {
|
|
133
|
+
const id = seedSchedule(180, { status: "paused", next_run_at: null });
|
|
134
|
+
for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
|
|
135
|
+
expect(applyFailureBackoff(id)).toBeNull();
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
test("开关设为 0 时完全不介入", () => {
|
|
139
|
+
const id = seedSchedule(180);
|
|
140
|
+
for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
|
|
141
|
+
process.env[ENV_KEY] = "0";
|
|
142
|
+
expect(applyFailureBackoff(id)).toBeNull();
|
|
143
|
+
expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
|
|
144
|
+
});
|
|
145
|
+
});
|
|
@@ -167,6 +167,65 @@ export function evaluateScheduleFailures(scheduleId: string, now = new Date()):
|
|
|
167
167
|
return notice;
|
|
168
168
|
}
|
|
169
169
|
|
|
170
|
+
/**
|
|
171
|
+
* #2622 —— 失败退避:连续失败达到 N 次后把**间隔翻倍**(上限 1 小时),成功一次立刻恢复原节奏。
|
|
172
|
+
*
|
|
173
|
+
* 为什么不是「暂停」:`#523` 已经有 `COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS`(默认关),
|
|
174
|
+
* 但它会把排程**停下来**,需要人来恢复。对“上游额度暂时打满”这类外部原因,暂停太久同样是损失;
|
|
175
|
+
* 退避只是**放慢重试**,外部原因一恢复就会自己跑回原节奏。
|
|
176
|
+
*
|
|
177
|
+
* 现场(2026-10-27 生产 24h):一个 `every_seconds=180` 的排程失败 351 次、成功 24 次 ——
|
|
178
|
+
* 失败与成功交替出现,所以“连续失败”很少攒到暂停阈值,而**每次失败都在按 3 分钟原速重试**。
|
|
179
|
+
*
|
|
180
|
+
* 存储复用 `backoff_effective_seconds`(派发时按 `max(every_seconds, backoff_effective_seconds)` 生效),
|
|
181
|
+
* 与 `no_change_backoff` 互斥:那套逻辑自己会写这一列,这里只在 `no_change_backoff = 0` 时介入。
|
|
182
|
+
*
|
|
183
|
+
* 环境变量:`COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS`(默认 6;0 = 关闭)。
|
|
184
|
+
*/
|
|
185
|
+
export function failureBackoffRuns(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS", 6, 0); }
|
|
186
|
+
|
|
187
|
+
const FAILURE_BACKOFF_CAP_SECONDS = 3600;
|
|
188
|
+
|
|
189
|
+
export function applyFailureBackoff(scheduleId: string, now = new Date()): { streak: number; effective_seconds: number | null } | null {
|
|
190
|
+
const from = failureBackoffRuns();
|
|
191
|
+
if (from <= 0) return null;
|
|
192
|
+
const row = db.get<any>(
|
|
193
|
+
`SELECT schedule_id, revision, status, schedule_type, schedule_json, no_change_backoff,
|
|
194
|
+
backoff_effective_seconds, next_run_at
|
|
195
|
+
FROM scheduled_tasks WHERE schedule_id = ?1`,
|
|
196
|
+
scheduleId,
|
|
197
|
+
);
|
|
198
|
+
if (!row || row.status !== "active" || row.no_change_backoff === 1) return null;
|
|
199
|
+
if (row.schedule_type !== "interval") return null;
|
|
200
|
+
let spec: any = null;
|
|
201
|
+
try { spec = JSON.parse(row.schedule_json || "{}"); } catch { spec = null; }
|
|
202
|
+
const base = spec && spec.type === "interval" ? Number(spec.every_seconds) : NaN;
|
|
203
|
+
if (!Number.isFinite(base) || base <= 0) return null;
|
|
204
|
+
|
|
205
|
+
const streak = failureStreak(scheduleId).consecutiveFailures;
|
|
206
|
+
const effective = streak >= from
|
|
207
|
+
? Math.min(FAILURE_BACKOFF_CAP_SECONDS, base * 2 ** Math.min(streak - from + 1, 6))
|
|
208
|
+
: null;
|
|
209
|
+
const target = effective ?? base;
|
|
210
|
+
const previous = row.backoff_effective_seconds ?? base;
|
|
211
|
+
let next = row.next_run_at;
|
|
212
|
+
if (target !== previous && next) {
|
|
213
|
+
// 放大:至少推到 now + target(不把已经更远的 next 拉近);
|
|
214
|
+
// 恢复:至少拉回 now + target(不把已经更近的 next 推远)。
|
|
215
|
+
const candidate = new Date(now.getTime() + target * 1000).toISOString();
|
|
216
|
+
const existing = Date.parse(next);
|
|
217
|
+
if (Number.isFinite(existing)) {
|
|
218
|
+
next = new Date(target > previous ? Math.max(existing, Date.parse(candidate)) : Math.min(existing, Date.parse(candidate))).toISOString();
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
db.run(
|
|
222
|
+
`UPDATE scheduled_tasks SET backoff_streak = ?1, backoff_effective_seconds = ?2, next_run_at = ?3,
|
|
223
|
+
updated_at = datetime('now') WHERE schedule_id = ?4 AND revision = ?5 AND status = 'active'`,
|
|
224
|
+
[streak, effective, next, scheduleId, row.revision],
|
|
225
|
+
);
|
|
226
|
+
return { streak, effective_seconds: effective };
|
|
227
|
+
}
|
|
228
|
+
|
|
170
229
|
/** GET /runs 的附加字段。 */
|
|
171
230
|
export function failureSummary(scheduleId: string): { consecutive_failures: number; failure_alert_threshold: number; last_failure_alert_at: string | null } {
|
|
172
231
|
const streak = failureStreak(scheduleId);
|
|
@@ -176,4 +235,10 @@ export function failureSummary(scheduleId: string): { consecutive_failures: numb
|
|
|
176
235
|
|
|
177
236
|
setScheduledRunTerminalHook((scheduleId, status) => {
|
|
178
237
|
if (status === "failed") evaluateScheduleFailures(scheduleId);
|
|
238
|
+
// #2622 —— 成功一次就把间隔恢复原样;失败则按连续失败次数放大间隔。
|
|
239
|
+
try {
|
|
240
|
+
applyFailureBackoff(scheduleId);
|
|
241
|
+
} catch (e: any) {
|
|
242
|
+
console.error(`[scheduled-tasks] failure backoff failed schedule=${scheduleId}: ${e?.message || e}`);
|
|
243
|
+
}
|
|
179
244
|
});
|