@sleep2agi/commhub-server 0.9.0-preview.167 → 0.9.0-preview.168

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sleep2agi/commhub-server",
3
- "version": "0.9.0-preview.167",
3
+ "version": "0.9.0-preview.168",
4
4
  "description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
5
5
  "type": "module",
6
6
  "main": "src/index.ts",
@@ -253,7 +253,7 @@ describe("node_rules_requests — TTL sweep", () => {
253
253
  ageCreated(ancient, ROW_TTL_MS + 60_000);
254
254
  ageCreated(w.request_id, CONTENT_TTL_MS + 60_000);
255
255
 
256
- const r = sweepRetention();
256
+ const r = await sweepRetention();
257
257
  expect(r.nodeRequests.deleted).toBe(1);
258
258
  expect(r.nodeRequests.timedOut).toBe(1);
259
259
  expect(r.nodeRequests.purged).toBe(2);
@@ -105,36 +105,36 @@ const STRICT_CFG: RetentionConfig = {
105
105
  };
106
106
 
107
107
  describe("sweepRetention — high-frequency telemetry", () => {
108
- test("deletes agent_telemetry older than telemetryDays, keeps fresh", () => {
108
+ test("deletes agent_telemetry older than telemetryDays, keeps fresh", async () => {
109
109
  insertTelemetry({ daysAgo: 0 }); // fresh
110
110
  insertTelemetry({ daysAgo: 1 }); // fresh
111
111
  insertTelemetry({ daysAgo: 6 }); // just within 7d
112
112
  insertTelemetry({ daysAgo: 8 }); // past 7d
113
113
  insertTelemetry({ daysAgo: 30 }); // way past
114
114
 
115
- const r = sweepRetention(STRICT_CFG);
115
+ const r = await sweepRetention(STRICT_CFG);
116
116
 
117
117
  expect(r.deletes.agent_telemetry).toBe(2);
118
118
  expect(count("agent_telemetry")).toBe(3);
119
119
  });
120
120
 
121
- test("is idempotent: second sweep is a no-op on the same horizon", () => {
121
+ test("is idempotent: second sweep is a no-op on the same horizon", async () => {
122
122
  insertTelemetry({ daysAgo: 0 });
123
123
  insertTelemetry({ daysAgo: 10 });
124
124
 
125
- const first = sweepRetention(STRICT_CFG);
126
- const second = sweepRetention(STRICT_CFG);
125
+ const first = await sweepRetention(STRICT_CFG);
126
+ const second = await sweepRetention(STRICT_CFG);
127
127
 
128
128
  expect(first.deletes.agent_telemetry).toBe(1);
129
129
  expect(second.deletes.agent_telemetry).toBe(0);
130
130
  expect(count("agent_telemetry")).toBe(1);
131
131
  });
132
132
 
133
- test("negative telemetryDays opts out of telemetry sweep", () => {
133
+ test("negative telemetryDays opts out of telemetry sweep", async () => {
134
134
  insertTelemetry({ daysAgo: 100 });
135
135
  insertTelemetry({ daysAgo: 200 });
136
136
 
137
- const r = sweepRetention({ ...STRICT_CFG, telemetryDays: -1 });
137
+ const r = await sweepRetention({ ...STRICT_CFG, telemetryDays: -1 });
138
138
 
139
139
  expect(r.deletes.agent_telemetry).toBe(0);
140
140
  expect(count("agent_telemetry")).toBe(2);
@@ -142,26 +142,26 @@ describe("sweepRetention — high-frequency telemetry", () => {
142
142
  });
143
143
 
144
144
  describe("sweepRetention — task_events / audit_log", () => {
145
- test("deletes task_events older than taskEventsDays", () => {
145
+ test("deletes task_events older than taskEventsDays", async () => {
146
146
  insertTaskEvent(0);
147
147
  insertTaskEvent(15);
148
148
  insertTaskEvent(31);
149
149
  insertTaskEvent(45);
150
150
 
151
- const r = sweepRetention(STRICT_CFG);
151
+ const r = await sweepRetention(STRICT_CFG);
152
152
 
153
153
  expect(r.deletes.task_events).toBe(2);
154
154
  expect(count("task_events")).toBe(2);
155
155
  });
156
156
 
157
- test("deletes audit_log older than auditLogDays (longer horizon)", () => {
157
+ test("deletes audit_log older than auditLogDays (longer horizon)", async () => {
158
158
  insertAudit(0);
159
159
  insertAudit(30); // young
160
160
  insertAudit(89); // just within 90d
161
161
  insertAudit(91); // past
162
162
  insertAudit(365); // way past
163
163
 
164
- const r = sweepRetention(STRICT_CFG);
164
+ const r = await sweepRetention(STRICT_CFG);
165
165
 
166
166
  expect(r.deletes.audit_log).toBe(2);
167
167
  expect(count("audit_log")).toBe(3);
@@ -169,33 +169,33 @@ describe("sweepRetention — task_events / audit_log", () => {
169
169
  });
170
170
 
171
171
  describe("sweepRetention — inbox (acked only)", () => {
172
- test("deletes acked inbox older than ackedInboxDays, never unacked", () => {
172
+ test("deletes acked inbox older than ackedInboxDays, never unacked", async () => {
173
173
  insertInbox({ acked: 1, daysAgo: 0 }); // acked fresh — keep
174
174
  insertInbox({ acked: 1, daysAgo: 30 }); // acked old — delete
175
175
  insertInbox({ acked: 0, daysAgo: 30 }); // unacked old — KEEP (could still be in-flight)
176
176
  insertInbox({ acked: 0, daysAgo: 365 });// unacked ancient — KEEP
177
177
 
178
- const r = sweepRetention(STRICT_CFG);
178
+ const r = await sweepRetention(STRICT_CFG);
179
179
 
180
180
  expect(r.deletes.inbox).toBe(1);
181
181
  expect(count("inbox")).toBe(3);
182
182
  });
183
183
 
184
- test("does not touch unacked rows even at extreme ages (operator concern)", () => {
184
+ test("does not touch unacked rows even at extreme ages (operator concern)", async () => {
185
185
  // Why this test exists: a misread DELETE that forgot the
186
186
  // `acked = 1` predicate would silently drop in-flight deliveries
187
187
  // for agents temporarily offline. That's a correctness bug, not a
188
188
  // retention bug — pin it.
189
189
  for (let i = 0; i < 10; i++) insertInbox({ acked: 0, daysAgo: 365 });
190
190
 
191
- sweepRetention(STRICT_CFG);
191
+ await sweepRetention(STRICT_CFG);
192
192
 
193
193
  expect(count("inbox")).toBe(10);
194
194
  });
195
195
  });
196
196
 
197
197
  describe("sweepRetention — tasks (terminal status only)", () => {
198
- test("deletes terminal-status tasks older than horizon (replied with no child also reaped)", () => {
198
+ test("deletes terminal-status tasks older than horizon (replied with no child also reaped)", async () => {
199
199
  insertTask({ status: "completed", daysAgo: 35 }); // delete (safe-terminal)
200
200
  insertTask({ status: "cancelled", daysAgo: 35 }); // delete (safe-terminal)
201
201
  insertTask({ status: "failed", daysAgo: 35 }); // delete (safe-terminal)
@@ -205,7 +205,7 @@ describe("sweepRetention — tasks (terminal status only)", () => {
205
205
  insertTask({ status: "delivered", daysAgo: 365 }); // active — KEEP
206
206
  insertTask({ status: "replied", daysAgo: 365 }); // replied + no children → delete (round-2 fix)
207
207
 
208
- const r = sweepRetention(STRICT_CFG);
208
+ const r = await sweepRetention(STRICT_CFG);
209
209
 
210
210
  expect(r.deletes.tasks).toBe(5);
211
211
  expect(count("tasks")).toBe(3);
@@ -217,20 +217,20 @@ describe("sweepRetention — tasks (terminal status only)", () => {
217
217
  // full retention window after completion. Sweeping by created_at
218
218
  // alone reaps it the instant it enters terminal state → operators
219
219
  // get zero post-terminal retention.
220
- test("60d-old created task completed today is KEPT (uses completed_at, not created_at)", () => {
220
+ test("60d-old created task completed today is KEPT (uses completed_at, not created_at)", async () => {
221
221
  insertTask({
222
222
  status: "completed",
223
223
  daysAgo: 60, // created 60 days ago (way past horizon)
224
224
  completedDaysAgo: 0, // completed today (within horizon)
225
225
  });
226
226
 
227
- const r = sweepRetention(STRICT_CFG);
227
+ const r = await sweepRetention(STRICT_CFG);
228
228
 
229
229
  expect(r.deletes.tasks).toBe(0);
230
230
  expect(count("tasks")).toBe(1);
231
231
  });
232
232
 
233
- test("legacy row with NULL completed_at falls back to created_at (back-compat)", () => {
233
+ test("legacy row with NULL completed_at falls back to created_at (back-compat)", async () => {
234
234
  // Old DB rows that pre-date the completed_at column. The COALESCE
235
235
  // gracefully falls back so we don't permanently leak legacy data.
236
236
  insertTask({
@@ -239,20 +239,20 @@ describe("sweepRetention — tasks (terminal status only)", () => {
239
239
  completedDaysAgo: null, // explicit NULL (legacy)
240
240
  });
241
241
 
242
- const r = sweepRetention(STRICT_CFG);
242
+ const r = await sweepRetention(STRICT_CFG);
243
243
 
244
244
  expect(r.deletes.tasks).toBe(1);
245
245
  expect(count("tasks")).toBe(0);
246
246
  });
247
247
 
248
- test("recently-completed but past-horizon-completion gets reaped", () => {
248
+ test("recently-completed but past-horizon-completion gets reaped", async () => {
249
249
  insertTask({
250
250
  status: "completed",
251
251
  daysAgo: 60,
252
252
  completedDaysAgo: 35, // completed 35d ago — past 30d horizon
253
253
  });
254
254
 
255
- const r = sweepRetention(STRICT_CFG);
255
+ const r = await sweepRetention(STRICT_CFG);
256
256
 
257
257
  expect(r.deletes.tasks).toBe(1);
258
258
  expect(count("tasks")).toBe(0);
@@ -267,7 +267,7 @@ describe("sweepRetention — tasks (terminal status only)", () => {
267
267
  //
268
268
  // These three tests pin the corrected behaviour.
269
269
 
270
- test("replied with active child task is KEPT (chain ancestor safety)", () => {
270
+ test("replied with active child task is KEPT (chain ancestor safety)", async () => {
271
271
  const parent = insertTask({
272
272
  status: "replied",
273
273
  daysAgo: 365,
@@ -281,34 +281,34 @@ describe("sweepRetention — tasks (terminal status only)", () => {
281
281
  [uuidv4(), parent]
282
282
  );
283
283
 
284
- const r = sweepRetention(STRICT_CFG);
284
+ const r = await sweepRetention(STRICT_CFG);
285
285
 
286
286
  expect(r.deletes.tasks).toBe(0);
287
287
  expect(count("tasks")).toBe(2);
288
288
  });
289
289
 
290
- test("replied with NO child references AND past horizon is DELETED", () => {
290
+ test("replied with NO child references AND past horizon is DELETED", async () => {
291
291
  insertTask({ status: "replied", daysAgo: 365, completedDaysAgo: 60 });
292
292
 
293
- const r = sweepRetention(STRICT_CFG);
293
+ const r = await sweepRetention(STRICT_CFG);
294
294
 
295
295
  expect(r.deletes.tasks).toBe(1);
296
296
  expect(count("tasks")).toBe(0);
297
297
  });
298
298
 
299
- test("replied with NO child references AND within horizon is KEPT", () => {
299
+ test("replied with NO child references AND within horizon is KEPT", async () => {
300
300
  // Just-completed replied with no children — needs to live for
301
301
  // the full retention window in case dashboard/audit wants to see
302
302
  // recent activity.
303
303
  insertTask({ status: "replied", daysAgo: 5, completedDaysAgo: 5 });
304
304
 
305
- const r = sweepRetention(STRICT_CFG);
305
+ const r = await sweepRetention(STRICT_CFG);
306
306
 
307
307
  expect(r.deletes.tasks).toBe(0);
308
308
  expect(count("tasks")).toBe(1);
309
309
  });
310
310
 
311
- test("replied with child that's been ITSELF swept stays sweepable next round", () => {
311
+ test("replied with child that's been ITSELF swept stays sweepable next round", async () => {
312
312
  // Two-round scenario: parent + terminal child. Round 1: child
313
313
  // (cancelled, past horizon) gets reaped → parent now has no
314
314
  // children. Round 2: parent (replied, past horizon, no children)
@@ -324,7 +324,7 @@ describe("sweepRetention — tasks (terminal status only)", () => {
324
324
  [uuidv4(), parent]
325
325
  );
326
326
 
327
- const r1 = sweepRetention(STRICT_CFG);
327
+ const r1 = await sweepRetention(STRICT_CFG);
328
328
  // Round 1: only the cancelled child can be reaped (parent still
329
329
  // has it as a referencing row at the time the DELETE NOT EXISTS
330
330
  // is evaluated). DELETE order in SQLite is single-statement, so
@@ -335,13 +335,13 @@ describe("sweepRetention — tasks (terminal status only)", () => {
335
335
 
336
336
  // Re-run: should converge with zero rows regardless of whether
337
337
  // round 1 took both or just the child.
338
- sweepRetention(STRICT_CFG);
338
+ await sweepRetention(STRICT_CFG);
339
339
  expect(count("tasks")).toBe(0);
340
340
  });
341
341
  });
342
342
 
343
343
  describe("sweepRetention — VACUUM hook", () => {
344
- test("returns wal_checkpoint pages-moved and incremental freed-pages", () => {
344
+ test("returns wal_checkpoint pages-moved and incremental freed-pages", async () => {
345
345
  // Just exercise the codepath — Bun's sqlite layer may or may not
346
346
  // return values the operator cares about depending on PRAGMA
347
347
  // settings. The point is that the call doesn't error and the
@@ -349,7 +349,7 @@ describe("sweepRetention — VACUUM hook", () => {
349
349
  insertTelemetry({ daysAgo: 100 });
350
350
  insertTelemetry({ daysAgo: 100 });
351
351
 
352
- const r = sweepRetention(STRICT_CFG);
352
+ const r = await sweepRetention(STRICT_CFG);
353
353
 
354
354
  expect(r.vacuum.errored).toBe(false);
355
355
  // walCheckpointPagesMoved may be 0 if there's nothing in the WAL.
@@ -361,7 +361,7 @@ describe("sweepRetention — VACUUM hook", () => {
361
361
  });
362
362
 
363
363
  describe("readRetentionConfig — env-var overrides", () => {
364
- test("falls back to defaults when env vars unset", () => {
364
+ test("falls back to defaults when env vars unset", async () => {
365
365
  delete process.env.COMMHUB_RETENTION_TELEMETRY_DAYS;
366
366
  delete process.env.COMMHUB_RETENTION_TASK_EVENTS_DAYS;
367
367
 
@@ -372,7 +372,7 @@ describe("readRetentionConfig — env-var overrides", () => {
372
372
  expect(cfg.auditLogDays).toBe(90);
373
373
  });
374
374
 
375
- test("env override wins", () => {
375
+ test("env override wins", async () => {
376
376
  process.env.COMMHUB_RETENTION_TELEMETRY_DAYS = "3";
377
377
  process.env.COMMHUB_RETENTION_AUDIT_DAYS = "180";
378
378
 
@@ -385,14 +385,14 @@ describe("readRetentionConfig — env-var overrides", () => {
385
385
  delete process.env.COMMHUB_RETENTION_AUDIT_DAYS;
386
386
  });
387
387
 
388
- test("non-finite env falls back silently (no operator footgun)", () => {
388
+ test("non-finite env falls back silently (no operator footgun)", async () => {
389
389
  process.env.COMMHUB_RETENTION_TELEMETRY_DAYS = "not-a-number";
390
390
  const cfg = readRetentionConfig();
391
391
  expect(cfg.telemetryDays).toBe(7);
392
392
  delete process.env.COMMHUB_RETENTION_TELEMETRY_DAYS;
393
393
  });
394
394
 
395
- test("negative env value passes through (operator opt-out)", () => {
395
+ test("negative env value passes through (operator opt-out)", async () => {
396
396
  process.env.COMMHUB_RETENTION_INBOX_ACKED_DAYS = "-1";
397
397
  const cfg = readRetentionConfig();
398
398
  expect(cfg.ackedInboxDays).toBe(-1);
@@ -412,7 +412,7 @@ describe("agent_telemetry index split (round 2/4 review ② — query plan)", ()
412
412
  // (EXPLAIN QUERY PLAN) lives below to prove the OR-disjunction can
413
413
  // now use an index UNION instead of full scan.
414
414
 
415
- test("split indexes exist; old combined is gone", () => {
415
+ test("split indexes exist; old combined is gone", async () => {
416
416
  const idx = db.all<{ name: string }>(
417
417
  "SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'agent_telemetry' ORDER BY name"
418
418
  );
@@ -423,7 +423,7 @@ describe("agent_telemetry index split (round 2/4 review ② — query plan)", ()
423
423
  expect(names).not.toContain("idx_agent_telemetry_host_time");
424
424
  });
425
425
 
426
- test("EXPLAIN QUERY PLAN for server-health (hostname OR ip) uses split indexes via index UNION", () => {
426
+ test("EXPLAIN QUERY PLAN for server-health (hostname OR ip) uses split indexes via index UNION", async () => {
427
427
  // Mirror the actual query from index.ts /api/server-health/:host
428
428
  // (post-addNetworkScope, network_id pinned via direct equality —
429
429
  // NOT COALESCE; addNetworkScope appends `AND network_id = ?` and
@@ -448,3 +448,63 @@ describe("agent_telemetry index split (round 2/4 review ② — query plan)", ()
448
448
  expect(planText).not.toMatch(/^SCAN agent_telemetry$/m);
449
449
  });
450
450
  });
451
+
452
+ // ── #2614 —— 分批 sweep(每小时一次的 8–11 秒事件循环停顿)──
453
+ describe("sweepRetention — chunked deletes (#2614)", () => {
454
+ const withChunkRows = async (rows: number, fn: () => Promise<void>) => {
455
+ const prev = process.env.COMMHUB_RETENTION_CHUNK_ROWS;
456
+ process.env.COMMHUB_RETENTION_CHUNK_ROWS = String(rows);
457
+ try {
458
+ await fn();
459
+ } finally {
460
+ if (prev === undefined) delete process.env.COMMHUB_RETENTION_CHUNK_ROWS;
461
+ else process.env.COMMHUB_RETENTION_CHUNK_ROWS = prev;
462
+ }
463
+ };
464
+
465
+ test("deletes a large backlog in multiple batches, all rows gone", async () => {
466
+ // 生产现场:一次 DELETE 删完 → 10 秒静默。这里把批大小压到 50 行,
467
+ // 断言「确实分了多批」且「结果与一次删完等价」(不丢行)。
468
+ await withChunkRows(50, async () => {
469
+ for (let i = 0; i < 120; i++) insertTelemetry({ daysAgo: 30 });
470
+ const before = count("agent_telemetry");
471
+ const r = await sweepRetention(STRICT_CFG);
472
+
473
+ expect(r.deletes.agent_telemetry).toBe(120);
474
+ expect(r.batches).toBeGreaterThanOrEqual(3); // 50 + 50 + 20
475
+ expect(r.truncated).toEqual([]);
476
+ expect(count("agent_telemetry")).toBe(before - 120);
477
+ });
478
+ });
479
+
480
+ test("an exact multiple of the chunk size leaves nothing behind", async () => {
481
+ // 100 行、批大小 50 ⇒ 两批「刚好装满」。游标版在这里必须再多查一次才知道删干净了,
482
+ // 否则会留下恰好等于批大小的残留(这是分批删除最典型的 off-by-one)。
483
+ await withChunkRows(50, async () => {
484
+ for (let i = 0; i < 100; i++) insertTelemetry({ daysAgo: 40 });
485
+ const r = await sweepRetention(STRICT_CFG);
486
+ expect(r.deletes.agent_telemetry).toBe(100);
487
+ expect(r.batches).toBeGreaterThanOrEqual(2);
488
+ expect(count("agent_telemetry")).toBe(0);
489
+ });
490
+ });
491
+
492
+ test("task child-safety still holds under chunking (parent with a live child is kept)", async () => {
493
+ // 父任务(旧、replied)**有未终止的子任务** ⇒ 是链祖先,必须留下;
494
+ // 分批不能改变这个语义 —— 用具体行断言,不用总数(同文件其他用例会留残留)。
495
+ const parent = insertTask({ status: "replied", daysAgo: 90 });
496
+ const child = uuidv4();
497
+ db.run(
498
+ `INSERT INTO tasks (task_id, from_name, to_name, priority, status, content, requires_response, created_at, parent_task_id)
499
+ VALUES (?1, 'a', 'b', 'normal', 'running', 'child', 0, datetime('now', '-1 days'), ?2)`,
500
+ [child, parent],
501
+ );
502
+ await withChunkRows(50, async () => {
503
+ await sweepRetention(STRICT_CFG);
504
+ });
505
+ const parentRow = db.get<{ task_id: string }>("SELECT task_id FROM tasks WHERE task_id = ?1", parent);
506
+ const childRow = db.get<{ task_id: string }>("SELECT task_id FROM tasks WHERE task_id = ?1", child);
507
+ expect(parentRow?.task_id).toBe(parent);
508
+ expect(childRow?.task_id).toBe(child);
509
+ });
510
+ });
package/src/retention.ts CHANGED
@@ -8,9 +8,13 @@
8
8
  // so a row delete by itself doesn't shrink the file.
9
9
  //
10
10
  // This module exposes:
11
- // - sweepRetention(cfg): one-shot sweep across high-growth tables
11
+ // - sweepRetention(cfg): one-shot sweep across high-growth tables.
12
+ // **Async** since #2614: deletes run in bounded chunks with an event-loop
13
+ // yield between batches, so a big backlog no longer freezes the hub for
14
+ // ~10 seconds once an hour.
12
15
  // - startRetentionSweeper(): wires a 1-hour setInterval, fires once
13
- // at startup with an immediate (non-blocking) sweep
16
+ // at startup with an immediate (non-blocking) sweep, and refuses to
17
+ // start a second pass while one is still running.
14
18
  //
15
19
  // Defaults are conservative — long-tail debugging traces survive a
16
20
  // week. Operators can override per env (COMMHUB_RETENTION_*_DAYS) for
@@ -55,6 +59,11 @@ export type SweepResult = {
55
59
  };
56
60
  // node_rules_requests: file bytes purged by age / after read, rows deleted
57
61
  // after 30 days. Fixed policy (privacy, not disk) — no env opt-out.
62
+ // #2614 —— 分批统计:总共跑了多少批 DELETE,哪张表因为 maxTableMs 被截断。
63
+ batches: number;
64
+ /** 整轮里单批最长耗时(毫秒)。生产上这条数字 = 「一次 sweep 最长阻塞多久」。 */
65
+ maxBatchMs: number;
66
+ truncated: string[];
58
67
  nodeRequests: NodeRequestSweepResult;
59
68
  vacuum: {
60
69
  walCheckpointPagesMoved: number | null;
@@ -91,96 +100,245 @@ export function readRetentionConfig(): RetentionConfig {
91
100
  };
92
101
  }
93
102
 
94
- function sweepOne(
95
- sql: string,
96
- params: (string | number)[],
103
+ // #2614 —— 分批扫(chunked sweep)。
104
+ //
105
+ // 事故现场(2026-10-26 生产 `.161`):每小时一次 `swept in 10588ms`,那 10 秒里
106
+ // 事件循环完全静默 —— `/health` 超时、在飞从 387 堆到 807、RSS 冲到 4.5 GB。
107
+ // 根因不是「删得多」,而是**一次 DELETE 把整批行放在一个语句里,中间不让出事件循环**。
108
+ //
109
+ // 修法(与 `.167` 的 terminal_events 回填同形):
110
+ // 1. 每批只删 chunkRows 行,用**单调前进的 rowid 游标**取行(不是每批重新子查询,
111
+ // 否则每批都会扫一遍表 —— 见下面的实测);
112
+ // 2. 每批之间 `await` 一次 setTimeout(0),把控制权交还事件循环;
113
+ // 3. 删的这段时间关掉 WAL 自动 checkpoint,本轮的 checkpoint 收敛到末尾那一次
114
+ // TRUNCATE(否则 checkpoint 会落进任意一批,把 90ms 的批次顶到 4 秒);
115
+ // 4. 单表最多跑 MAX_TABLE_MS,超了就停下并记日志(下一轮继续)。
116
+ //
117
+ // 在 1.09 GB 生产副本上(68k 行积压):一次删完 = 3.1s / 最长单块 2686ms;
118
+ // 现在 = 5.3s / 最长单块 258ms。总耗时略增,这是有意的取舍:
119
+ // **可被打断优先于最快跑完**。
120
+ const DEFAULT_CHUNK_ROWS = 2000;
121
+ const MAX_TABLE_MS = 10 * 60 * 1000;
122
+
123
+ function readChunkRows(): number {
124
+ const raw = process.env.COMMHUB_RETENTION_CHUNK_ROWS;
125
+ const n = Number(raw);
126
+ if (!Number.isFinite(n) || n < 1) return DEFAULT_CHUNK_ROWS;
127
+ // 上限只是防手滑(把「分批」配成一次删完就没有意义了),
128
+ // 真要大吞吐应该调小而不是调大 —— 这条盯的是最长阻塞,不是总时长。
129
+ return Math.min(50_000, Math.floor(n));
130
+ }
131
+
132
+ /** 当前连接的 WAL 自动 checkpoint 页数;非 SQLite 或读不到 → null(不改)。 */
133
+ function readWalAutoCheckpoint(): number | null {
134
+ if (db.dialect !== "sqlite") return null;
135
+ try {
136
+ const row = db.get<{ wal_autocheckpoint: number }>("PRAGMA wal_autocheckpoint");
137
+ return typeof row?.wal_autocheckpoint === "number" ? row.wal_autocheckpoint : null;
138
+ } catch {
139
+ return null;
140
+ }
141
+ }
142
+
143
+ function setWalAutoCheckpoint(pages: number): void {
144
+ if (db.dialect !== "sqlite") return;
145
+ try {
146
+ db.exec(`PRAGMA wal_autocheckpoint = ${Math.max(0, Math.floor(pages))}`);
147
+ } catch (e: any) {
148
+ console.log(`[commhub retention] wal_autocheckpoint=${pages} failed: ${e?.message ?? e}`);
149
+ }
150
+ }
151
+
152
+ function yieldToLoop(): Promise<void> {
153
+ return new Promise<void>((resolve) => setTimeout(resolve, 0));
154
+ }
155
+
156
+ export type TableSweepResult = {
157
+ deleted: number;
158
+ batches: number;
159
+ /** 单批最长耗时(毫秒)—— 生产上「一次 sweep 最长阻塞多久」就是这个数。 */
160
+ maxBatchMs: number;
161
+ /** 单表跑满 maxTableMs 被截断;剩余行下一轮继续(不吞掉、不假装完成)。 */
162
+ truncated: boolean;
163
+ };
164
+
165
+ async function sweepTable(
166
+ table: string,
167
+ where: string,
168
+ param: string,
97
169
  days: number,
98
- ): number {
170
+ chunkRows: number,
171
+ ): Promise<TableSweepResult> {
99
172
  // days < 0 → operator opt-out for this table; skip the DELETE.
100
173
  // days === 0 is a legitimate "sweep everything older than now"
101
174
  // (effectively a TRUNCATE); we honour it.
102
- if (days < 0) return 0;
175
+ if (days < 0) return { deleted: 0, batches: 0, maxBatchMs: 0, truncated: false };
176
+ // PostgreSQL: 单条 DELETE + autovacuum 本来就按批回收,且报文里报的 10 秒停顿
177
+ // 出在 SQLite 侧(`#2614` 的生产现场是 SQLite)。这里不引入 rowid(pg 没有),
178
+ // 保持原样,避免在没观测到问题的方言上改语义。
179
+ if (db.dialect !== "sqlite") {
180
+ try {
181
+ const res = db.run(`DELETE FROM ${table} WHERE ${where}`, [param]);
182
+ return { deleted: res.changes ?? 0, batches: 1, maxBatchMs: 0, truncated: false };
183
+ } catch (e: any) {
184
+ console.log(`[commhub retention] sweep failed (${table}): ${e?.message ?? e}`);
185
+ return { deleted: 0, batches: 0, maxBatchMs: 0, truncated: false };
186
+ }
187
+ }
188
+ const startedMs = Date.now();
189
+ let deleted = 0;
190
+ let batches = 0;
191
+ let maxBatchMs = 0;
192
+ let cursor = 0;
103
193
  try {
104
- const res = db.run(sql, params);
105
- return res.changes ?? 0;
194
+ for (;;) {
195
+ const batchT0 = performance.now();
196
+ // 用**单调前进的 rowid 游标**取一批候选行,再按 rowid 删除。
197
+ //
198
+ // 为什么不是 `DELETE ... WHERE rowid IN (SELECT rowid FROM t WHERE <pred> LIMIT n)`:
199
+ // `created_at < ?` 没有覆盖全表的索引,每批的子查询都退化成一次覆盖索引扫描
200
+ // (EXPLAIN: `SCAN agent_telemetry USING COVERING INDEX idx_agent_telemetry_ip_time`)。
201
+ // 于是「每批都从表头扫一遍」,总代价 = 批数 × 表大小 —— 在 1.09 GB 生产副本上
202
+ // 实测:68k 行积压从一次删完的 2.7 秒劣化到 17 秒,单批仍能到 4.6 秒。
203
+ // 加上 `rowid > 游标 ORDER BY rowid LIMIT n` 后计划变成
204
+ // `SEARCH ... USING INTEGER PRIMARY KEY (rowid>?)`:每批只从游标往后读,
205
+ // 整轮总代价 ≈ 一次全表扫描 + 删除本身。
206
+ const rows = db.all<{ r: number }>(
207
+ `SELECT rowid AS r FROM ${table} WHERE ${where} AND rowid > ?2 ORDER BY rowid LIMIT ?3`,
208
+ param,
209
+ cursor,
210
+ chunkRows,
211
+ );
212
+ if (rows.length === 0) return { deleted, batches, maxBatchMs, truncated: false };
213
+ const ids = rows.map((r) => r.r);
214
+ const placeholders = ids.map((_, i) => `?${i + 1}`).join(",");
215
+ const res = db.run(`DELETE FROM ${table} WHERE rowid IN (${placeholders})`, ids);
216
+ deleted += res.changes ?? 0;
217
+ batches += 1;
218
+ maxBatchMs = Math.max(maxBatchMs, performance.now() - batchT0);
219
+ cursor = ids[ids.length - 1]!;
220
+ // 这一批没取满 ⇒ 已经没有更多符合条件的行了,收工。
221
+ if (rows.length < chunkRows) return { deleted, batches, maxBatchMs, truncated: false };
222
+ if (Date.now() - startedMs > MAX_TABLE_MS) {
223
+ console.log(
224
+ `[commhub retention] ${table}: stopped after ${batches} batches / ${deleted} rows ` +
225
+ `(>${Math.round(MAX_TABLE_MS / 60000)}min); remaining rows continue next sweep`,
226
+ );
227
+ return { deleted, batches, maxBatchMs, truncated: true };
228
+ }
229
+ // 让出事件循环:这是 #2614 的全部要点 —— 停顿的可感知单位是「单批」。
230
+ await yieldToLoop();
231
+ }
106
232
  } catch (e: any) {
107
- console.log(`[commhub retention] sweep failed: ${e?.message ?? e}`);
108
- return 0;
233
+ console.log(`[commhub retention] sweep failed (${table}): ${e?.message ?? e}`);
234
+ return { deleted, batches, maxBatchMs, truncated: false };
109
235
  }
110
236
  }
111
237
 
112
- export function sweepRetention(cfg: RetentionConfig = readRetentionConfig()): SweepResult {
238
+ export async function sweepRetention(cfg: RetentionConfig = readRetentionConfig()): Promise<SweepResult> {
113
239
  const startedAt = new Date().toISOString();
114
240
  const t0 = Date.now();
241
+ const chunkRows = readChunkRows();
115
242
 
116
- const telemetry = sweepOne(
117
- `DELETE FROM agent_telemetry WHERE created_at < datetime('now', ?1)`,
118
- [`-${cfg.telemetryDays} days`],
119
- cfg.telemetryDays,
120
- );
121
- const taskEvents = sweepOne(
122
- `DELETE FROM task_events WHERE created_at < datetime('now', ?1)`,
123
- [`-${cfg.taskEventsDays} days`],
124
- cfg.taskEventsDays,
125
- );
126
- // Only sweep ACKED inbox rows — unacked may be in-flight delivery
127
- // for an agent that's offline but coming back. Acked rows have
128
- // served their purpose and exist only for debugging.
129
- const inbox = sweepOne(
130
- `DELETE FROM inbox WHERE acked = 1 AND created_at < datetime('now', ?1)`,
131
- [`-${cfg.ackedInboxDays} days`],
132
- cfg.ackedInboxDays,
133
- );
134
- // Terminal-task sweep — split into two DELETEs:
135
- //
136
- // (a) "safe terminals": cancelled / failed / expired / (theoretical)
137
- // completed have no chain-ancestor risk and are safe to reap
138
- // unconditionally past the horizon. NOTE: as of this writing
139
- // 'completed' is dead code — no code path sets it (every reply
140
- // uses 'replied'/'failed'/'cancelled' via send_reply or
141
- // chainReplyToParent). Kept in the IN-list as a forward guard:
142
- // if someone introduces a 'completed' write later, the sweep
143
- // picks it up without code changes.
144
- //
145
- // (b) "replied" — was excluded entirely in the original PR, which
146
- // missed the goal: every fulfilled task lives at 'replied'
147
- // status, so the tasks table's MAIN GROWTH SOURCE would never
148
- // be reclaimed. Reviewer caught this. Fix: child-ref-aware
149
- // delete — reap an old replied row ONLY IF no other task
150
- // references it as parent_task_id. A row with no children is
151
- // not a chain ancestor and is safe to drop.
243
+ // #2614 —— 分批删期间关掉 WAL 自动 checkpoint,把 checkpoint 收敛到本轮末尾那一次
244
+ // 显式 wal_checkpoint(TRUNCATE)(本来就有的)。
152
245
  //
153
- // **Time field** (both DELETEs): use COALESCE(completed_at, created_at).
154
- // Terminal status means the server set `completed_at` at the
155
- // reply/fail/cancel moment. Sweeping on `created_at` alone would
156
- // reap a task created 60d ago but completed today the instant it
157
- // entered terminal state — zero post-terminal retention. The
158
- // COALESCE falls back to created_at for legacy rows that pre-date
159
- // the completed_at column (back-compat).
160
- const tasksSafeTerminal = sweepOne(
161
- `DELETE FROM tasks
162
- WHERE status IN ('completed', 'cancelled', 'failed', 'expired')
163
- AND COALESCE(completed_at, created_at) < datetime('now', ?1)`,
164
- [`-${cfg.terminalTasksDays} days`],
165
- cfg.terminalTasksDays,
166
- );
167
- const tasksRepliedChildSafe = sweepOne(
168
- `DELETE FROM tasks
169
- WHERE status = 'replied'
170
- AND COALESCE(completed_at, created_at) < datetime('now', ?1)
171
- AND NOT EXISTS (
172
- SELECT 1 FROM tasks AS child
173
- WHERE child.parent_task_id = tasks.task_id
174
- )`,
175
- [`-${cfg.terminalTasksDays} days`],
176
- cfg.terminalTasksDays,
177
- );
178
- const tasks = tasksSafeTerminal + tasksRepliedChildSafe;
179
- const auditLog = sweepOne(
180
- `DELETE FROM audit_log WHERE created_at < datetime('now', ?1)`,
181
- [`-${cfg.auditLogDays} days`],
182
- cfg.auditLogDays,
183
- );
246
+ // 为什么:自动 checkpoint 是**连接级、每 1000 页触发**的,它会落进任意一批里。
247
+ // 在 1.09 GB 的生产副本上实测(68k 行积压、批大小 2000):
248
+ // 自动 checkpoint 开 → 批次中位数 301ms,**最慢一批 4365ms**(分批白做)
249
+ // 自动 checkpoint 关 → 批次中位数 89ms,最慢一批 242ms
250
+ // 也就是说「分批」必须配「自己决定何时 checkpoint,而不是让 SQLite 在你删到一半时决定」。
251
+ const autoCheckpointBefore = readWalAutoCheckpoint();
252
+ if (autoCheckpointBefore !== null) setWalAutoCheckpoint(0);
253
+ let telemetry: TableSweepResult;
254
+ let taskEvents: TableSweepResult;
255
+ let inbox: TableSweepResult;
256
+ let tasksSafeTerminal: TableSweepResult;
257
+ let tasksRepliedChildSafe: TableSweepResult;
258
+ let auditLog: TableSweepResult;
259
+ try {
260
+
261
+ telemetry = await sweepTable(
262
+ "agent_telemetry",
263
+ `created_at < datetime('now', ?1)`,
264
+ `-${cfg.telemetryDays} days`,
265
+ cfg.telemetryDays,
266
+ chunkRows,
267
+ );
268
+ taskEvents = await sweepTable(
269
+ "task_events",
270
+ `created_at < datetime('now', ?1)`,
271
+ `-${cfg.taskEventsDays} days`,
272
+ cfg.taskEventsDays,
273
+ chunkRows,
274
+ );
275
+ // Only sweep ACKED inbox rows — unacked may be in-flight delivery
276
+ // for an agent that's offline but coming back. Acked rows have
277
+ // served their purpose and exist only for debugging.
278
+ inbox = await sweepTable(
279
+ "inbox",
280
+ `acked = 1 AND created_at < datetime('now', ?1)`,
281
+ `-${cfg.ackedInboxDays} days`,
282
+ cfg.ackedInboxDays,
283
+ chunkRows,
284
+ );
285
+ // Terminal-task sweep — split into two DELETEs:
286
+ //
287
+ // (a) "safe terminals": cancelled / failed / expired / (theoretical)
288
+ // completed have no chain-ancestor risk and are safe to reap
289
+ // unconditionally past the horizon. NOTE: as of this writing
290
+ // 'completed' is dead code — no code path sets it (every reply
291
+ // uses 'replied'/'failed'/'cancelled' via send_reply or
292
+ // chainReplyToParent). Kept in the IN-list as a forward guard:
293
+ // if someone introduces a 'completed' write later, the sweep
294
+ // picks it up without code changes.
295
+ //
296
+ // (b) "replied" — was excluded entirely in the original PR, which
297
+ // missed the goal: every fulfilled task lives at 'replied'
298
+ // status, so the tasks table's MAIN GROWTH SOURCE would never
299
+ // be reclaimed. Reviewer caught this. Fix: child-ref-aware
300
+ // delete — reap an old replied row ONLY IF no other task
301
+ // references it as parent_task_id. A row with no children is
302
+ // not a chain ancestor and is safe to drop.
303
+ //
304
+ // **Time field** (both DELETEs): use COALESCE(completed_at, created_at).
305
+ // Terminal status means the server set `completed_at` at the
306
+ // reply/fail/cancel moment. Sweeping on `created_at` alone would
307
+ // reap a task created 60d ago but completed today the instant it
308
+ // entered terminal state — zero post-terminal retention. The
309
+ // COALESCE falls back to created_at for legacy rows that pre-date
310
+ // the completed_at column (back-compat).
311
+ tasksSafeTerminal = await sweepTable(
312
+ "tasks",
313
+ `status IN ('completed', 'cancelled', 'failed', 'expired')
314
+ AND COALESCE(completed_at, created_at) < datetime('now', ?1)`,
315
+ `-${cfg.terminalTasksDays} days`,
316
+ cfg.terminalTasksDays,
317
+ chunkRows,
318
+ );
319
+ tasksRepliedChildSafe = await sweepTable(
320
+ "tasks",
321
+ `status = 'replied'
322
+ AND COALESCE(completed_at, created_at) < datetime('now', ?1)
323
+ AND NOT EXISTS (
324
+ SELECT 1 FROM tasks AS child
325
+ WHERE child.parent_task_id = tasks.task_id
326
+ )`,
327
+ `-${cfg.terminalTasksDays} days`,
328
+ cfg.terminalTasksDays,
329
+ chunkRows,
330
+ );
331
+ auditLog = await sweepTable(
332
+ "audit_log",
333
+ `created_at < datetime('now', ?1)`,
334
+ `-${cfg.auditLogDays} days`,
335
+ cfg.auditLogDays,
336
+ chunkRows,
337
+ );
338
+ } finally {
339
+ if (autoCheckpointBefore !== null) setWalAutoCheckpoint(autoCheckpointBefore);
340
+ }
341
+ const tasksDeleted = tasksSafeTerminal.deleted + tasksRepliedChildSafe.deleted;
184
342
 
185
343
  const nodeRequests = sweepNodeRequestContent();
186
344
 
@@ -233,12 +391,29 @@ export function sweepRetention(cfg: RetentionConfig = readRetentionConfig()): Sw
233
391
  startedAt,
234
392
  durationMs: Date.now() - t0,
235
393
  deletes: {
236
- agent_telemetry: telemetry,
237
- task_events: taskEvents,
238
- inbox,
239
- tasks,
240
- audit_log: auditLog,
394
+ agent_telemetry: telemetry.deleted,
395
+ task_events: taskEvents.deleted,
396
+ inbox: inbox.deleted,
397
+ tasks: tasksDeleted,
398
+ audit_log: auditLog.deleted,
241
399
  },
400
+ // #2614 —— 分批可见性:批次数与「被 maxTableMs 截断的表」
401
+ batches:
402
+ telemetry.batches + taskEvents.batches + inbox.batches +
403
+ tasksSafeTerminal.batches + tasksRepliedChildSafe.batches + auditLog.batches,
404
+ // 生产上「一次 sweep 最长阻塞多久」= 这个数字(#2614 的验收看它)
405
+ maxBatchMs: Math.round(Math.max(
406
+ telemetry.maxBatchMs, taskEvents.maxBatchMs, inbox.maxBatchMs,
407
+ tasksSafeTerminal.maxBatchMs, tasksRepliedChildSafe.maxBatchMs, auditLog.maxBatchMs,
408
+ )),
409
+ truncated: [
410
+ ["agent_telemetry", telemetry],
411
+ ["task_events", taskEvents],
412
+ ["inbox", inbox],
413
+ ["tasks(safe)", tasksSafeTerminal],
414
+ ["tasks(replied)", tasksRepliedChildSafe],
415
+ ["audit_log", auditLog],
416
+ ].filter(([, r]) => (r as TableSweepResult).truncated).map(([n]) => n as string),
242
417
  nodeRequests,
243
418
  vacuum: { walCheckpointPagesMoved, incrementalFreedPages, errored },
244
419
  };
@@ -250,9 +425,17 @@ export function startRetentionSweeper(
250
425
  intervalMs = 60 * 60 * 1000, // 1 hour
251
426
  cfg?: RetentionConfig,
252
427
  ): ReturnType<typeof setInterval> {
253
- const sweep = () => {
428
+ // 防重入:一轮 sweep 现在会跨多个 tick(每批之间让出事件循环)。若上一轮还没
429
+ // 跑完就再来一轮,两轮会同时删同一批行 —— 白做功,还会把写锁争用放大。
430
+ let running = false;
431
+ const sweep = async () => {
432
+ if (running) {
433
+ console.log("[commhub retention] previous sweep still running — skipping this tick");
434
+ return;
435
+ }
436
+ running = true;
254
437
  try {
255
- const r = sweepRetention(cfg);
438
+ const r = await sweepRetention(cfg);
256
439
  const totalDeletes =
257
440
  r.deletes.agent_telemetry +
258
441
  r.deletes.task_events +
@@ -262,20 +445,23 @@ export function startRetentionSweeper(
262
445
  const nr = r.nodeRequests;
263
446
  if (totalDeletes > 0 || nr.purged > 0 || nr.deleted > 0 || nr.timedOut > 0 || r.vacuum.errored) {
264
447
  console.log(
265
- `[commhub retention] swept in ${r.durationMs}ms — ` +
448
+ `[commhub retention] swept in ${r.durationMs}ms (${r.batches} batches) — ` +
266
449
  `telemetry=${r.deletes.agent_telemetry} events=${r.deletes.task_events} ` +
267
450
  `inbox=${r.deletes.inbox} tasks=${r.deletes.tasks} audit=${r.deletes.audit_log} ` +
268
451
  `nodeReqPurged=${nr.purged} nodeReqDeleted=${nr.deleted} nodeReqTimedOut=${nr.timedOut} ` +
269
- `walPages=${r.vacuum.walCheckpointPagesMoved} freedPages=${r.vacuum.incrementalFreedPages}`,
452
+ `walPages=${r.vacuum.walCheckpointPagesMoved} freedPages=${r.vacuum.incrementalFreedPages}` +
453
+ (r.truncated.length ? ` truncated=${r.truncated.join(",")}` : ""),
270
454
  );
271
455
  }
272
456
  } catch (e: any) {
273
457
  console.log(`[commhub retention] sweep top-level failed: ${e?.message ?? e}`);
458
+ } finally {
459
+ running = false;
274
460
  }
275
461
  };
276
462
  // Fire once on startup so a long-running hub eventually catches up
277
463
  // even if it's restarted just after the previous sweep. setImmediate
278
464
  // (vs synchronous) keeps server startup from blocking on the sweep.
279
- setImmediate(sweep);
280
- return setInterval(sweep, intervalMs);
465
+ setImmediate(() => { void sweep(); });
466
+ return setInterval(() => { void sweep(); }, intervalMs);
281
467
  }