@sleep2agi/commhub-server 0.9.0-preview.167 → 0.9.0-preview.168
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/node-request-retention.test.ts +1 -1
- package/src/retention.test.ts +100 -40
- package/src/retention.ts +276 -90
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sleep2agi/commhub-server",
|
|
3
|
-
"version": "0.9.0-preview.
|
|
3
|
+
"version": "0.9.0-preview.168",
|
|
4
4
|
"description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -253,7 +253,7 @@ describe("node_rules_requests — TTL sweep", () => {
|
|
|
253
253
|
ageCreated(ancient, ROW_TTL_MS + 60_000);
|
|
254
254
|
ageCreated(w.request_id, CONTENT_TTL_MS + 60_000);
|
|
255
255
|
|
|
256
|
-
const r = sweepRetention();
|
|
256
|
+
const r = await sweepRetention();
|
|
257
257
|
expect(r.nodeRequests.deleted).toBe(1);
|
|
258
258
|
expect(r.nodeRequests.timedOut).toBe(1);
|
|
259
259
|
expect(r.nodeRequests.purged).toBe(2);
|
package/src/retention.test.ts
CHANGED
|
@@ -105,36 +105,36 @@ const STRICT_CFG: RetentionConfig = {
|
|
|
105
105
|
};
|
|
106
106
|
|
|
107
107
|
describe("sweepRetention — high-frequency telemetry", () => {
|
|
108
|
-
test("deletes agent_telemetry older than telemetryDays, keeps fresh", () => {
|
|
108
|
+
test("deletes agent_telemetry older than telemetryDays, keeps fresh", async () => {
|
|
109
109
|
insertTelemetry({ daysAgo: 0 }); // fresh
|
|
110
110
|
insertTelemetry({ daysAgo: 1 }); // fresh
|
|
111
111
|
insertTelemetry({ daysAgo: 6 }); // just within 7d
|
|
112
112
|
insertTelemetry({ daysAgo: 8 }); // past 7d
|
|
113
113
|
insertTelemetry({ daysAgo: 30 }); // way past
|
|
114
114
|
|
|
115
|
-
const r = sweepRetention(STRICT_CFG);
|
|
115
|
+
const r = await sweepRetention(STRICT_CFG);
|
|
116
116
|
|
|
117
117
|
expect(r.deletes.agent_telemetry).toBe(2);
|
|
118
118
|
expect(count("agent_telemetry")).toBe(3);
|
|
119
119
|
});
|
|
120
120
|
|
|
121
|
-
test("is idempotent: second sweep is a no-op on the same horizon", () => {
|
|
121
|
+
test("is idempotent: second sweep is a no-op on the same horizon", async () => {
|
|
122
122
|
insertTelemetry({ daysAgo: 0 });
|
|
123
123
|
insertTelemetry({ daysAgo: 10 });
|
|
124
124
|
|
|
125
|
-
const first = sweepRetention(STRICT_CFG);
|
|
126
|
-
const second = sweepRetention(STRICT_CFG);
|
|
125
|
+
const first = await sweepRetention(STRICT_CFG);
|
|
126
|
+
const second = await sweepRetention(STRICT_CFG);
|
|
127
127
|
|
|
128
128
|
expect(first.deletes.agent_telemetry).toBe(1);
|
|
129
129
|
expect(second.deletes.agent_telemetry).toBe(0);
|
|
130
130
|
expect(count("agent_telemetry")).toBe(1);
|
|
131
131
|
});
|
|
132
132
|
|
|
133
|
-
test("negative telemetryDays opts out of telemetry sweep", () => {
|
|
133
|
+
test("negative telemetryDays opts out of telemetry sweep", async () => {
|
|
134
134
|
insertTelemetry({ daysAgo: 100 });
|
|
135
135
|
insertTelemetry({ daysAgo: 200 });
|
|
136
136
|
|
|
137
|
-
const r = sweepRetention({ ...STRICT_CFG, telemetryDays: -1 });
|
|
137
|
+
const r = await sweepRetention({ ...STRICT_CFG, telemetryDays: -1 });
|
|
138
138
|
|
|
139
139
|
expect(r.deletes.agent_telemetry).toBe(0);
|
|
140
140
|
expect(count("agent_telemetry")).toBe(2);
|
|
@@ -142,26 +142,26 @@ describe("sweepRetention — high-frequency telemetry", () => {
|
|
|
142
142
|
});
|
|
143
143
|
|
|
144
144
|
describe("sweepRetention — task_events / audit_log", () => {
|
|
145
|
-
test("deletes task_events older than taskEventsDays", () => {
|
|
145
|
+
test("deletes task_events older than taskEventsDays", async () => {
|
|
146
146
|
insertTaskEvent(0);
|
|
147
147
|
insertTaskEvent(15);
|
|
148
148
|
insertTaskEvent(31);
|
|
149
149
|
insertTaskEvent(45);
|
|
150
150
|
|
|
151
|
-
const r = sweepRetention(STRICT_CFG);
|
|
151
|
+
const r = await sweepRetention(STRICT_CFG);
|
|
152
152
|
|
|
153
153
|
expect(r.deletes.task_events).toBe(2);
|
|
154
154
|
expect(count("task_events")).toBe(2);
|
|
155
155
|
});
|
|
156
156
|
|
|
157
|
-
test("deletes audit_log older than auditLogDays (longer horizon)", () => {
|
|
157
|
+
test("deletes audit_log older than auditLogDays (longer horizon)", async () => {
|
|
158
158
|
insertAudit(0);
|
|
159
159
|
insertAudit(30); // young
|
|
160
160
|
insertAudit(89); // just within 90d
|
|
161
161
|
insertAudit(91); // past
|
|
162
162
|
insertAudit(365); // way past
|
|
163
163
|
|
|
164
|
-
const r = sweepRetention(STRICT_CFG);
|
|
164
|
+
const r = await sweepRetention(STRICT_CFG);
|
|
165
165
|
|
|
166
166
|
expect(r.deletes.audit_log).toBe(2);
|
|
167
167
|
expect(count("audit_log")).toBe(3);
|
|
@@ -169,33 +169,33 @@ describe("sweepRetention — task_events / audit_log", () => {
|
|
|
169
169
|
});
|
|
170
170
|
|
|
171
171
|
describe("sweepRetention — inbox (acked only)", () => {
|
|
172
|
-
test("deletes acked inbox older than ackedInboxDays, never unacked", () => {
|
|
172
|
+
test("deletes acked inbox older than ackedInboxDays, never unacked", async () => {
|
|
173
173
|
insertInbox({ acked: 1, daysAgo: 0 }); // acked fresh — keep
|
|
174
174
|
insertInbox({ acked: 1, daysAgo: 30 }); // acked old — delete
|
|
175
175
|
insertInbox({ acked: 0, daysAgo: 30 }); // unacked old — KEEP (could still be in-flight)
|
|
176
176
|
insertInbox({ acked: 0, daysAgo: 365 });// unacked ancient — KEEP
|
|
177
177
|
|
|
178
|
-
const r = sweepRetention(STRICT_CFG);
|
|
178
|
+
const r = await sweepRetention(STRICT_CFG);
|
|
179
179
|
|
|
180
180
|
expect(r.deletes.inbox).toBe(1);
|
|
181
181
|
expect(count("inbox")).toBe(3);
|
|
182
182
|
});
|
|
183
183
|
|
|
184
|
-
test("does not touch unacked rows even at extreme ages (operator concern)", () => {
|
|
184
|
+
test("does not touch unacked rows even at extreme ages (operator concern)", async () => {
|
|
185
185
|
// Why this test exists: a misread DELETE that forgot the
|
|
186
186
|
// `acked = 1` predicate would silently drop in-flight deliveries
|
|
187
187
|
// for agents temporarily offline. That's a correctness bug, not a
|
|
188
188
|
// retention bug — pin it.
|
|
189
189
|
for (let i = 0; i < 10; i++) insertInbox({ acked: 0, daysAgo: 365 });
|
|
190
190
|
|
|
191
|
-
sweepRetention(STRICT_CFG);
|
|
191
|
+
await sweepRetention(STRICT_CFG);
|
|
192
192
|
|
|
193
193
|
expect(count("inbox")).toBe(10);
|
|
194
194
|
});
|
|
195
195
|
});
|
|
196
196
|
|
|
197
197
|
describe("sweepRetention — tasks (terminal status only)", () => {
|
|
198
|
-
test("deletes terminal-status tasks older than horizon (replied with no child also reaped)", () => {
|
|
198
|
+
test("deletes terminal-status tasks older than horizon (replied with no child also reaped)", async () => {
|
|
199
199
|
insertTask({ status: "completed", daysAgo: 35 }); // delete (safe-terminal)
|
|
200
200
|
insertTask({ status: "cancelled", daysAgo: 35 }); // delete (safe-terminal)
|
|
201
201
|
insertTask({ status: "failed", daysAgo: 35 }); // delete (safe-terminal)
|
|
@@ -205,7 +205,7 @@ describe("sweepRetention — tasks (terminal status only)", () => {
|
|
|
205
205
|
insertTask({ status: "delivered", daysAgo: 365 }); // active — KEEP
|
|
206
206
|
insertTask({ status: "replied", daysAgo: 365 }); // replied + no children → delete (round-2 fix)
|
|
207
207
|
|
|
208
|
-
const r = sweepRetention(STRICT_CFG);
|
|
208
|
+
const r = await sweepRetention(STRICT_CFG);
|
|
209
209
|
|
|
210
210
|
expect(r.deletes.tasks).toBe(5);
|
|
211
211
|
expect(count("tasks")).toBe(3);
|
|
@@ -217,20 +217,20 @@ describe("sweepRetention — tasks (terminal status only)", () => {
|
|
|
217
217
|
// full retention window after completion. Sweeping by created_at
|
|
218
218
|
// alone reaps it the instant it enters terminal state → operators
|
|
219
219
|
// get zero post-terminal retention.
|
|
220
|
-
test("60d-old created task completed today is KEPT (uses completed_at, not created_at)", () => {
|
|
220
|
+
test("60d-old created task completed today is KEPT (uses completed_at, not created_at)", async () => {
|
|
221
221
|
insertTask({
|
|
222
222
|
status: "completed",
|
|
223
223
|
daysAgo: 60, // created 60 days ago (way past horizon)
|
|
224
224
|
completedDaysAgo: 0, // completed today (within horizon)
|
|
225
225
|
});
|
|
226
226
|
|
|
227
|
-
const r = sweepRetention(STRICT_CFG);
|
|
227
|
+
const r = await sweepRetention(STRICT_CFG);
|
|
228
228
|
|
|
229
229
|
expect(r.deletes.tasks).toBe(0);
|
|
230
230
|
expect(count("tasks")).toBe(1);
|
|
231
231
|
});
|
|
232
232
|
|
|
233
|
-
test("legacy row with NULL completed_at falls back to created_at (back-compat)", () => {
|
|
233
|
+
test("legacy row with NULL completed_at falls back to created_at (back-compat)", async () => {
|
|
234
234
|
// Old DB rows that pre-date the completed_at column. The COALESCE
|
|
235
235
|
// gracefully falls back so we don't permanently leak legacy data.
|
|
236
236
|
insertTask({
|
|
@@ -239,20 +239,20 @@ describe("sweepRetention — tasks (terminal status only)", () => {
|
|
|
239
239
|
completedDaysAgo: null, // explicit NULL (legacy)
|
|
240
240
|
});
|
|
241
241
|
|
|
242
|
-
const r = sweepRetention(STRICT_CFG);
|
|
242
|
+
const r = await sweepRetention(STRICT_CFG);
|
|
243
243
|
|
|
244
244
|
expect(r.deletes.tasks).toBe(1);
|
|
245
245
|
expect(count("tasks")).toBe(0);
|
|
246
246
|
});
|
|
247
247
|
|
|
248
|
-
test("recently-completed but past-horizon-completion gets reaped", () => {
|
|
248
|
+
test("recently-completed but past-horizon-completion gets reaped", async () => {
|
|
249
249
|
insertTask({
|
|
250
250
|
status: "completed",
|
|
251
251
|
daysAgo: 60,
|
|
252
252
|
completedDaysAgo: 35, // completed 35d ago — past 30d horizon
|
|
253
253
|
});
|
|
254
254
|
|
|
255
|
-
const r = sweepRetention(STRICT_CFG);
|
|
255
|
+
const r = await sweepRetention(STRICT_CFG);
|
|
256
256
|
|
|
257
257
|
expect(r.deletes.tasks).toBe(1);
|
|
258
258
|
expect(count("tasks")).toBe(0);
|
|
@@ -267,7 +267,7 @@ describe("sweepRetention — tasks (terminal status only)", () => {
|
|
|
267
267
|
//
|
|
268
268
|
// These three tests pin the corrected behaviour.
|
|
269
269
|
|
|
270
|
-
test("replied with active child task is KEPT (chain ancestor safety)", () => {
|
|
270
|
+
test("replied with active child task is KEPT (chain ancestor safety)", async () => {
|
|
271
271
|
const parent = insertTask({
|
|
272
272
|
status: "replied",
|
|
273
273
|
daysAgo: 365,
|
|
@@ -281,34 +281,34 @@ describe("sweepRetention — tasks (terminal status only)", () => {
|
|
|
281
281
|
[uuidv4(), parent]
|
|
282
282
|
);
|
|
283
283
|
|
|
284
|
-
const r = sweepRetention(STRICT_CFG);
|
|
284
|
+
const r = await sweepRetention(STRICT_CFG);
|
|
285
285
|
|
|
286
286
|
expect(r.deletes.tasks).toBe(0);
|
|
287
287
|
expect(count("tasks")).toBe(2);
|
|
288
288
|
});
|
|
289
289
|
|
|
290
|
-
test("replied with NO child references AND past horizon is DELETED", () => {
|
|
290
|
+
test("replied with NO child references AND past horizon is DELETED", async () => {
|
|
291
291
|
insertTask({ status: "replied", daysAgo: 365, completedDaysAgo: 60 });
|
|
292
292
|
|
|
293
|
-
const r = sweepRetention(STRICT_CFG);
|
|
293
|
+
const r = await sweepRetention(STRICT_CFG);
|
|
294
294
|
|
|
295
295
|
expect(r.deletes.tasks).toBe(1);
|
|
296
296
|
expect(count("tasks")).toBe(0);
|
|
297
297
|
});
|
|
298
298
|
|
|
299
|
-
test("replied with NO child references AND within horizon is KEPT", () => {
|
|
299
|
+
test("replied with NO child references AND within horizon is KEPT", async () => {
|
|
300
300
|
// Just-completed replied with no children — needs to live for
|
|
301
301
|
// the full retention window in case dashboard/audit wants to see
|
|
302
302
|
// recent activity.
|
|
303
303
|
insertTask({ status: "replied", daysAgo: 5, completedDaysAgo: 5 });
|
|
304
304
|
|
|
305
|
-
const r = sweepRetention(STRICT_CFG);
|
|
305
|
+
const r = await sweepRetention(STRICT_CFG);
|
|
306
306
|
|
|
307
307
|
expect(r.deletes.tasks).toBe(0);
|
|
308
308
|
expect(count("tasks")).toBe(1);
|
|
309
309
|
});
|
|
310
310
|
|
|
311
|
-
test("replied with child that's been ITSELF swept stays sweepable next round", () => {
|
|
311
|
+
test("replied with child that's been ITSELF swept stays sweepable next round", async () => {
|
|
312
312
|
// Two-round scenario: parent + terminal child. Round 1: child
|
|
313
313
|
// (cancelled, past horizon) gets reaped → parent now has no
|
|
314
314
|
// children. Round 2: parent (replied, past horizon, no children)
|
|
@@ -324,7 +324,7 @@ describe("sweepRetention — tasks (terminal status only)", () => {
|
|
|
324
324
|
[uuidv4(), parent]
|
|
325
325
|
);
|
|
326
326
|
|
|
327
|
-
const r1 = sweepRetention(STRICT_CFG);
|
|
327
|
+
const r1 = await sweepRetention(STRICT_CFG);
|
|
328
328
|
// Round 1: only the cancelled child can be reaped (parent still
|
|
329
329
|
// has it as a referencing row at the time the DELETE NOT EXISTS
|
|
330
330
|
// is evaluated). DELETE order in SQLite is single-statement, so
|
|
@@ -335,13 +335,13 @@ describe("sweepRetention — tasks (terminal status only)", () => {
|
|
|
335
335
|
|
|
336
336
|
// Re-run: should converge with zero rows regardless of whether
|
|
337
337
|
// round 1 took both or just the child.
|
|
338
|
-
sweepRetention(STRICT_CFG);
|
|
338
|
+
await sweepRetention(STRICT_CFG);
|
|
339
339
|
expect(count("tasks")).toBe(0);
|
|
340
340
|
});
|
|
341
341
|
});
|
|
342
342
|
|
|
343
343
|
describe("sweepRetention — VACUUM hook", () => {
|
|
344
|
-
test("returns wal_checkpoint pages-moved and incremental freed-pages", () => {
|
|
344
|
+
test("returns wal_checkpoint pages-moved and incremental freed-pages", async () => {
|
|
345
345
|
// Just exercise the codepath — Bun's sqlite layer may or may not
|
|
346
346
|
// return values the operator cares about depending on PRAGMA
|
|
347
347
|
// settings. The point is that the call doesn't error and the
|
|
@@ -349,7 +349,7 @@ describe("sweepRetention — VACUUM hook", () => {
|
|
|
349
349
|
insertTelemetry({ daysAgo: 100 });
|
|
350
350
|
insertTelemetry({ daysAgo: 100 });
|
|
351
351
|
|
|
352
|
-
const r = sweepRetention(STRICT_CFG);
|
|
352
|
+
const r = await sweepRetention(STRICT_CFG);
|
|
353
353
|
|
|
354
354
|
expect(r.vacuum.errored).toBe(false);
|
|
355
355
|
// walCheckpointPagesMoved may be 0 if there's nothing in the WAL.
|
|
@@ -361,7 +361,7 @@ describe("sweepRetention — VACUUM hook", () => {
|
|
|
361
361
|
});
|
|
362
362
|
|
|
363
363
|
describe("readRetentionConfig — env-var overrides", () => {
|
|
364
|
-
test("falls back to defaults when env vars unset", () => {
|
|
364
|
+
test("falls back to defaults when env vars unset", async () => {
|
|
365
365
|
delete process.env.COMMHUB_RETENTION_TELEMETRY_DAYS;
|
|
366
366
|
delete process.env.COMMHUB_RETENTION_TASK_EVENTS_DAYS;
|
|
367
367
|
|
|
@@ -372,7 +372,7 @@ describe("readRetentionConfig — env-var overrides", () => {
|
|
|
372
372
|
expect(cfg.auditLogDays).toBe(90);
|
|
373
373
|
});
|
|
374
374
|
|
|
375
|
-
test("env override wins", () => {
|
|
375
|
+
test("env override wins", async () => {
|
|
376
376
|
process.env.COMMHUB_RETENTION_TELEMETRY_DAYS = "3";
|
|
377
377
|
process.env.COMMHUB_RETENTION_AUDIT_DAYS = "180";
|
|
378
378
|
|
|
@@ -385,14 +385,14 @@ describe("readRetentionConfig — env-var overrides", () => {
|
|
|
385
385
|
delete process.env.COMMHUB_RETENTION_AUDIT_DAYS;
|
|
386
386
|
});
|
|
387
387
|
|
|
388
|
-
test("non-finite env falls back silently (no operator footgun)", () => {
|
|
388
|
+
test("non-finite env falls back silently (no operator footgun)", async () => {
|
|
389
389
|
process.env.COMMHUB_RETENTION_TELEMETRY_DAYS = "not-a-number";
|
|
390
390
|
const cfg = readRetentionConfig();
|
|
391
391
|
expect(cfg.telemetryDays).toBe(7);
|
|
392
392
|
delete process.env.COMMHUB_RETENTION_TELEMETRY_DAYS;
|
|
393
393
|
});
|
|
394
394
|
|
|
395
|
-
test("negative env value passes through (operator opt-out)", () => {
|
|
395
|
+
test("negative env value passes through (operator opt-out)", async () => {
|
|
396
396
|
process.env.COMMHUB_RETENTION_INBOX_ACKED_DAYS = "-1";
|
|
397
397
|
const cfg = readRetentionConfig();
|
|
398
398
|
expect(cfg.ackedInboxDays).toBe(-1);
|
|
@@ -412,7 +412,7 @@ describe("agent_telemetry index split (round 2/4 review ② — query plan)", ()
|
|
|
412
412
|
// (EXPLAIN QUERY PLAN) lives below to prove the OR-disjunction can
|
|
413
413
|
// now use an index UNION instead of full scan.
|
|
414
414
|
|
|
415
|
-
test("split indexes exist; old combined is gone", () => {
|
|
415
|
+
test("split indexes exist; old combined is gone", async () => {
|
|
416
416
|
const idx = db.all<{ name: string }>(
|
|
417
417
|
"SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'agent_telemetry' ORDER BY name"
|
|
418
418
|
);
|
|
@@ -423,7 +423,7 @@ describe("agent_telemetry index split (round 2/4 review ② — query plan)", ()
|
|
|
423
423
|
expect(names).not.toContain("idx_agent_telemetry_host_time");
|
|
424
424
|
});
|
|
425
425
|
|
|
426
|
-
test("EXPLAIN QUERY PLAN for server-health (hostname OR ip) uses split indexes via index UNION", () => {
|
|
426
|
+
test("EXPLAIN QUERY PLAN for server-health (hostname OR ip) uses split indexes via index UNION", async () => {
|
|
427
427
|
// Mirror the actual query from index.ts /api/server-health/:host
|
|
428
428
|
// (post-addNetworkScope, network_id pinned via direct equality —
|
|
429
429
|
// NOT COALESCE; addNetworkScope appends `AND network_id = ?` and
|
|
@@ -448,3 +448,63 @@ describe("agent_telemetry index split (round 2/4 review ② — query plan)", ()
|
|
|
448
448
|
expect(planText).not.toMatch(/^SCAN agent_telemetry$/m);
|
|
449
449
|
});
|
|
450
450
|
});
|
|
451
|
+
|
|
452
|
+
// ── #2614 —— 分批 sweep(每小时一次的 8–11 秒事件循环停顿)──
|
|
453
|
+
describe("sweepRetention — chunked deletes (#2614)", () => {
|
|
454
|
+
const withChunkRows = async (rows: number, fn: () => Promise<void>) => {
|
|
455
|
+
const prev = process.env.COMMHUB_RETENTION_CHUNK_ROWS;
|
|
456
|
+
process.env.COMMHUB_RETENTION_CHUNK_ROWS = String(rows);
|
|
457
|
+
try {
|
|
458
|
+
await fn();
|
|
459
|
+
} finally {
|
|
460
|
+
if (prev === undefined) delete process.env.COMMHUB_RETENTION_CHUNK_ROWS;
|
|
461
|
+
else process.env.COMMHUB_RETENTION_CHUNK_ROWS = prev;
|
|
462
|
+
}
|
|
463
|
+
};
|
|
464
|
+
|
|
465
|
+
test("deletes a large backlog in multiple batches, all rows gone", async () => {
|
|
466
|
+
// 生产现场:一次 DELETE 删完 → 10 秒静默。这里把批大小压到 50 行,
|
|
467
|
+
// 断言「确实分了多批」且「结果与一次删完等价」(不丢行)。
|
|
468
|
+
await withChunkRows(50, async () => {
|
|
469
|
+
for (let i = 0; i < 120; i++) insertTelemetry({ daysAgo: 30 });
|
|
470
|
+
const before = count("agent_telemetry");
|
|
471
|
+
const r = await sweepRetention(STRICT_CFG);
|
|
472
|
+
|
|
473
|
+
expect(r.deletes.agent_telemetry).toBe(120);
|
|
474
|
+
expect(r.batches).toBeGreaterThanOrEqual(3); // 50 + 50 + 20
|
|
475
|
+
expect(r.truncated).toEqual([]);
|
|
476
|
+
expect(count("agent_telemetry")).toBe(before - 120);
|
|
477
|
+
});
|
|
478
|
+
});
|
|
479
|
+
|
|
480
|
+
test("an exact multiple of the chunk size leaves nothing behind", async () => {
|
|
481
|
+
// 100 行、批大小 50 ⇒ 两批「刚好装满」。游标版在这里必须再多查一次才知道删干净了,
|
|
482
|
+
// 否则会留下恰好等于批大小的残留(这是分批删除最典型的 off-by-one)。
|
|
483
|
+
await withChunkRows(50, async () => {
|
|
484
|
+
for (let i = 0; i < 100; i++) insertTelemetry({ daysAgo: 40 });
|
|
485
|
+
const r = await sweepRetention(STRICT_CFG);
|
|
486
|
+
expect(r.deletes.agent_telemetry).toBe(100);
|
|
487
|
+
expect(r.batches).toBeGreaterThanOrEqual(2);
|
|
488
|
+
expect(count("agent_telemetry")).toBe(0);
|
|
489
|
+
});
|
|
490
|
+
});
|
|
491
|
+
|
|
492
|
+
test("task child-safety still holds under chunking (parent with a live child is kept)", async () => {
|
|
493
|
+
// 父任务(旧、replied)**有未终止的子任务** ⇒ 是链祖先,必须留下;
|
|
494
|
+
// 分批不能改变这个语义 —— 用具体行断言,不用总数(同文件其他用例会留残留)。
|
|
495
|
+
const parent = insertTask({ status: "replied", daysAgo: 90 });
|
|
496
|
+
const child = uuidv4();
|
|
497
|
+
db.run(
|
|
498
|
+
`INSERT INTO tasks (task_id, from_name, to_name, priority, status, content, requires_response, created_at, parent_task_id)
|
|
499
|
+
VALUES (?1, 'a', 'b', 'normal', 'running', 'child', 0, datetime('now', '-1 days'), ?2)`,
|
|
500
|
+
[child, parent],
|
|
501
|
+
);
|
|
502
|
+
await withChunkRows(50, async () => {
|
|
503
|
+
await sweepRetention(STRICT_CFG);
|
|
504
|
+
});
|
|
505
|
+
const parentRow = db.get<{ task_id: string }>("SELECT task_id FROM tasks WHERE task_id = ?1", parent);
|
|
506
|
+
const childRow = db.get<{ task_id: string }>("SELECT task_id FROM tasks WHERE task_id = ?1", child);
|
|
507
|
+
expect(parentRow?.task_id).toBe(parent);
|
|
508
|
+
expect(childRow?.task_id).toBe(child);
|
|
509
|
+
});
|
|
510
|
+
});
|
package/src/retention.ts
CHANGED
|
@@ -8,9 +8,13 @@
|
|
|
8
8
|
// so a row delete by itself doesn't shrink the file.
|
|
9
9
|
//
|
|
10
10
|
// This module exposes:
|
|
11
|
-
// - sweepRetention(cfg): one-shot sweep across high-growth tables
|
|
11
|
+
// - sweepRetention(cfg): one-shot sweep across high-growth tables.
|
|
12
|
+
// **Async** since #2614: deletes run in bounded chunks with an event-loop
|
|
13
|
+
// yield between batches, so a big backlog no longer freezes the hub for
|
|
14
|
+
// ~10 seconds once an hour.
|
|
12
15
|
// - startRetentionSweeper(): wires a 1-hour setInterval, fires once
|
|
13
|
-
// at startup with an immediate (non-blocking) sweep
|
|
16
|
+
// at startup with an immediate (non-blocking) sweep, and refuses to
|
|
17
|
+
// start a second pass while one is still running.
|
|
14
18
|
//
|
|
15
19
|
// Defaults are conservative — long-tail debugging traces survive a
|
|
16
20
|
// week. Operators can override per env (COMMHUB_RETENTION_*_DAYS) for
|
|
@@ -55,6 +59,11 @@ export type SweepResult = {
|
|
|
55
59
|
};
|
|
56
60
|
// node_rules_requests: file bytes purged by age / after read, rows deleted
|
|
57
61
|
// after 30 days. Fixed policy (privacy, not disk) — no env opt-out.
|
|
62
|
+
// #2614 —— 分批统计:总共跑了多少批 DELETE,哪张表因为 maxTableMs 被截断。
|
|
63
|
+
batches: number;
|
|
64
|
+
/** 整轮里单批最长耗时(毫秒)。生产上这条数字 = 「一次 sweep 最长阻塞多久」。 */
|
|
65
|
+
maxBatchMs: number;
|
|
66
|
+
truncated: string[];
|
|
58
67
|
nodeRequests: NodeRequestSweepResult;
|
|
59
68
|
vacuum: {
|
|
60
69
|
walCheckpointPagesMoved: number | null;
|
|
@@ -91,96 +100,245 @@ export function readRetentionConfig(): RetentionConfig {
|
|
|
91
100
|
};
|
|
92
101
|
}
|
|
93
102
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
103
|
+
// #2614 —— 分批扫(chunked sweep)。
|
|
104
|
+
//
|
|
105
|
+
// 事故现场(2026-10-26 生产 `.161`):每小时一次 `swept in 10588ms`,那 10 秒里
|
|
106
|
+
// 事件循环完全静默 —— `/health` 超时、在飞从 387 堆到 807、RSS 冲到 4.5 GB。
|
|
107
|
+
// 根因不是「删得多」,而是**一次 DELETE 把整批行放在一个语句里,中间不让出事件循环**。
|
|
108
|
+
//
|
|
109
|
+
// 修法(与 `.167` 的 terminal_events 回填同形):
|
|
110
|
+
// 1. 每批只删 chunkRows 行,用**单调前进的 rowid 游标**取行(不是每批重新子查询,
|
|
111
|
+
// 否则每批都会扫一遍表 —— 见下面的实测);
|
|
112
|
+
// 2. 每批之间 `await` 一次 setTimeout(0),把控制权交还事件循环;
|
|
113
|
+
// 3. 删的这段时间关掉 WAL 自动 checkpoint,本轮的 checkpoint 收敛到末尾那一次
|
|
114
|
+
// TRUNCATE(否则 checkpoint 会落进任意一批,把 90ms 的批次顶到 4 秒);
|
|
115
|
+
// 4. 单表最多跑 MAX_TABLE_MS,超了就停下并记日志(下一轮继续)。
|
|
116
|
+
//
|
|
117
|
+
// 在 1.09 GB 生产副本上(68k 行积压):一次删完 = 3.1s / 最长单块 2686ms;
|
|
118
|
+
// 现在 = 5.3s / 最长单块 258ms。总耗时略增,这是有意的取舍:
|
|
119
|
+
// **可被打断优先于最快跑完**。
|
|
120
|
+
const DEFAULT_CHUNK_ROWS = 2000;
|
|
121
|
+
const MAX_TABLE_MS = 10 * 60 * 1000;
|
|
122
|
+
|
|
123
|
+
function readChunkRows(): number {
|
|
124
|
+
const raw = process.env.COMMHUB_RETENTION_CHUNK_ROWS;
|
|
125
|
+
const n = Number(raw);
|
|
126
|
+
if (!Number.isFinite(n) || n < 1) return DEFAULT_CHUNK_ROWS;
|
|
127
|
+
// 上限只是防手滑(把「分批」配成一次删完就没有意义了),
|
|
128
|
+
// 真要大吞吐应该调小而不是调大 —— 这条盯的是最长阻塞,不是总时长。
|
|
129
|
+
return Math.min(50_000, Math.floor(n));
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** 当前连接的 WAL 自动 checkpoint 页数;非 SQLite 或读不到 → null(不改)。 */
|
|
133
|
+
function readWalAutoCheckpoint(): number | null {
|
|
134
|
+
if (db.dialect !== "sqlite") return null;
|
|
135
|
+
try {
|
|
136
|
+
const row = db.get<{ wal_autocheckpoint: number }>("PRAGMA wal_autocheckpoint");
|
|
137
|
+
return typeof row?.wal_autocheckpoint === "number" ? row.wal_autocheckpoint : null;
|
|
138
|
+
} catch {
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function setWalAutoCheckpoint(pages: number): void {
|
|
144
|
+
if (db.dialect !== "sqlite") return;
|
|
145
|
+
try {
|
|
146
|
+
db.exec(`PRAGMA wal_autocheckpoint = ${Math.max(0, Math.floor(pages))}`);
|
|
147
|
+
} catch (e: any) {
|
|
148
|
+
console.log(`[commhub retention] wal_autocheckpoint=${pages} failed: ${e?.message ?? e}`);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function yieldToLoop(): Promise<void> {
|
|
153
|
+
return new Promise<void>((resolve) => setTimeout(resolve, 0));
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export type TableSweepResult = {
|
|
157
|
+
deleted: number;
|
|
158
|
+
batches: number;
|
|
159
|
+
/** 单批最长耗时(毫秒)—— 生产上「一次 sweep 最长阻塞多久」就是这个数。 */
|
|
160
|
+
maxBatchMs: number;
|
|
161
|
+
/** 单表跑满 maxTableMs 被截断;剩余行下一轮继续(不吞掉、不假装完成)。 */
|
|
162
|
+
truncated: boolean;
|
|
163
|
+
};
|
|
164
|
+
|
|
165
|
+
async function sweepTable(
|
|
166
|
+
table: string,
|
|
167
|
+
where: string,
|
|
168
|
+
param: string,
|
|
97
169
|
days: number,
|
|
98
|
-
|
|
170
|
+
chunkRows: number,
|
|
171
|
+
): Promise<TableSweepResult> {
|
|
99
172
|
// days < 0 → operator opt-out for this table; skip the DELETE.
|
|
100
173
|
// days === 0 is a legitimate "sweep everything older than now"
|
|
101
174
|
// (effectively a TRUNCATE); we honour it.
|
|
102
|
-
if (days < 0) return 0;
|
|
175
|
+
if (days < 0) return { deleted: 0, batches: 0, maxBatchMs: 0, truncated: false };
|
|
176
|
+
// PostgreSQL: 单条 DELETE + autovacuum 本来就按批回收,且报文里报的 10 秒停顿
|
|
177
|
+
// 出在 SQLite 侧(`#2614` 的生产现场是 SQLite)。这里不引入 rowid(pg 没有),
|
|
178
|
+
// 保持原样,避免在没观测到问题的方言上改语义。
|
|
179
|
+
if (db.dialect !== "sqlite") {
|
|
180
|
+
try {
|
|
181
|
+
const res = db.run(`DELETE FROM ${table} WHERE ${where}`, [param]);
|
|
182
|
+
return { deleted: res.changes ?? 0, batches: 1, maxBatchMs: 0, truncated: false };
|
|
183
|
+
} catch (e: any) {
|
|
184
|
+
console.log(`[commhub retention] sweep failed (${table}): ${e?.message ?? e}`);
|
|
185
|
+
return { deleted: 0, batches: 0, maxBatchMs: 0, truncated: false };
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
const startedMs = Date.now();
|
|
189
|
+
let deleted = 0;
|
|
190
|
+
let batches = 0;
|
|
191
|
+
let maxBatchMs = 0;
|
|
192
|
+
let cursor = 0;
|
|
103
193
|
try {
|
|
104
|
-
|
|
105
|
-
|
|
194
|
+
for (;;) {
|
|
195
|
+
const batchT0 = performance.now();
|
|
196
|
+
// 用**单调前进的 rowid 游标**取一批候选行,再按 rowid 删除。
|
|
197
|
+
//
|
|
198
|
+
// 为什么不是 `DELETE ... WHERE rowid IN (SELECT rowid FROM t WHERE <pred> LIMIT n)`:
|
|
199
|
+
// `created_at < ?` 没有覆盖全表的索引,每批的子查询都退化成一次覆盖索引扫描
|
|
200
|
+
// (EXPLAIN: `SCAN agent_telemetry USING COVERING INDEX idx_agent_telemetry_ip_time`)。
|
|
201
|
+
// 于是「每批都从表头扫一遍」,总代价 = 批数 × 表大小 —— 在 1.09 GB 生产副本上
|
|
202
|
+
// 实测:68k 行积压从一次删完的 2.7 秒劣化到 17 秒,单批仍能到 4.6 秒。
|
|
203
|
+
// 加上 `rowid > 游标 ORDER BY rowid LIMIT n` 后计划变成
|
|
204
|
+
// `SEARCH ... USING INTEGER PRIMARY KEY (rowid>?)`:每批只从游标往后读,
|
|
205
|
+
// 整轮总代价 ≈ 一次全表扫描 + 删除本身。
|
|
206
|
+
const rows = db.all<{ r: number }>(
|
|
207
|
+
`SELECT rowid AS r FROM ${table} WHERE ${where} AND rowid > ?2 ORDER BY rowid LIMIT ?3`,
|
|
208
|
+
param,
|
|
209
|
+
cursor,
|
|
210
|
+
chunkRows,
|
|
211
|
+
);
|
|
212
|
+
if (rows.length === 0) return { deleted, batches, maxBatchMs, truncated: false };
|
|
213
|
+
const ids = rows.map((r) => r.r);
|
|
214
|
+
const placeholders = ids.map((_, i) => `?${i + 1}`).join(",");
|
|
215
|
+
const res = db.run(`DELETE FROM ${table} WHERE rowid IN (${placeholders})`, ids);
|
|
216
|
+
deleted += res.changes ?? 0;
|
|
217
|
+
batches += 1;
|
|
218
|
+
maxBatchMs = Math.max(maxBatchMs, performance.now() - batchT0);
|
|
219
|
+
cursor = ids[ids.length - 1]!;
|
|
220
|
+
// 这一批没取满 ⇒ 已经没有更多符合条件的行了,收工。
|
|
221
|
+
if (rows.length < chunkRows) return { deleted, batches, maxBatchMs, truncated: false };
|
|
222
|
+
if (Date.now() - startedMs > MAX_TABLE_MS) {
|
|
223
|
+
console.log(
|
|
224
|
+
`[commhub retention] ${table}: stopped after ${batches} batches / ${deleted} rows ` +
|
|
225
|
+
`(>${Math.round(MAX_TABLE_MS / 60000)}min); remaining rows continue next sweep`,
|
|
226
|
+
);
|
|
227
|
+
return { deleted, batches, maxBatchMs, truncated: true };
|
|
228
|
+
}
|
|
229
|
+
// 让出事件循环:这是 #2614 的全部要点 —— 停顿的可感知单位是「单批」。
|
|
230
|
+
await yieldToLoop();
|
|
231
|
+
}
|
|
106
232
|
} catch (e: any) {
|
|
107
|
-
console.log(`[commhub retention] sweep failed: ${e?.message ?? e}`);
|
|
108
|
-
return
|
|
233
|
+
console.log(`[commhub retention] sweep failed (${table}): ${e?.message ?? e}`);
|
|
234
|
+
return { deleted, batches, maxBatchMs, truncated: false };
|
|
109
235
|
}
|
|
110
236
|
}
|
|
111
237
|
|
|
112
|
-
export function sweepRetention(cfg: RetentionConfig = readRetentionConfig()): SweepResult {
|
|
238
|
+
export async function sweepRetention(cfg: RetentionConfig = readRetentionConfig()): Promise<SweepResult> {
|
|
113
239
|
const startedAt = new Date().toISOString();
|
|
114
240
|
const t0 = Date.now();
|
|
241
|
+
const chunkRows = readChunkRows();
|
|
115
242
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
[`-${cfg.telemetryDays} days`],
|
|
119
|
-
cfg.telemetryDays,
|
|
120
|
-
);
|
|
121
|
-
const taskEvents = sweepOne(
|
|
122
|
-
`DELETE FROM task_events WHERE created_at < datetime('now', ?1)`,
|
|
123
|
-
[`-${cfg.taskEventsDays} days`],
|
|
124
|
-
cfg.taskEventsDays,
|
|
125
|
-
);
|
|
126
|
-
// Only sweep ACKED inbox rows — unacked may be in-flight delivery
|
|
127
|
-
// for an agent that's offline but coming back. Acked rows have
|
|
128
|
-
// served their purpose and exist only for debugging.
|
|
129
|
-
const inbox = sweepOne(
|
|
130
|
-
`DELETE FROM inbox WHERE acked = 1 AND created_at < datetime('now', ?1)`,
|
|
131
|
-
[`-${cfg.ackedInboxDays} days`],
|
|
132
|
-
cfg.ackedInboxDays,
|
|
133
|
-
);
|
|
134
|
-
// Terminal-task sweep — split into two DELETEs:
|
|
135
|
-
//
|
|
136
|
-
// (a) "safe terminals": cancelled / failed / expired / (theoretical)
|
|
137
|
-
// completed have no chain-ancestor risk and are safe to reap
|
|
138
|
-
// unconditionally past the horizon. NOTE: as of this writing
|
|
139
|
-
// 'completed' is dead code — no code path sets it (every reply
|
|
140
|
-
// uses 'replied'/'failed'/'cancelled' via send_reply or
|
|
141
|
-
// chainReplyToParent). Kept in the IN-list as a forward guard:
|
|
142
|
-
// if someone introduces a 'completed' write later, the sweep
|
|
143
|
-
// picks it up without code changes.
|
|
144
|
-
//
|
|
145
|
-
// (b) "replied" — was excluded entirely in the original PR, which
|
|
146
|
-
// missed the goal: every fulfilled task lives at 'replied'
|
|
147
|
-
// status, so the tasks table's MAIN GROWTH SOURCE would never
|
|
148
|
-
// be reclaimed. Reviewer caught this. Fix: child-ref-aware
|
|
149
|
-
// delete — reap an old replied row ONLY IF no other task
|
|
150
|
-
// references it as parent_task_id. A row with no children is
|
|
151
|
-
// not a chain ancestor and is safe to drop.
|
|
243
|
+
// #2614 —— 分批删期间关掉 WAL 自动 checkpoint,把 checkpoint 收敛到本轮末尾那一次
|
|
244
|
+
// 显式 wal_checkpoint(TRUNCATE)(本来就有的)。
|
|
152
245
|
//
|
|
153
|
-
//
|
|
154
|
-
//
|
|
155
|
-
//
|
|
156
|
-
//
|
|
157
|
-
//
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
246
|
+
// 为什么:自动 checkpoint 是**连接级、每 1000 页触发**的,它会落进任意一批里。
|
|
247
|
+
// 在 1.09 GB 的生产副本上实测(68k 行积压、批大小 2000):
|
|
248
|
+
// 自动 checkpoint 开 → 批次中位数 301ms,**最慢一批 4365ms**(分批白做)
|
|
249
|
+
// 自动 checkpoint 关 → 批次中位数 89ms,最慢一批 242ms
|
|
250
|
+
// 也就是说「分批」必须配「自己决定何时 checkpoint,而不是让 SQLite 在你删到一半时决定」。
|
|
251
|
+
const autoCheckpointBefore = readWalAutoCheckpoint();
|
|
252
|
+
if (autoCheckpointBefore !== null) setWalAutoCheckpoint(0);
|
|
253
|
+
let telemetry: TableSweepResult;
|
|
254
|
+
let taskEvents: TableSweepResult;
|
|
255
|
+
let inbox: TableSweepResult;
|
|
256
|
+
let tasksSafeTerminal: TableSweepResult;
|
|
257
|
+
let tasksRepliedChildSafe: TableSweepResult;
|
|
258
|
+
let auditLog: TableSweepResult;
|
|
259
|
+
try {
|
|
260
|
+
|
|
261
|
+
telemetry = await sweepTable(
|
|
262
|
+
"agent_telemetry",
|
|
263
|
+
`created_at < datetime('now', ?1)`,
|
|
264
|
+
`-${cfg.telemetryDays} days`,
|
|
265
|
+
cfg.telemetryDays,
|
|
266
|
+
chunkRows,
|
|
267
|
+
);
|
|
268
|
+
taskEvents = await sweepTable(
|
|
269
|
+
"task_events",
|
|
270
|
+
`created_at < datetime('now', ?1)`,
|
|
271
|
+
`-${cfg.taskEventsDays} days`,
|
|
272
|
+
cfg.taskEventsDays,
|
|
273
|
+
chunkRows,
|
|
274
|
+
);
|
|
275
|
+
// Only sweep ACKED inbox rows — unacked may be in-flight delivery
|
|
276
|
+
// for an agent that's offline but coming back. Acked rows have
|
|
277
|
+
// served their purpose and exist only for debugging.
|
|
278
|
+
inbox = await sweepTable(
|
|
279
|
+
"inbox",
|
|
280
|
+
`acked = 1 AND created_at < datetime('now', ?1)`,
|
|
281
|
+
`-${cfg.ackedInboxDays} days`,
|
|
282
|
+
cfg.ackedInboxDays,
|
|
283
|
+
chunkRows,
|
|
284
|
+
);
|
|
285
|
+
// Terminal-task sweep — split into two DELETEs:
|
|
286
|
+
//
|
|
287
|
+
// (a) "safe terminals": cancelled / failed / expired / (theoretical)
|
|
288
|
+
// completed have no chain-ancestor risk and are safe to reap
|
|
289
|
+
// unconditionally past the horizon. NOTE: as of this writing
|
|
290
|
+
// 'completed' is dead code — no code path sets it (every reply
|
|
291
|
+
// uses 'replied'/'failed'/'cancelled' via send_reply or
|
|
292
|
+
// chainReplyToParent). Kept in the IN-list as a forward guard:
|
|
293
|
+
// if someone introduces a 'completed' write later, the sweep
|
|
294
|
+
// picks it up without code changes.
|
|
295
|
+
//
|
|
296
|
+
// (b) "replied" — was excluded entirely in the original PR, which
|
|
297
|
+
// missed the goal: every fulfilled task lives at 'replied'
|
|
298
|
+
// status, so the tasks table's MAIN GROWTH SOURCE would never
|
|
299
|
+
// be reclaimed. Reviewer caught this. Fix: child-ref-aware
|
|
300
|
+
// delete — reap an old replied row ONLY IF no other task
|
|
301
|
+
// references it as parent_task_id. A row with no children is
|
|
302
|
+
// not a chain ancestor and is safe to drop.
|
|
303
|
+
//
|
|
304
|
+
// **Time field** (both DELETEs): use COALESCE(completed_at, created_at).
|
|
305
|
+
// Terminal status means the server set `completed_at` at the
|
|
306
|
+
// reply/fail/cancel moment. Sweeping on `created_at` alone would
|
|
307
|
+
// reap a task created 60d ago but completed today the instant it
|
|
308
|
+
// entered terminal state — zero post-terminal retention. The
|
|
309
|
+
// COALESCE falls back to created_at for legacy rows that pre-date
|
|
310
|
+
// the completed_at column (back-compat).
|
|
311
|
+
tasksSafeTerminal = await sweepTable(
|
|
312
|
+
"tasks",
|
|
313
|
+
`status IN ('completed', 'cancelled', 'failed', 'expired')
|
|
314
|
+
AND COALESCE(completed_at, created_at) < datetime('now', ?1)`,
|
|
315
|
+
`-${cfg.terminalTasksDays} days`,
|
|
316
|
+
cfg.terminalTasksDays,
|
|
317
|
+
chunkRows,
|
|
318
|
+
);
|
|
319
|
+
tasksRepliedChildSafe = await sweepTable(
|
|
320
|
+
"tasks",
|
|
321
|
+
`status = 'replied'
|
|
322
|
+
AND COALESCE(completed_at, created_at) < datetime('now', ?1)
|
|
323
|
+
AND NOT EXISTS (
|
|
324
|
+
SELECT 1 FROM tasks AS child
|
|
325
|
+
WHERE child.parent_task_id = tasks.task_id
|
|
326
|
+
)`,
|
|
327
|
+
`-${cfg.terminalTasksDays} days`,
|
|
328
|
+
cfg.terminalTasksDays,
|
|
329
|
+
chunkRows,
|
|
330
|
+
);
|
|
331
|
+
auditLog = await sweepTable(
|
|
332
|
+
"audit_log",
|
|
333
|
+
`created_at < datetime('now', ?1)`,
|
|
334
|
+
`-${cfg.auditLogDays} days`,
|
|
335
|
+
cfg.auditLogDays,
|
|
336
|
+
chunkRows,
|
|
337
|
+
);
|
|
338
|
+
} finally {
|
|
339
|
+
if (autoCheckpointBefore !== null) setWalAutoCheckpoint(autoCheckpointBefore);
|
|
340
|
+
}
|
|
341
|
+
const tasksDeleted = tasksSafeTerminal.deleted + tasksRepliedChildSafe.deleted;
|
|
184
342
|
|
|
185
343
|
const nodeRequests = sweepNodeRequestContent();
|
|
186
344
|
|
|
@@ -233,12 +391,29 @@ export function sweepRetention(cfg: RetentionConfig = readRetentionConfig()): Sw
|
|
|
233
391
|
startedAt,
|
|
234
392
|
durationMs: Date.now() - t0,
|
|
235
393
|
deletes: {
|
|
236
|
-
agent_telemetry: telemetry,
|
|
237
|
-
task_events: taskEvents,
|
|
238
|
-
inbox,
|
|
239
|
-
tasks,
|
|
240
|
-
audit_log: auditLog,
|
|
394
|
+
agent_telemetry: telemetry.deleted,
|
|
395
|
+
task_events: taskEvents.deleted,
|
|
396
|
+
inbox: inbox.deleted,
|
|
397
|
+
tasks: tasksDeleted,
|
|
398
|
+
audit_log: auditLog.deleted,
|
|
241
399
|
},
|
|
400
|
+
// #2614 —— 分批可见性:批次数与「被 maxTableMs 截断的表」
|
|
401
|
+
batches:
|
|
402
|
+
telemetry.batches + taskEvents.batches + inbox.batches +
|
|
403
|
+
tasksSafeTerminal.batches + tasksRepliedChildSafe.batches + auditLog.batches,
|
|
404
|
+
// 生产上「一次 sweep 最长阻塞多久」= 这个数字(#2614 的验收看它)
|
|
405
|
+
maxBatchMs: Math.round(Math.max(
|
|
406
|
+
telemetry.maxBatchMs, taskEvents.maxBatchMs, inbox.maxBatchMs,
|
|
407
|
+
tasksSafeTerminal.maxBatchMs, tasksRepliedChildSafe.maxBatchMs, auditLog.maxBatchMs,
|
|
408
|
+
)),
|
|
409
|
+
truncated: [
|
|
410
|
+
["agent_telemetry", telemetry],
|
|
411
|
+
["task_events", taskEvents],
|
|
412
|
+
["inbox", inbox],
|
|
413
|
+
["tasks(safe)", tasksSafeTerminal],
|
|
414
|
+
["tasks(replied)", tasksRepliedChildSafe],
|
|
415
|
+
["audit_log", auditLog],
|
|
416
|
+
].filter(([, r]) => (r as TableSweepResult).truncated).map(([n]) => n as string),
|
|
242
417
|
nodeRequests,
|
|
243
418
|
vacuum: { walCheckpointPagesMoved, incrementalFreedPages, errored },
|
|
244
419
|
};
|
|
@@ -250,9 +425,17 @@ export function startRetentionSweeper(
|
|
|
250
425
|
intervalMs = 60 * 60 * 1000, // 1 hour
|
|
251
426
|
cfg?: RetentionConfig,
|
|
252
427
|
): ReturnType<typeof setInterval> {
|
|
253
|
-
|
|
428
|
+
// 防重入:一轮 sweep 现在会跨多个 tick(每批之间让出事件循环)。若上一轮还没
|
|
429
|
+
// 跑完就再来一轮,两轮会同时删同一批行 —— 白做功,还会把写锁争用放大。
|
|
430
|
+
let running = false;
|
|
431
|
+
const sweep = async () => {
|
|
432
|
+
if (running) {
|
|
433
|
+
console.log("[commhub retention] previous sweep still running — skipping this tick");
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
436
|
+
running = true;
|
|
254
437
|
try {
|
|
255
|
-
const r = sweepRetention(cfg);
|
|
438
|
+
const r = await sweepRetention(cfg);
|
|
256
439
|
const totalDeletes =
|
|
257
440
|
r.deletes.agent_telemetry +
|
|
258
441
|
r.deletes.task_events +
|
|
@@ -262,20 +445,23 @@ export function startRetentionSweeper(
|
|
|
262
445
|
const nr = r.nodeRequests;
|
|
263
446
|
if (totalDeletes > 0 || nr.purged > 0 || nr.deleted > 0 || nr.timedOut > 0 || r.vacuum.errored) {
|
|
264
447
|
console.log(
|
|
265
|
-
`[commhub retention] swept in ${r.durationMs}ms — ` +
|
|
448
|
+
`[commhub retention] swept in ${r.durationMs}ms (${r.batches} batches) — ` +
|
|
266
449
|
`telemetry=${r.deletes.agent_telemetry} events=${r.deletes.task_events} ` +
|
|
267
450
|
`inbox=${r.deletes.inbox} tasks=${r.deletes.tasks} audit=${r.deletes.audit_log} ` +
|
|
268
451
|
`nodeReqPurged=${nr.purged} nodeReqDeleted=${nr.deleted} nodeReqTimedOut=${nr.timedOut} ` +
|
|
269
|
-
`walPages=${r.vacuum.walCheckpointPagesMoved} freedPages=${r.vacuum.incrementalFreedPages}
|
|
452
|
+
`walPages=${r.vacuum.walCheckpointPagesMoved} freedPages=${r.vacuum.incrementalFreedPages}` +
|
|
453
|
+
(r.truncated.length ? ` truncated=${r.truncated.join(",")}` : ""),
|
|
270
454
|
);
|
|
271
455
|
}
|
|
272
456
|
} catch (e: any) {
|
|
273
457
|
console.log(`[commhub retention] sweep top-level failed: ${e?.message ?? e}`);
|
|
458
|
+
} finally {
|
|
459
|
+
running = false;
|
|
274
460
|
}
|
|
275
461
|
};
|
|
276
462
|
// Fire once on startup so a long-running hub eventually catches up
|
|
277
463
|
// even if it's restarted just after the previous sweep. setImmediate
|
|
278
464
|
// (vs synchronous) keeps server startup from blocking on the sweep.
|
|
279
|
-
setImmediate(sweep);
|
|
280
|
-
return setInterval(sweep, intervalMs);
|
|
465
|
+
setImmediate(() => { void sweep(); });
|
|
466
|
+
return setInterval(() => { void sweep(); }, intervalMs);
|
|
281
467
|
}
|