@sleep2agi/commhub-server 0.9.0-preview.168 → 0.9.0-preview.169
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/retention.ts +37 -17
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sleep2agi/commhub-server",
|
|
3
|
-
"version": "0.9.0-preview.
|
|
3
|
+
"version": "0.9.0-preview.169",
|
|
4
4
|
"description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
package/src/retention.ts
CHANGED
|
@@ -107,16 +107,23 @@ export function readRetentionConfig(): RetentionConfig {
|
|
|
107
107
|
// 根因不是「删得多」,而是**一次 DELETE 把整批行放在一个语句里,中间不让出事件循环**。
|
|
108
108
|
//
|
|
109
109
|
// 修法(与 `.167` 的 terminal_events 回填同形):
|
|
110
|
-
// 1. 每批只删 chunkRows
|
|
111
|
-
//
|
|
110
|
+
// 1. 每批只删 chunkRows 行,用 `DELETE … WHERE rowid IN (SELECT rowid FROM t WHERE <谓词> LIMIT n)`
|
|
111
|
+
// —— 谓词有索引时这是索引范围扫描(O(n));没有索引时它退化成一次全扫,
|
|
112
|
+
// 所以「谓词必须有索引」是这条路径的前提(见下面的三条实测);
|
|
112
113
|
// 2. 每批之间 `await` 一次 setTimeout(0),把控制权交还事件循环;
|
|
113
114
|
// 3. 删的这段时间关掉 WAL 自动 checkpoint,本轮的 checkpoint 收敛到末尾那一次
|
|
114
115
|
// TRUNCATE(否则 checkpoint 会落进任意一批,把 90ms 的批次顶到 4 秒);
|
|
115
116
|
// 4. 单表最多跑 MAX_TABLE_MS,超了就停下并记日志(下一轮继续)。
|
|
116
117
|
//
|
|
117
|
-
//
|
|
118
|
-
//
|
|
119
|
-
//
|
|
118
|
+
// 三条实测(都在 1.09 GB 生产副本上):
|
|
119
|
+
// · 一次删完 3.1s / 单块 2686ms
|
|
120
|
+
// · 分批 + rowid 游标(副本里谓词**没有**索引) 5.3s / 单块 258ms
|
|
121
|
+
// · 分批 + rowid 游标(生产,谓词**有**索引) → 单块 4716ms 🔴 打回
|
|
122
|
+
// `ORDER BY rowid` 让优化器放弃 created_at 索引,为了凑够一批从 rowid=0 扫全表;
|
|
123
|
+
// 只删 266 行花了 4.7 秒(`.168` 上线第一拍的真实日志)。
|
|
124
|
+
// · 分批 + 索引驱动 LIMIT(现状,生产副本) 1.8s / 单块 249ms
|
|
125
|
+
// ⇒ 教训:**副本的 schema 也必须与生产一致**,否则「按副本优化」会变成新的全表扫。
|
|
126
|
+
// 总耗时略增是有意的取舍:**可被打断优先于最快跑完**。
|
|
120
127
|
const DEFAULT_CHUNK_ROWS = 2000;
|
|
121
128
|
const MAX_TABLE_MS = 10 * 60 * 1000;
|
|
122
129
|
|
|
@@ -189,7 +196,6 @@ async function sweepTable(
|
|
|
189
196
|
let deleted = 0;
|
|
190
197
|
let batches = 0;
|
|
191
198
|
let maxBatchMs = 0;
|
|
192
|
-
let cursor = 0;
|
|
193
199
|
try {
|
|
194
200
|
for (;;) {
|
|
195
201
|
const batchT0 = performance.now();
|
|
@@ -204,9 +210,8 @@ async function sweepTable(
|
|
|
204
210
|
// `SEARCH ... USING INTEGER PRIMARY KEY (rowid>?)`:每批只从游标往后读,
|
|
205
211
|
// 整轮总代价 ≈ 一次全表扫描 + 删除本身。
|
|
206
212
|
const rows = db.all<{ r: number }>(
|
|
207
|
-
`SELECT rowid AS r FROM ${table} WHERE ${where}
|
|
213
|
+
`SELECT rowid AS r FROM ${table} WHERE ${where} LIMIT ?2`,
|
|
208
214
|
param,
|
|
209
|
-
cursor,
|
|
210
215
|
chunkRows,
|
|
211
216
|
);
|
|
212
217
|
if (rows.length === 0) return { deleted, batches, maxBatchMs, truncated: false };
|
|
@@ -216,7 +221,6 @@ async function sweepTable(
|
|
|
216
221
|
deleted += res.changes ?? 0;
|
|
217
222
|
batches += 1;
|
|
218
223
|
maxBatchMs = Math.max(maxBatchMs, performance.now() - batchT0);
|
|
219
|
-
cursor = ids[ids.length - 1]!;
|
|
220
224
|
// 这一批没取满 ⇒ 已经没有更多符合条件的行了,收工。
|
|
221
225
|
if (rows.length < chunkRows) return { deleted, batches, maxBatchMs, truncated: false };
|
|
222
226
|
if (Date.now() - startedMs > MAX_TABLE_MS) {
|
|
@@ -361,16 +365,32 @@ export async function sweepRetention(cfg: RetentionConfig = readRetentionConfig(
|
|
|
361
365
|
let incrementalFreedPages: number | null = null;
|
|
362
366
|
let errored = false;
|
|
363
367
|
const sqliteVacuum = db.dialect === "sqlite";
|
|
364
|
-
if (sqliteVacuum)
|
|
368
|
+
if (sqliteVacuum) {
|
|
365
369
|
// wal_checkpoint(TRUNCATE) returns a row { busy, log, checkpointed }.
|
|
366
370
|
// We expose `log` (page count moved into the main DB) for ops visibility.
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
371
|
+
//
|
|
372
|
+
// #2614 —— 重试 3 次:本轮刚跑完一批 `db.all` 时,同连接上可能还有未重置的
|
|
373
|
+
// 读语句,`wal_checkpoint` 会直接报 `database table is locked`(SQLITE_LOCKED,
|
|
374
|
+
// 不是 BUSY)。这是**瞬时**的:让出一次事件循环再试就好。别把它记成 errored
|
|
375
|
+
// —— 那会让「本轮 vacuum 是否出错」这个字段说谎。
|
|
376
|
+
let lastErr: any = null;
|
|
377
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
378
|
+
try {
|
|
379
|
+
const checkpoint = db.get<{ busy: number; log: number; checkpointed: number }>(
|
|
380
|
+
"PRAGMA wal_checkpoint(TRUNCATE)",
|
|
381
|
+
);
|
|
382
|
+
walCheckpointPagesMoved = checkpoint?.log ?? null;
|
|
383
|
+
lastErr = null;
|
|
384
|
+
break;
|
|
385
|
+
} catch (e: any) {
|
|
386
|
+
lastErr = e;
|
|
387
|
+
await yieldToLoop();
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
if (lastErr) {
|
|
391
|
+
console.log(`[commhub retention] wal_checkpoint failed after 3 attempts: ${lastErr?.message ?? lastErr}`);
|
|
392
|
+
errored = true;
|
|
393
|
+
}
|
|
374
394
|
}
|
|
375
395
|
if (sqliteVacuum) try {
|
|
376
396
|
// incremental_vacuum is a no-op on databases not created with
|