@sleep2agi/commhub-server 0.9.0-preview.168 → 0.9.0-preview.170
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/empty-node-sweeper.test.ts +104 -0
- package/src/empty-node-sweeper.ts +88 -0
- package/src/retention.ts +65 -26
- package/src/server.ts +18 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sleep2agi/commhub-server",
|
|
3
|
-
"version": "0.9.0-preview.
|
|
3
|
+
"version": "0.9.0-preview.170",
|
|
4
4
|
"description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// #2596 —— 空白 node 行的清理:只删「三身份字段全空 + 无 session + 够旧」的行。
|
|
2
|
+
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
3
|
+
import { db } from "./db.js";
|
|
4
|
+
import { readEmptyNodeAgeDays, sweepEmptyNodeRows } from "./empty-node-sweeper.js";
|
|
5
|
+
|
|
6
|
+
let networkId = "";
|
|
7
|
+
|
|
8
|
+
function insertNode(
|
|
9
|
+
nodeId: string,
|
|
10
|
+
alias: string,
|
|
11
|
+
opts: { identity?: boolean; updatedDaysAgo?: number; createdDaysAgo?: number } = {},
|
|
12
|
+
) {
|
|
13
|
+
const { identity = false, updatedDaysAgo = 10, createdDaysAgo = 30 } = opts;
|
|
14
|
+
db.run(
|
|
15
|
+
`INSERT INTO nodes (node_id, node_name, alias, runtime, hostname, config_path, network_id, created_at, updated_at)
|
|
16
|
+
VALUES (?1, ?2, ?2, ?3, ?4, ?5, ?6, datetime('now', ?8), datetime('now', ?7))`,
|
|
17
|
+
[
|
|
18
|
+
nodeId,
|
|
19
|
+
alias,
|
|
20
|
+
identity ? "codex-app-server-sdk" : null,
|
|
21
|
+
identity ? "host-a" : null,
|
|
22
|
+
identity ? "/data/x" : null,
|
|
23
|
+
networkId,
|
|
24
|
+
`-${updatedDaysAgo} days`,
|
|
25
|
+
`-${createdDaysAgo} days`,
|
|
26
|
+
],
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function exists(nodeId: string): boolean {
|
|
31
|
+
return !!db.get<{ node_id: string }>("SELECT node_id FROM nodes WHERE node_id = ?1", [nodeId]);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
beforeAll(async () => {
|
|
35
|
+
const { register } = await import("./auth.js");
|
|
36
|
+
const auth = register(`empty_gc_${Date.now()}`, "EmptyGc-Str0ng-1!", undefined, "watcher");
|
|
37
|
+
expect(auth.ok).toBe(true);
|
|
38
|
+
networkId = auth.network_id!;
|
|
39
|
+
|
|
40
|
+
insertNode("gc-empty-old", "ghost-a"); // 该删
|
|
41
|
+
insertNode("gc-empty-fresh", "ghost-b", { createdDaysAgo: 0, updatedDaysAgo: 0 }); // 太新:可能正在注册
|
|
42
|
+
insertNode("gc-real", "real-node", { identity: true }); // 有身份:绝不删
|
|
43
|
+
insertNode("gc-real-alias-2", "real-node", { identity: true }); // 不同机器同 alias:也绝不删
|
|
44
|
+
// 空壳但有 session(在线但还没上报身份)⇒ 不删
|
|
45
|
+
insertNode("gc-empty-with-session", "ghost-c");
|
|
46
|
+
db.run(
|
|
47
|
+
`INSERT INTO sessions (resume_id, alias, node_id, network_id, status, registered_at)
|
|
48
|
+
VALUES ('res_gc', 'ghost-c', 'gc-empty-with-session', ?1, 'offline', '2026-10-01 00:00:00')`,
|
|
49
|
+
[networkId],
|
|
50
|
+
);
|
|
51
|
+
}, 30_000);
|
|
52
|
+
|
|
53
|
+
afterAll(() => {
|
|
54
|
+
delete process.env.COMMHUB_EMPTY_NODE_GC_DAYS;
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
describe("sweepEmptyNodeRows (#2596)", () => {
|
|
58
|
+
test("dry run reports the target but deletes nothing", () => {
|
|
59
|
+
const r = sweepEmptyNodeRows({ dryRun: true });
|
|
60
|
+
expect(r.ids).toContain("gc-empty-old");
|
|
61
|
+
expect(r.deleted).toBe(0);
|
|
62
|
+
expect(exists("gc-empty-old")).toBe(true);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test("deletes only the old, identity-less, session-less placeholder", () => {
|
|
66
|
+
const r = sweepEmptyNodeRows();
|
|
67
|
+
expect(exists("gc-empty-old")).toBe(false);
|
|
68
|
+
expect(r.deleted).toBeGreaterThanOrEqual(1);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test("keeps fresh placeholders (they may be mid-registration)", () => {
|
|
72
|
+
sweepEmptyNodeRows();
|
|
73
|
+
expect(exists("gc-empty-fresh")).toBe(true);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
test("keeps rows with an identity, even when two nodes share an alias", () => {
|
|
77
|
+
sweepEmptyNodeRows();
|
|
78
|
+
expect(exists("gc-real")).toBe(true);
|
|
79
|
+
expect(exists("gc-real-alias-2")).toBe(true);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test("keeps a placeholder that still has a session", () => {
|
|
83
|
+
sweepEmptyNodeRows();
|
|
84
|
+
expect(exists("gc-empty-with-session")).toBe(true);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("a fresh bulk update does NOT protect an old empty row (created_at is the clock)", () => {
|
|
88
|
+
// 生产实测:2026-10-26 的一次批量更新把 08/09 月建的空壳 updated_at 全刷成 10-26,
|
|
89
|
+
// 若按 updated_at 判龄,2 天阈值永远删不掉它们。
|
|
90
|
+
insertNode("gc-empty-bulk-touched", "ghost-d", { createdDaysAgo: 40, updatedDaysAgo: 0 });
|
|
91
|
+
sweepEmptyNodeRows();
|
|
92
|
+
expect(exists("gc-empty-bulk-touched")).toBe(false);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("env <= 0 disables the sweep", () => {
|
|
96
|
+
process.env.COMMHUB_EMPTY_NODE_GC_DAYS = "0";
|
|
97
|
+
try {
|
|
98
|
+
expect(readEmptyNodeAgeDays()).toBe(0);
|
|
99
|
+
expect(sweepEmptyNodeRows().deleted).toBe(0);
|
|
100
|
+
} finally {
|
|
101
|
+
delete process.env.COMMHUB_EMPTY_NODE_GC_DAYS;
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
});
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// #2596 —— 组织架构页那些“重复名字”的真身:**注册失败留下的空白 node 行**。
|
|
2
|
+
//
|
|
3
|
+
// 生产实测(2026-10-27,只读):
|
|
4
|
+
// ```
|
|
5
|
+
// nodes 总数 312
|
|
6
|
+
// runtime / hostname / config_path 全为空、且没有任何 session 的行 = 12
|
|
7
|
+
// 形态高度一致:同一 alias 在几分钟内连着出现,最后一个才被填上真实身份
|
|
8
|
+
// TMA门户鲸2号 ×5 09-17 15:47:52 → 16:03:05 (6 分钟注册 5 次,第 6 次成功)
|
|
9
|
+
// ```
|
|
10
|
+
// ⇒ 注册流程先建行、节点再补身份;补不上的尝试留下永久空壳,被组织架构页当成 node 渲染。
|
|
11
|
+
//
|
|
12
|
+
// 这里只做**清理**,不动注册流程:
|
|
13
|
+
// · 只删「三个身份字段全空 + 没有任何 session」的行 —— 这种行不可能提供任何功能;
|
|
14
|
+
// · 只删 `created_at` 早于 N 天(默认 2 天)的行 —— 正在注册中的行自然不会被误删;
|
|
15
|
+
// (用 created_at 不用 updated_at:后者会被无关的批量更新刷新,见下方注释里的生产实例)
|
|
16
|
+
// · 删除前把整行导成 JSON 到日志(可人工恢复),并返回被删的 id 列表。
|
|
17
|
+
//
|
|
18
|
+
// 为什么不用「同 alias 只留一条」这种规则:不同机器共用 alias 是**数据问题不是脏数据**
|
|
19
|
+
// (Mac打包狗 / TM微软牛 / TM汇报牛 这类),删了会丢真实节点。空壳判据才安全。
|
|
20
|
+
|
|
21
|
+
import { db } from "./db.js";
|
|
22
|
+
|
|
23
|
+
export const EMPTY_NODE_AGE_DAYS_ENV = "COMMHUB_EMPTY_NODE_GC_DAYS";
|
|
24
|
+
export const EMPTY_NODE_AGE_DAYS_DEFAULT = 2;
|
|
25
|
+
|
|
26
|
+
export type EmptyNodeSweepResult = {
|
|
27
|
+
deleted: number;
|
|
28
|
+
ids: string[];
|
|
29
|
+
ageDays: number;
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
export function readEmptyNodeAgeDays(raw = process.env[EMPTY_NODE_AGE_DAYS_ENV]): number {
|
|
33
|
+
if (raw === undefined || raw === "") return EMPTY_NODE_AGE_DAYS_DEFAULT;
|
|
34
|
+
const n = Number(raw);
|
|
35
|
+
if (!Number.isFinite(n)) return EMPTY_NODE_AGE_DAYS_DEFAULT;
|
|
36
|
+
return Math.floor(n);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const IDENTITY_EMPTY = `COALESCE(runtime, '') = '' AND COALESCE(hostname, '') = '' AND COALESCE(config_path, '') = ''`;
|
|
40
|
+
|
|
41
|
+
export function sweepEmptyNodeRows(
|
|
42
|
+
opts: { olderThanDays?: number; dryRun?: boolean } = {},
|
|
43
|
+
): EmptyNodeSweepResult {
|
|
44
|
+
const ageDays = opts.olderThanDays ?? readEmptyNodeAgeDays();
|
|
45
|
+
// <=0 关闭(运维可以彻底停掉这个清理)
|
|
46
|
+
if (ageDays <= 0) return { deleted: 0, ids: [], ageDays: 0 };
|
|
47
|
+
|
|
48
|
+
const rows = db.all<{ node_id: string }>(
|
|
49
|
+
`SELECT n.node_id
|
|
50
|
+
FROM nodes n
|
|
51
|
+
WHERE ${IDENTITY_EMPTY.replace(/\b(runtime|hostname|config_path)\b/g, "n.$1")}
|
|
52
|
+
-- #2596 —— 用 created_at 而不是 updated_at:空壳行会被无关的批量更新
|
|
53
|
+
-- 把 updated_at 顶到「今天」(生产实测:2026-10-26 的一次批量更新把 08/09 月
|
|
54
|
+
-- 建的空壳全部刷成 10-26),那样 2 天的阈值就永远删不掉它们。
|
|
55
|
+
-- created_at 才是“这行空了多久”的稳定量;在注册中的行 created_at 必然是新的。
|
|
56
|
+
AND n.created_at < datetime('now', ?1)
|
|
57
|
+
AND NOT EXISTS (SELECT 1 FROM sessions s WHERE s.node_id = n.node_id)
|
|
58
|
+
LIMIT 500`,
|
|
59
|
+
`-${ageDays} days`,
|
|
60
|
+
);
|
|
61
|
+
if (rows.length === 0) return { deleted: 0, ids: [], ageDays };
|
|
62
|
+
|
|
63
|
+
const ids = rows.map((r) => r.node_id);
|
|
64
|
+
if (opts.dryRun) return { deleted: 0, ids, ageDays };
|
|
65
|
+
|
|
66
|
+
// 删前留证:整行 JSON 打到日志里(运维可据此恢复),再逐条删。
|
|
67
|
+
try {
|
|
68
|
+
const full = db.all<Record<string, unknown>>(
|
|
69
|
+
`SELECT * FROM nodes WHERE node_id IN (${ids.map((_, i) => `?${i + 1}`).join(",")})`,
|
|
70
|
+
ids,
|
|
71
|
+
);
|
|
72
|
+
console.log(`[commhub empty-node-gc] deleting ${ids.length} placeholder node row(s): ${JSON.stringify(full)}`);
|
|
73
|
+
} catch (e: any) {
|
|
74
|
+
console.log(`[commhub empty-node-gc] backup dump failed, aborting: ${e?.message ?? e}`);
|
|
75
|
+
return { deleted: 0, ids: [], ageDays };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
let deleted = 0;
|
|
79
|
+
for (const id of ids) {
|
|
80
|
+
try {
|
|
81
|
+
const res = db.run(`DELETE FROM nodes WHERE node_id = ?1 AND ${IDENTITY_EMPTY}`, [id]);
|
|
82
|
+
deleted += res.changes ?? 0;
|
|
83
|
+
} catch (e: any) {
|
|
84
|
+
console.log(`[commhub empty-node-gc] delete ${id} failed: ${e?.message ?? e}`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return { deleted, ids, ageDays };
|
|
88
|
+
}
|
package/src/retention.ts
CHANGED
|
@@ -63,6 +63,8 @@ export type SweepResult = {
|
|
|
63
63
|
batches: number;
|
|
64
64
|
/** 整轮里单批最长耗时(毫秒)。生产上这条数字 = 「一次 sweep 最长阻塞多久」。 */
|
|
65
65
|
maxBatchMs: number;
|
|
66
|
+
/** 各阶段耗时(毫秒):多张表 + vacuum。生产上「15 秒里谁花的」靠它,不靠猜。 */
|
|
67
|
+
phases: Record<string, number>;
|
|
66
68
|
truncated: string[];
|
|
67
69
|
nodeRequests: NodeRequestSweepResult;
|
|
68
70
|
vacuum: {
|
|
@@ -107,17 +109,26 @@ export function readRetentionConfig(): RetentionConfig {
|
|
|
107
109
|
// 根因不是「删得多」,而是**一次 DELETE 把整批行放在一个语句里,中间不让出事件循环**。
|
|
108
110
|
//
|
|
109
111
|
// 修法(与 `.167` 的 terminal_events 回填同形):
|
|
110
|
-
// 1. 每批只删 chunkRows
|
|
111
|
-
//
|
|
112
|
+
// 1. 每批只删 chunkRows 行,用 `DELETE … WHERE rowid IN (SELECT rowid FROM t WHERE <谓词> LIMIT n)`
|
|
113
|
+
// —— 谓词有索引时这是索引范围扫描(O(n));没有索引时它退化成一次全扫,
|
|
114
|
+
// 所以「谓词必须有索引」是这条路径的前提(见下面的三条实测);
|
|
112
115
|
// 2. 每批之间 `await` 一次 setTimeout(0),把控制权交还事件循环;
|
|
113
116
|
// 3. 删的这段时间关掉 WAL 自动 checkpoint,本轮的 checkpoint 收敛到末尾那一次
|
|
114
117
|
// TRUNCATE(否则 checkpoint 会落进任意一批,把 90ms 的批次顶到 4 秒);
|
|
115
118
|
// 4. 单表最多跑 MAX_TABLE_MS,超了就停下并记日志(下一轮继续)。
|
|
116
119
|
//
|
|
117
|
-
//
|
|
118
|
-
//
|
|
119
|
-
//
|
|
120
|
+
// 三条实测(都在 1.09 GB 生产副本上):
|
|
121
|
+
// · 一次删完 3.1s / 单块 2686ms
|
|
122
|
+
// · 分批 + rowid 游标(副本里谓词**没有**索引) 5.3s / 单块 258ms
|
|
123
|
+
// · 分批 + rowid 游标(生产,谓词**有**索引) → 单块 4716ms 🔴 打回
|
|
124
|
+
// `ORDER BY rowid` 让优化器放弃 created_at 索引,为了凑够一批从 rowid=0 扫全表;
|
|
125
|
+
// 只删 266 行花了 4.7 秒(`.168` 上线第一拍的真实日志)。
|
|
126
|
+
// · 分批 + 索引驱动 LIMIT(现状,生产副本) 1.8s / 单块 249ms
|
|
127
|
+
// ⇒ 教训:**副本的 schema 也必须与生产一致**,否则「按副本优化」会变成新的全表扫。
|
|
128
|
+
// 总耗时略增是有意的取舍:**可被打断优先于最快跑完**。
|
|
120
129
|
const DEFAULT_CHUNK_ROWS = 2000;
|
|
130
|
+
/** 每次 sweep 最多回收多少空闲页(≈4 KB/页)。见下面的 incremental_vacuum 注释。 */
|
|
131
|
+
const INCREMENTAL_VACUUM_PAGES = 200;
|
|
121
132
|
const MAX_TABLE_MS = 10 * 60 * 1000;
|
|
122
133
|
|
|
123
134
|
function readChunkRows(): number {
|
|
@@ -156,6 +167,8 @@ function yieldToLoop(): Promise<void> {
|
|
|
156
167
|
export type TableSweepResult = {
|
|
157
168
|
deleted: number;
|
|
158
169
|
batches: number;
|
|
170
|
+
/** 这张表这轮花了多久(毫秒)—— 生产上排查「15 秒里到底谁花的」靠它。 */
|
|
171
|
+
ms: number;
|
|
159
172
|
/** 单批最长耗时(毫秒)—— 生产上「一次 sweep 最长阻塞多久」就是这个数。 */
|
|
160
173
|
maxBatchMs: number;
|
|
161
174
|
/** 单表跑满 maxTableMs 被截断;剩余行下一轮继续(不吞掉、不假装完成)。 */
|
|
@@ -172,24 +185,24 @@ async function sweepTable(
|
|
|
172
185
|
// days < 0 → operator opt-out for this table; skip the DELETE.
|
|
173
186
|
// days === 0 is a legitimate "sweep everything older than now"
|
|
174
187
|
// (effectively a TRUNCATE); we honour it.
|
|
175
|
-
if (days < 0) return { deleted: 0, batches: 0, maxBatchMs: 0, truncated: false };
|
|
188
|
+
if (days < 0) return { deleted: 0, batches: 0, maxBatchMs: 0, ms: 0, truncated: false };
|
|
176
189
|
// PostgreSQL: 单条 DELETE + autovacuum 本来就按批回收,且报文里报的 10 秒停顿
|
|
177
190
|
// 出在 SQLite 侧(`#2614` 的生产现场是 SQLite)。这里不引入 rowid(pg 没有),
|
|
178
191
|
// 保持原样,避免在没观测到问题的方言上改语义。
|
|
179
192
|
if (db.dialect !== "sqlite") {
|
|
180
193
|
try {
|
|
181
194
|
const res = db.run(`DELETE FROM ${table} WHERE ${where}`, [param]);
|
|
182
|
-
return { deleted: res.changes ?? 0, batches: 1, maxBatchMs: 0, truncated: false };
|
|
195
|
+
return { deleted: res.changes ?? 0, batches: 1, maxBatchMs: 0, ms: 0, truncated: false };
|
|
183
196
|
} catch (e: any) {
|
|
184
197
|
console.log(`[commhub retention] sweep failed (${table}): ${e?.message ?? e}`);
|
|
185
|
-
return { deleted: 0, batches: 0, maxBatchMs: 0, truncated: false };
|
|
198
|
+
return { deleted: 0, batches: 0, maxBatchMs: 0, ms: 0, truncated: false };
|
|
186
199
|
}
|
|
187
200
|
}
|
|
188
201
|
const startedMs = Date.now();
|
|
202
|
+
const tTable0 = performance.now();
|
|
189
203
|
let deleted = 0;
|
|
190
204
|
let batches = 0;
|
|
191
205
|
let maxBatchMs = 0;
|
|
192
|
-
let cursor = 0;
|
|
193
206
|
try {
|
|
194
207
|
for (;;) {
|
|
195
208
|
const batchT0 = performance.now();
|
|
@@ -204,34 +217,32 @@ async function sweepTable(
|
|
|
204
217
|
// `SEARCH ... USING INTEGER PRIMARY KEY (rowid>?)`:每批只从游标往后读,
|
|
205
218
|
// 整轮总代价 ≈ 一次全表扫描 + 删除本身。
|
|
206
219
|
const rows = db.all<{ r: number }>(
|
|
207
|
-
`SELECT rowid AS r FROM ${table} WHERE ${where}
|
|
220
|
+
`SELECT rowid AS r FROM ${table} WHERE ${where} LIMIT ?2`,
|
|
208
221
|
param,
|
|
209
|
-
cursor,
|
|
210
222
|
chunkRows,
|
|
211
223
|
);
|
|
212
|
-
if (rows.length === 0) return { deleted, batches, maxBatchMs, truncated: false };
|
|
224
|
+
if (rows.length === 0) return { deleted, batches, maxBatchMs, ms: Math.round(performance.now() - tTable0), truncated: false };
|
|
213
225
|
const ids = rows.map((r) => r.r);
|
|
214
226
|
const placeholders = ids.map((_, i) => `?${i + 1}`).join(",");
|
|
215
227
|
const res = db.run(`DELETE FROM ${table} WHERE rowid IN (${placeholders})`, ids);
|
|
216
228
|
deleted += res.changes ?? 0;
|
|
217
229
|
batches += 1;
|
|
218
230
|
maxBatchMs = Math.max(maxBatchMs, performance.now() - batchT0);
|
|
219
|
-
cursor = ids[ids.length - 1]!;
|
|
220
231
|
// 这一批没取满 ⇒ 已经没有更多符合条件的行了,收工。
|
|
221
|
-
if (rows.length < chunkRows) return { deleted, batches, maxBatchMs, truncated: false };
|
|
232
|
+
if (rows.length < chunkRows) return { deleted, batches, maxBatchMs, ms: Math.round(performance.now() - tTable0), truncated: false };
|
|
222
233
|
if (Date.now() - startedMs > MAX_TABLE_MS) {
|
|
223
234
|
console.log(
|
|
224
235
|
`[commhub retention] ${table}: stopped after ${batches} batches / ${deleted} rows ` +
|
|
225
236
|
`(>${Math.round(MAX_TABLE_MS / 60000)}min); remaining rows continue next sweep`,
|
|
226
237
|
);
|
|
227
|
-
return { deleted, batches, maxBatchMs, truncated: true };
|
|
238
|
+
return { deleted, batches, maxBatchMs, ms: Math.round(performance.now() - tTable0), truncated: true };
|
|
228
239
|
}
|
|
229
240
|
// 让出事件循环:这是 #2614 的全部要点 —— 停顿的可感知单位是「单批」。
|
|
230
241
|
await yieldToLoop();
|
|
231
242
|
}
|
|
232
243
|
} catch (e: any) {
|
|
233
244
|
console.log(`[commhub retention] sweep failed (${table}): ${e?.message ?? e}`);
|
|
234
|
-
return { deleted, batches, maxBatchMs, truncated: false };
|
|
245
|
+
return { deleted, batches, maxBatchMs, ms: Math.round(performance.now() - tTable0), truncated: false };
|
|
235
246
|
}
|
|
236
247
|
}
|
|
237
248
|
|
|
@@ -361,16 +372,32 @@ export async function sweepRetention(cfg: RetentionConfig = readRetentionConfig(
|
|
|
361
372
|
let incrementalFreedPages: number | null = null;
|
|
362
373
|
let errored = false;
|
|
363
374
|
const sqliteVacuum = db.dialect === "sqlite";
|
|
364
|
-
if (sqliteVacuum)
|
|
375
|
+
if (sqliteVacuum) {
|
|
365
376
|
// wal_checkpoint(TRUNCATE) returns a row { busy, log, checkpointed }.
|
|
366
377
|
// We expose `log` (page count moved into the main DB) for ops visibility.
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
378
|
+
//
|
|
379
|
+
// #2614 —— 重试 3 次:本轮刚跑完一批 `db.all` 时,同连接上可能还有未重置的
|
|
380
|
+
// 读语句,`wal_checkpoint` 会直接报 `database table is locked`(SQLITE_LOCKED,
|
|
381
|
+
// 不是 BUSY)。这是**瞬时**的:让出一次事件循环再试就好。别把它记成 errored
|
|
382
|
+
// —— 那会让「本轮 vacuum 是否出错」这个字段说谎。
|
|
383
|
+
let lastErr: any = null;
|
|
384
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
385
|
+
try {
|
|
386
|
+
const checkpoint = db.get<{ busy: number; log: number; checkpointed: number }>(
|
|
387
|
+
"PRAGMA wal_checkpoint(TRUNCATE)",
|
|
388
|
+
);
|
|
389
|
+
walCheckpointPagesMoved = checkpoint?.log ?? null;
|
|
390
|
+
lastErr = null;
|
|
391
|
+
break;
|
|
392
|
+
} catch (e: any) {
|
|
393
|
+
lastErr = e;
|
|
394
|
+
await yieldToLoop();
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
if (lastErr) {
|
|
398
|
+
console.log(`[commhub retention] wal_checkpoint failed after 3 attempts: ${lastErr?.message ?? lastErr}`);
|
|
399
|
+
errored = true;
|
|
400
|
+
}
|
|
374
401
|
}
|
|
375
402
|
if (sqliteVacuum) try {
|
|
376
403
|
// incremental_vacuum is a no-op on databases not created with
|
|
@@ -378,7 +405,11 @@ export async function sweepRetention(cfg: RetentionConfig = readRetentionConfig(
|
|
|
378
405
|
// weren't, the operator would have to run a full one-time VACUUM
|
|
379
406
|
// to migrate — out of scope for the sweeper.
|
|
380
407
|
const freelistBefore = db.get<{ freelist_count: number }>("PRAGMA freelist_count");
|
|
381
|
-
|
|
408
|
+
// #2614 —— 必须带页数上限。`PRAGMA incremental_vacuum`(不带参数)会**一次回收
|
|
409
|
+
// 全部空闲页**:生产 `.169` 上线时那一拍整轮 15.3s,其中大部分是这个语句
|
|
410
|
+
// (它走 db.exec,不进 slow-sql 日志,所以第一眼看不见)。
|
|
411
|
+
// 每小时回 200 页(≈800 KB)足够跟上增长,又不会变成一次长阻塞;剩下的下次继续。
|
|
412
|
+
db.exec(`PRAGMA incremental_vacuum(${INCREMENTAL_VACUUM_PAGES})`);
|
|
382
413
|
const freelistAfter = db.get<{ freelist_count: number }>("PRAGMA freelist_count");
|
|
383
414
|
incrementalFreedPages =
|
|
384
415
|
(freelistBefore?.freelist_count ?? 0) - (freelistAfter?.freelist_count ?? 0);
|
|
@@ -402,6 +433,13 @@ export async function sweepRetention(cfg: RetentionConfig = readRetentionConfig(
|
|
|
402
433
|
telemetry.batches + taskEvents.batches + inbox.batches +
|
|
403
434
|
tasksSafeTerminal.batches + tasksRepliedChildSafe.batches + auditLog.batches,
|
|
404
435
|
// 生产上「一次 sweep 最长阻塞多久」= 这个数字(#2614 的验收看它)
|
|
436
|
+
phases: {
|
|
437
|
+
telemetry: telemetry.ms,
|
|
438
|
+
events: taskEvents.ms,
|
|
439
|
+
inbox: inbox.ms,
|
|
440
|
+
tasks: tasksSafeTerminal.ms + tasksRepliedChildSafe.ms,
|
|
441
|
+
audit: auditLog.ms,
|
|
442
|
+
},
|
|
405
443
|
maxBatchMs: Math.round(Math.max(
|
|
406
444
|
telemetry.maxBatchMs, taskEvents.maxBatchMs, inbox.maxBatchMs,
|
|
407
445
|
tasksSafeTerminal.maxBatchMs, tasksRepliedChildSafe.maxBatchMs, auditLog.maxBatchMs,
|
|
@@ -445,7 +483,8 @@ export function startRetentionSweeper(
|
|
|
445
483
|
const nr = r.nodeRequests;
|
|
446
484
|
if (totalDeletes > 0 || nr.purged > 0 || nr.deleted > 0 || nr.timedOut > 0 || r.vacuum.errored) {
|
|
447
485
|
console.log(
|
|
448
|
-
`[commhub retention] swept in ${r.durationMs}ms (${r.batches} batches) — ` +
|
|
486
|
+
`[commhub retention] swept in ${r.durationMs}ms (${r.batches} batches, max ${r.maxBatchMs}ms) — ` +
|
|
487
|
+
`phases ${Object.entries(r.phases).map(([k, v]) => `${k}=${v}ms`).join(" ")} · ` +
|
|
449
488
|
`telemetry=${r.deletes.agent_telemetry} events=${r.deletes.task_events} ` +
|
|
450
489
|
`inbox=${r.deletes.inbox} tasks=${r.deletes.tasks} audit=${r.deletes.audit_log} ` +
|
|
451
490
|
`nodeReqPurged=${nr.purged} nodeReqDeleted=${nr.deleted} nodeReqTimedOut=${nr.timedOut} ` +
|
package/src/server.ts
CHANGED
|
@@ -77,6 +77,7 @@ import {
|
|
|
77
77
|
import { mkdirSync, writeFileSync, readFileSync, readdirSync, existsSync, statSync, unlinkSync, renameSync } from "fs";
|
|
78
78
|
import { dirname as pathDirname, join as pathJoin } from "path";
|
|
79
79
|
import { startRetentionSweeper } from "./retention.js";
|
|
80
|
+
import { sweepEmptyNodeRows } from "./empty-node-sweeper.js";
|
|
80
81
|
import { startStaleSessionSweeper } from "./stale-sweeper.js";
|
|
81
82
|
import {
|
|
82
83
|
AUDIT_LOG_REST_SELECT,
|
|
@@ -5639,6 +5640,22 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
|
|
|
5639
5640
|
: 60 * 60 * 1000;
|
|
5640
5641
|
const retentionSweeperTimer = startRetentionSweeper(sweepIntervalMs);
|
|
5641
5642
|
|
|
5643
|
+
// #2596 —— 空白 node 行(注册失败留下的空壳:runtime/hostname/config_path 全空、
|
|
5644
|
+
// 没有 session)会让组织架构页出现“重复名字”。启动跑一次 + 每天一次。
|
|
5645
|
+
// 判据写死在 empty-node-sweeper.ts:只删够旧、无身份、无 session 的行;删前把整行 JSON 打进日志。
|
|
5646
|
+
const sweepEmptyNodes = () => {
|
|
5647
|
+
try {
|
|
5648
|
+
const r = sweepEmptyNodeRows();
|
|
5649
|
+
if (r.deleted > 0) {
|
|
5650
|
+
console.log(`[patrol] removed ${r.deleted} empty placeholder node row(s) older than ${r.ageDays}d`);
|
|
5651
|
+
}
|
|
5652
|
+
} catch (e: any) {
|
|
5653
|
+
console.log(`[patrol] empty node sweep failed: ${e?.message ?? e}`);
|
|
5654
|
+
}
|
|
5655
|
+
};
|
|
5656
|
+
setImmediate(sweepEmptyNodes);
|
|
5657
|
+
const emptyNodeGcTimer = setInterval(sweepEmptyNodes, 24 * 60 * 60 * 1000);
|
|
5658
|
+
|
|
5642
5659
|
// #2614 —— 事件循环停顿监测。放这里是因为 startHub() 才是“真的在跑一个 Hub”
|
|
5643
5660
|
// 的入口(测试用 bootServer() 不会背这个 timer)。停顿日志与
|
|
5644
5661
|
// `[commhub retention]` / `[commhub slow-sql]` 共用前缀族,便于一次 grep 出
|
|
@@ -5707,6 +5724,7 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
|
|
|
5707
5724
|
// jobs must not — otherwise a test (or a future caller) that stops the
|
|
5708
5725
|
// server would still hang on live intervals it cannot reach.
|
|
5709
5726
|
(retentionSweeperTimer as any)?.unref?.();
|
|
5727
|
+
(emptyNodeGcTimer as any)?.unref?.();
|
|
5710
5728
|
(eventLoopLagTimer as any)?.unref?.();
|
|
5711
5729
|
(aliasBaselineTimer as any)?.unref?.();
|
|
5712
5730
|
(staleSweeperTimer as any)?.unref?.();
|