@sleep2agi/commhub-server 0.9.0-preview.169 → 0.9.0-preview.171
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/broadcast-fanout-2721.test.ts +20 -1
- package/src/broadcast-fanout.ts +27 -0
- package/src/empty-node-sweeper.test.ts +104 -0
- package/src/empty-node-sweeper.ts +88 -0
- package/src/retention.ts +28 -9
- package/src/server.ts +36 -1
- package/src/version-stats.test.ts +66 -0
- package/src/version-stats.ts +109 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sleep2agi/commhub-server",
|
|
3
|
-
"version": "0.9.0-preview.
|
|
3
|
+
"version": "0.9.0-preview.171",
|
|
4
4
|
"description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
13
13
|
import { db } from "./db.js";
|
|
14
|
-
import { BROADCAST_CHUNK, enqueueBroadcast } from
|
|
14
|
+
import { BROADCAST_CHUNK, enqueueBroadcast, broadcastRecentState } from './broadcast-fanout.js';
|
|
15
15
|
|
|
16
16
|
const stamp = `${Date.now()}`;
|
|
17
17
|
let NET = "";
|
|
@@ -97,3 +97,22 @@ describe("#2721 broadcast fan-out is chunked", () => {
|
|
|
97
97
|
expect(yields).toBe(0);
|
|
98
98
|
});
|
|
99
99
|
});
|
|
100
|
+
|
|
101
|
+
// #2721 —— /health 的 `broadcast` 一格:最近一次广播的时间/人数/批数/耗时。
|
|
102
|
+
describe('broadcastRecentState', () => {
|
|
103
|
+
test('records the last broadcast so a stall can be reconciled afterwards', async () => {
|
|
104
|
+
const targets = Array.from({ length: 7 }, (_, i) => ({ alias: `r${i}`, node_id: null, network_id: 'n1' }));
|
|
105
|
+
await enqueueBroadcast(targets, 'hello', 'hub', 'n1', {
|
|
106
|
+
isActive: () => true,
|
|
107
|
+
notify: () => {},
|
|
108
|
+
yieldBetweenChunks: async () => {},
|
|
109
|
+
chunkSize: 3,
|
|
110
|
+
});
|
|
111
|
+
const rec = broadcastRecentState();
|
|
112
|
+
expect(rec).not.toBeNull();
|
|
113
|
+
expect(rec!.recipients).toBe(7);
|
|
114
|
+
expect(rec!.chunks).toBe(3); // ceil(7/3)
|
|
115
|
+
expect(typeof rec!.took_ms).toBe('number');
|
|
116
|
+
expect(Number.isNaN(Date.parse(rec!.at))).toBe(false);
|
|
117
|
+
});
|
|
118
|
+
});
|
package/src/broadcast-fanout.ts
CHANGED
|
@@ -20,6 +20,26 @@ import { db } from "./db.js";
|
|
|
20
20
|
/** 一批多少个收件人。50 是“一次事务不至于太长、又明显少于一次全量”的折中。 */
|
|
21
21
|
export const BROADCAST_CHUNK = 50;
|
|
22
22
|
|
|
23
|
+
/**
|
|
24
|
+
* #2721 —— 最近一次广播的「事后可对上时间的」记录。
|
|
25
|
+
*
|
|
26
|
+
* 为什么要有:07:11 那次事故里,我花了不少时间才把“143.9 秒停顿”与“07:10 那条广播”
|
|
27
|
+
* 对上 —— 因为停顿记录(`event_loop.recent_stalls`)里只有时间与 RSS,没有“当时发生了什么”。
|
|
28
|
+
* 广播是唯一一种能一次性唤醒几百个 agent 的动作,值得在 `/health` 里留一格。
|
|
29
|
+
*/
|
|
30
|
+
export type BroadcastRecord = {
|
|
31
|
+
at: string;
|
|
32
|
+
recipients: number;
|
|
33
|
+
chunks: number;
|
|
34
|
+
took_ms: number;
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
let lastBroadcast: BroadcastRecord | null = null;
|
|
38
|
+
|
|
39
|
+
export function broadcastRecentState(): BroadcastRecord | null {
|
|
40
|
+
return lastBroadcast;
|
|
41
|
+
}
|
|
42
|
+
|
|
23
43
|
export type BroadcastTarget = { alias: string; node_id: string | null; network_id: string | null };
|
|
24
44
|
export type BroadcastDelivered = { alias: string; netId: string | null };
|
|
25
45
|
|
|
@@ -40,6 +60,7 @@ export async function enqueueBroadcast(
|
|
|
40
60
|
effectiveNetId: string | null,
|
|
41
61
|
deps: BroadcastDeps,
|
|
42
62
|
): Promise<{ ids: string[]; delivered: BroadcastDelivered[] }> {
|
|
63
|
+
const startedMs = Date.now();
|
|
43
64
|
const chunk = deps.chunkSize && deps.chunkSize > 0 ? deps.chunkSize : BROADCAST_CHUNK;
|
|
44
65
|
const ids: string[] = [];
|
|
45
66
|
const delivered: BroadcastDelivered[] = [];
|
|
@@ -67,5 +88,11 @@ export async function enqueueBroadcast(
|
|
|
67
88
|
}
|
|
68
89
|
|
|
69
90
|
for (const d of delivered) deps.notify(d.alias, d.netId);
|
|
91
|
+
lastBroadcast = {
|
|
92
|
+
at: new Date().toISOString(),
|
|
93
|
+
recipients: delivered.length,
|
|
94
|
+
chunks: Math.ceil(targets.length / chunk),
|
|
95
|
+
took_ms: Date.now() - startedMs,
|
|
96
|
+
};
|
|
70
97
|
return { ids, delivered };
|
|
71
98
|
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// #2596 —— 空白 node 行的清理:只删「三身份字段全空 + 无 session + 够旧」的行。
|
|
2
|
+
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
3
|
+
import { db } from "./db.js";
|
|
4
|
+
import { readEmptyNodeAgeDays, sweepEmptyNodeRows } from "./empty-node-sweeper.js";
|
|
5
|
+
|
|
6
|
+
let networkId = "";
|
|
7
|
+
|
|
8
|
+
function insertNode(
|
|
9
|
+
nodeId: string,
|
|
10
|
+
alias: string,
|
|
11
|
+
opts: { identity?: boolean; updatedDaysAgo?: number; createdDaysAgo?: number } = {},
|
|
12
|
+
) {
|
|
13
|
+
const { identity = false, updatedDaysAgo = 10, createdDaysAgo = 30 } = opts;
|
|
14
|
+
db.run(
|
|
15
|
+
`INSERT INTO nodes (node_id, node_name, alias, runtime, hostname, config_path, network_id, created_at, updated_at)
|
|
16
|
+
VALUES (?1, ?2, ?2, ?3, ?4, ?5, ?6, datetime('now', ?8), datetime('now', ?7))`,
|
|
17
|
+
[
|
|
18
|
+
nodeId,
|
|
19
|
+
alias,
|
|
20
|
+
identity ? "codex-app-server-sdk" : null,
|
|
21
|
+
identity ? "host-a" : null,
|
|
22
|
+
identity ? "/data/x" : null,
|
|
23
|
+
networkId,
|
|
24
|
+
`-${updatedDaysAgo} days`,
|
|
25
|
+
`-${createdDaysAgo} days`,
|
|
26
|
+
],
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function exists(nodeId: string): boolean {
|
|
31
|
+
return !!db.get<{ node_id: string }>("SELECT node_id FROM nodes WHERE node_id = ?1", [nodeId]);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
beforeAll(async () => {
|
|
35
|
+
const { register } = await import("./auth.js");
|
|
36
|
+
const auth = register(`empty_gc_${Date.now()}`, "EmptyGc-Str0ng-1!", undefined, "watcher");
|
|
37
|
+
expect(auth.ok).toBe(true);
|
|
38
|
+
networkId = auth.network_id!;
|
|
39
|
+
|
|
40
|
+
insertNode("gc-empty-old", "ghost-a"); // 该删
|
|
41
|
+
insertNode("gc-empty-fresh", "ghost-b", { createdDaysAgo: 0, updatedDaysAgo: 0 }); // 太新:可能正在注册
|
|
42
|
+
insertNode("gc-real", "real-node", { identity: true }); // 有身份:绝不删
|
|
43
|
+
insertNode("gc-real-alias-2", "real-node", { identity: true }); // 不同机器同 alias:也绝不删
|
|
44
|
+
// 空壳但有 session(在线但还没上报身份)⇒ 不删
|
|
45
|
+
insertNode("gc-empty-with-session", "ghost-c");
|
|
46
|
+
db.run(
|
|
47
|
+
`INSERT INTO sessions (resume_id, alias, node_id, network_id, status, registered_at)
|
|
48
|
+
VALUES ('res_gc', 'ghost-c', 'gc-empty-with-session', ?1, 'offline', '2026-10-01 00:00:00')`,
|
|
49
|
+
[networkId],
|
|
50
|
+
);
|
|
51
|
+
}, 30_000);
|
|
52
|
+
|
|
53
|
+
afterAll(() => {
|
|
54
|
+
delete process.env.COMMHUB_EMPTY_NODE_GC_DAYS;
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
describe("sweepEmptyNodeRows (#2596)", () => {
|
|
58
|
+
test("dry run reports the target but deletes nothing", () => {
|
|
59
|
+
const r = sweepEmptyNodeRows({ dryRun: true });
|
|
60
|
+
expect(r.ids).toContain("gc-empty-old");
|
|
61
|
+
expect(r.deleted).toBe(0);
|
|
62
|
+
expect(exists("gc-empty-old")).toBe(true);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test("deletes only the old, identity-less, session-less placeholder", () => {
|
|
66
|
+
const r = sweepEmptyNodeRows();
|
|
67
|
+
expect(exists("gc-empty-old")).toBe(false);
|
|
68
|
+
expect(r.deleted).toBeGreaterThanOrEqual(1);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test("keeps fresh placeholders (they may be mid-registration)", () => {
|
|
72
|
+
sweepEmptyNodeRows();
|
|
73
|
+
expect(exists("gc-empty-fresh")).toBe(true);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
test("keeps rows with an identity, even when two nodes share an alias", () => {
|
|
77
|
+
sweepEmptyNodeRows();
|
|
78
|
+
expect(exists("gc-real")).toBe(true);
|
|
79
|
+
expect(exists("gc-real-alias-2")).toBe(true);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test("keeps a placeholder that still has a session", () => {
|
|
83
|
+
sweepEmptyNodeRows();
|
|
84
|
+
expect(exists("gc-empty-with-session")).toBe(true);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("a fresh bulk update does NOT protect an old empty row (created_at is the clock)", () => {
|
|
88
|
+
// 生产实测:2026-10-26 的一次批量更新把 08/09 月建的空壳 updated_at 全刷成 10-26,
|
|
89
|
+
// 若按 updated_at 判龄,2 天阈值永远删不掉它们。
|
|
90
|
+
insertNode("gc-empty-bulk-touched", "ghost-d", { createdDaysAgo: 40, updatedDaysAgo: 0 });
|
|
91
|
+
sweepEmptyNodeRows();
|
|
92
|
+
expect(exists("gc-empty-bulk-touched")).toBe(false);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("env <= 0 disables the sweep", () => {
|
|
96
|
+
process.env.COMMHUB_EMPTY_NODE_GC_DAYS = "0";
|
|
97
|
+
try {
|
|
98
|
+
expect(readEmptyNodeAgeDays()).toBe(0);
|
|
99
|
+
expect(sweepEmptyNodeRows().deleted).toBe(0);
|
|
100
|
+
} finally {
|
|
101
|
+
delete process.env.COMMHUB_EMPTY_NODE_GC_DAYS;
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
});
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// #2596 —— 组织架构页那些“重复名字”的真身:**注册失败留下的空白 node 行**。
|
|
2
|
+
//
|
|
3
|
+
// 生产实测(2026-10-27,只读):
|
|
4
|
+
// ```
|
|
5
|
+
// nodes 总数 312
|
|
6
|
+
// runtime / hostname / config_path 全为空、且没有任何 session 的行 = 12
|
|
7
|
+
// 形态高度一致:同一 alias 在几分钟内连着出现,最后一个才被填上真实身份
|
|
8
|
+
// TMA门户鲸2号 ×5 09-17 15:47:52 → 16:03:05 (6 分钟注册 5 次,第 6 次成功)
|
|
9
|
+
// ```
|
|
10
|
+
// ⇒ 注册流程先建行、节点再补身份;补不上的尝试留下永久空壳,被组织架构页当成 node 渲染。
|
|
11
|
+
//
|
|
12
|
+
// 这里只做**清理**,不动注册流程:
|
|
13
|
+
// · 只删「三个身份字段全空 + 没有任何 session」的行 —— 这种行不可能提供任何功能;
|
|
14
|
+
// · 只删 `created_at` 早于 N 天(默认 2 天)的行 —— 正在注册中的行自然不会被误删;
|
|
15
|
+
// (用 created_at 不用 updated_at:后者会被无关的批量更新刷新,见下方注释里的生产实例)
|
|
16
|
+
// · 删除前把整行导成 JSON 到日志(可人工恢复),并返回被删的 id 列表。
|
|
17
|
+
//
|
|
18
|
+
// 为什么不用「同 alias 只留一条」这种规则:不同机器共用 alias 是**数据问题不是脏数据**
|
|
19
|
+
// (Mac打包狗 / TM微软牛 / TM汇报牛 这类),删了会丢真实节点。空壳判据才安全。
|
|
20
|
+
|
|
21
|
+
import { db } from "./db.js";
|
|
22
|
+
|
|
23
|
+
export const EMPTY_NODE_AGE_DAYS_ENV = "COMMHUB_EMPTY_NODE_GC_DAYS";
|
|
24
|
+
export const EMPTY_NODE_AGE_DAYS_DEFAULT = 2;
|
|
25
|
+
|
|
26
|
+
export type EmptyNodeSweepResult = {
|
|
27
|
+
deleted: number;
|
|
28
|
+
ids: string[];
|
|
29
|
+
ageDays: number;
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
export function readEmptyNodeAgeDays(raw = process.env[EMPTY_NODE_AGE_DAYS_ENV]): number {
|
|
33
|
+
if (raw === undefined || raw === "") return EMPTY_NODE_AGE_DAYS_DEFAULT;
|
|
34
|
+
const n = Number(raw);
|
|
35
|
+
if (!Number.isFinite(n)) return EMPTY_NODE_AGE_DAYS_DEFAULT;
|
|
36
|
+
return Math.floor(n);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const IDENTITY_EMPTY = `COALESCE(runtime, '') = '' AND COALESCE(hostname, '') = '' AND COALESCE(config_path, '') = ''`;
|
|
40
|
+
|
|
41
|
+
export function sweepEmptyNodeRows(
|
|
42
|
+
opts: { olderThanDays?: number; dryRun?: boolean } = {},
|
|
43
|
+
): EmptyNodeSweepResult {
|
|
44
|
+
const ageDays = opts.olderThanDays ?? readEmptyNodeAgeDays();
|
|
45
|
+
// <=0 关闭(运维可以彻底停掉这个清理)
|
|
46
|
+
if (ageDays <= 0) return { deleted: 0, ids: [], ageDays: 0 };
|
|
47
|
+
|
|
48
|
+
const rows = db.all<{ node_id: string }>(
|
|
49
|
+
`SELECT n.node_id
|
|
50
|
+
FROM nodes n
|
|
51
|
+
WHERE ${IDENTITY_EMPTY.replace(/\b(runtime|hostname|config_path)\b/g, "n.$1")}
|
|
52
|
+
-- #2596 —— 用 created_at 而不是 updated_at:空壳行会被无关的批量更新
|
|
53
|
+
-- 把 updated_at 顶到「今天」(生产实测:2026-10-26 的一次批量更新把 08/09 月
|
|
54
|
+
-- 建的空壳全部刷成 10-26),那样 2 天的阈值就永远删不掉它们。
|
|
55
|
+
-- created_at 才是“这行空了多久”的稳定量;在注册中的行 created_at 必然是新的。
|
|
56
|
+
AND n.created_at < datetime('now', ?1)
|
|
57
|
+
AND NOT EXISTS (SELECT 1 FROM sessions s WHERE s.node_id = n.node_id)
|
|
58
|
+
LIMIT 500`,
|
|
59
|
+
`-${ageDays} days`,
|
|
60
|
+
);
|
|
61
|
+
if (rows.length === 0) return { deleted: 0, ids: [], ageDays };
|
|
62
|
+
|
|
63
|
+
const ids = rows.map((r) => r.node_id);
|
|
64
|
+
if (opts.dryRun) return { deleted: 0, ids, ageDays };
|
|
65
|
+
|
|
66
|
+
// 删前留证:整行 JSON 打到日志里(运维可据此恢复),再逐条删。
|
|
67
|
+
try {
|
|
68
|
+
const full = db.all<Record<string, unknown>>(
|
|
69
|
+
`SELECT * FROM nodes WHERE node_id IN (${ids.map((_, i) => `?${i + 1}`).join(",")})`,
|
|
70
|
+
ids,
|
|
71
|
+
);
|
|
72
|
+
console.log(`[commhub empty-node-gc] deleting ${ids.length} placeholder node row(s): ${JSON.stringify(full)}`);
|
|
73
|
+
} catch (e: any) {
|
|
74
|
+
console.log(`[commhub empty-node-gc] backup dump failed, aborting: ${e?.message ?? e}`);
|
|
75
|
+
return { deleted: 0, ids: [], ageDays };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
let deleted = 0;
|
|
79
|
+
for (const id of ids) {
|
|
80
|
+
try {
|
|
81
|
+
const res = db.run(`DELETE FROM nodes WHERE node_id = ?1 AND ${IDENTITY_EMPTY}`, [id]);
|
|
82
|
+
deleted += res.changes ?? 0;
|
|
83
|
+
} catch (e: any) {
|
|
84
|
+
console.log(`[commhub empty-node-gc] delete ${id} failed: ${e?.message ?? e}`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return { deleted, ids, ageDays };
|
|
88
|
+
}
|
package/src/retention.ts
CHANGED
|
@@ -63,6 +63,8 @@ export type SweepResult = {
|
|
|
63
63
|
batches: number;
|
|
64
64
|
/** 整轮里单批最长耗时(毫秒)。生产上这条数字 = 「一次 sweep 最长阻塞多久」。 */
|
|
65
65
|
maxBatchMs: number;
|
|
66
|
+
/** 各阶段耗时(毫秒):多张表 + vacuum。生产上「15 秒里谁花的」靠它,不靠猜。 */
|
|
67
|
+
phases: Record<string, number>;
|
|
66
68
|
truncated: string[];
|
|
67
69
|
nodeRequests: NodeRequestSweepResult;
|
|
68
70
|
vacuum: {
|
|
@@ -125,6 +127,8 @@ export function readRetentionConfig(): RetentionConfig {
|
|
|
125
127
|
// ⇒ 教训:**副本的 schema 也必须与生产一致**,否则「按副本优化」会变成新的全表扫。
|
|
126
128
|
// 总耗时略增是有意的取舍:**可被打断优先于最快跑完**。
|
|
127
129
|
const DEFAULT_CHUNK_ROWS = 2000;
|
|
130
|
+
/** 每次 sweep 最多回收多少空闲页(≈4 KB/页)。见下面的 incremental_vacuum 注释。 */
|
|
131
|
+
const INCREMENTAL_VACUUM_PAGES = 200;
|
|
128
132
|
const MAX_TABLE_MS = 10 * 60 * 1000;
|
|
129
133
|
|
|
130
134
|
function readChunkRows(): number {
|
|
@@ -163,6 +167,8 @@ function yieldToLoop(): Promise<void> {
|
|
|
163
167
|
export type TableSweepResult = {
|
|
164
168
|
deleted: number;
|
|
165
169
|
batches: number;
|
|
170
|
+
/** 这张表这轮花了多久(毫秒)—— 生产上排查「15 秒里到底谁花的」靠它。 */
|
|
171
|
+
ms: number;
|
|
166
172
|
/** 单批最长耗时(毫秒)—— 生产上「一次 sweep 最长阻塞多久」就是这个数。 */
|
|
167
173
|
maxBatchMs: number;
|
|
168
174
|
/** 单表跑满 maxTableMs 被截断;剩余行下一轮继续(不吞掉、不假装完成)。 */
|
|
@@ -179,20 +185,21 @@ async function sweepTable(
|
|
|
179
185
|
// days < 0 → operator opt-out for this table; skip the DELETE.
|
|
180
186
|
// days === 0 is a legitimate "sweep everything older than now"
|
|
181
187
|
// (effectively a TRUNCATE); we honour it.
|
|
182
|
-
if (days < 0) return { deleted: 0, batches: 0, maxBatchMs: 0, truncated: false };
|
|
188
|
+
if (days < 0) return { deleted: 0, batches: 0, maxBatchMs: 0, ms: 0, truncated: false };
|
|
183
189
|
// PostgreSQL: 单条 DELETE + autovacuum 本来就按批回收,且报文里报的 10 秒停顿
|
|
184
190
|
// 出在 SQLite 侧(`#2614` 的生产现场是 SQLite)。这里不引入 rowid(pg 没有),
|
|
185
191
|
// 保持原样,避免在没观测到问题的方言上改语义。
|
|
186
192
|
if (db.dialect !== "sqlite") {
|
|
187
193
|
try {
|
|
188
194
|
const res = db.run(`DELETE FROM ${table} WHERE ${where}`, [param]);
|
|
189
|
-
return { deleted: res.changes ?? 0, batches: 1, maxBatchMs: 0, truncated: false };
|
|
195
|
+
return { deleted: res.changes ?? 0, batches: 1, maxBatchMs: 0, ms: 0, truncated: false };
|
|
190
196
|
} catch (e: any) {
|
|
191
197
|
console.log(`[commhub retention] sweep failed (${table}): ${e?.message ?? e}`);
|
|
192
|
-
return { deleted: 0, batches: 0, maxBatchMs: 0, truncated: false };
|
|
198
|
+
return { deleted: 0, batches: 0, maxBatchMs: 0, ms: 0, truncated: false };
|
|
193
199
|
}
|
|
194
200
|
}
|
|
195
201
|
const startedMs = Date.now();
|
|
202
|
+
const tTable0 = performance.now();
|
|
196
203
|
let deleted = 0;
|
|
197
204
|
let batches = 0;
|
|
198
205
|
let maxBatchMs = 0;
|
|
@@ -214,7 +221,7 @@ async function sweepTable(
|
|
|
214
221
|
param,
|
|
215
222
|
chunkRows,
|
|
216
223
|
);
|
|
217
|
-
if (rows.length === 0) return { deleted, batches, maxBatchMs, truncated: false };
|
|
224
|
+
if (rows.length === 0) return { deleted, batches, maxBatchMs, ms: Math.round(performance.now() - tTable0), truncated: false };
|
|
218
225
|
const ids = rows.map((r) => r.r);
|
|
219
226
|
const placeholders = ids.map((_, i) => `?${i + 1}`).join(",");
|
|
220
227
|
const res = db.run(`DELETE FROM ${table} WHERE rowid IN (${placeholders})`, ids);
|
|
@@ -222,20 +229,20 @@ async function sweepTable(
|
|
|
222
229
|
batches += 1;
|
|
223
230
|
maxBatchMs = Math.max(maxBatchMs, performance.now() - batchT0);
|
|
224
231
|
// 这一批没取满 ⇒ 已经没有更多符合条件的行了,收工。
|
|
225
|
-
if (rows.length < chunkRows) return { deleted, batches, maxBatchMs, truncated: false };
|
|
232
|
+
if (rows.length < chunkRows) return { deleted, batches, maxBatchMs, ms: Math.round(performance.now() - tTable0), truncated: false };
|
|
226
233
|
if (Date.now() - startedMs > MAX_TABLE_MS) {
|
|
227
234
|
console.log(
|
|
228
235
|
`[commhub retention] ${table}: stopped after ${batches} batches / ${deleted} rows ` +
|
|
229
236
|
`(>${Math.round(MAX_TABLE_MS / 60000)}min); remaining rows continue next sweep`,
|
|
230
237
|
);
|
|
231
|
-
return { deleted, batches, maxBatchMs, truncated: true };
|
|
238
|
+
return { deleted, batches, maxBatchMs, ms: Math.round(performance.now() - tTable0), truncated: true };
|
|
232
239
|
}
|
|
233
240
|
// 让出事件循环:这是 #2614 的全部要点 —— 停顿的可感知单位是「单批」。
|
|
234
241
|
await yieldToLoop();
|
|
235
242
|
}
|
|
236
243
|
} catch (e: any) {
|
|
237
244
|
console.log(`[commhub retention] sweep failed (${table}): ${e?.message ?? e}`);
|
|
238
|
-
return { deleted, batches, maxBatchMs, truncated: false };
|
|
245
|
+
return { deleted, batches, maxBatchMs, ms: Math.round(performance.now() - tTable0), truncated: false };
|
|
239
246
|
}
|
|
240
247
|
}
|
|
241
248
|
|
|
@@ -398,7 +405,11 @@ export async function sweepRetention(cfg: RetentionConfig = readRetentionConfig(
|
|
|
398
405
|
// weren't, the operator would have to run a full one-time VACUUM
|
|
399
406
|
// to migrate — out of scope for the sweeper.
|
|
400
407
|
const freelistBefore = db.get<{ freelist_count: number }>("PRAGMA freelist_count");
|
|
401
|
-
|
|
408
|
+
// #2614 —— 必须带页数上限。`PRAGMA incremental_vacuum`(不带参数)会**一次回收
|
|
409
|
+
// 全部空闲页**:生产 `.169` 上线时那一拍整轮 15.3s,其中大部分是这个语句
|
|
410
|
+
// (它走 db.exec,不进 slow-sql 日志,所以第一眼看不见)。
|
|
411
|
+
// 每小时回 200 页(≈800 KB)足够跟上增长,又不会变成一次长阻塞;剩下的下次继续。
|
|
412
|
+
db.exec(`PRAGMA incremental_vacuum(${INCREMENTAL_VACUUM_PAGES})`);
|
|
402
413
|
const freelistAfter = db.get<{ freelist_count: number }>("PRAGMA freelist_count");
|
|
403
414
|
incrementalFreedPages =
|
|
404
415
|
(freelistBefore?.freelist_count ?? 0) - (freelistAfter?.freelist_count ?? 0);
|
|
@@ -422,6 +433,13 @@ export async function sweepRetention(cfg: RetentionConfig = readRetentionConfig(
|
|
|
422
433
|
telemetry.batches + taskEvents.batches + inbox.batches +
|
|
423
434
|
tasksSafeTerminal.batches + tasksRepliedChildSafe.batches + auditLog.batches,
|
|
424
435
|
// 生产上「一次 sweep 最长阻塞多久」= 这个数字(#2614 的验收看它)
|
|
436
|
+
phases: {
|
|
437
|
+
telemetry: telemetry.ms,
|
|
438
|
+
events: taskEvents.ms,
|
|
439
|
+
inbox: inbox.ms,
|
|
440
|
+
tasks: tasksSafeTerminal.ms + tasksRepliedChildSafe.ms,
|
|
441
|
+
audit: auditLog.ms,
|
|
442
|
+
},
|
|
425
443
|
maxBatchMs: Math.round(Math.max(
|
|
426
444
|
telemetry.maxBatchMs, taskEvents.maxBatchMs, inbox.maxBatchMs,
|
|
427
445
|
tasksSafeTerminal.maxBatchMs, tasksRepliedChildSafe.maxBatchMs, auditLog.maxBatchMs,
|
|
@@ -465,7 +483,8 @@ export function startRetentionSweeper(
|
|
|
465
483
|
const nr = r.nodeRequests;
|
|
466
484
|
if (totalDeletes > 0 || nr.purged > 0 || nr.deleted > 0 || nr.timedOut > 0 || r.vacuum.errored) {
|
|
467
485
|
console.log(
|
|
468
|
-
`[commhub retention] swept in ${r.durationMs}ms (${r.batches} batches) — ` +
|
|
486
|
+
`[commhub retention] swept in ${r.durationMs}ms (${r.batches} batches, max ${r.maxBatchMs}ms) — ` +
|
|
487
|
+
`phases ${Object.entries(r.phases).map(([k, v]) => `${k}=${v}ms`).join(" ")} · ` +
|
|
469
488
|
`telemetry=${r.deletes.agent_telemetry} events=${r.deletes.task_events} ` +
|
|
470
489
|
`inbox=${r.deletes.inbox} tasks=${r.deletes.tasks} audit=${r.deletes.audit_log} ` +
|
|
471
490
|
`nodeReqPurged=${nr.purged} nodeReqDeleted=${nr.deleted} nodeReqTimedOut=${nr.timedOut} ` +
|
package/src/server.ts
CHANGED
|
@@ -31,7 +31,7 @@ import { createSSEStream, createNetworkObserverStream, createUserEventStream, pu
|
|
|
31
31
|
import { enterMcpInflight, leaveMcpInflight, getMcpInflightStats } from "./mcp-inflight.js";
|
|
32
32
|
import { eventLoopStats, startEventLoopLagMonitor } from "./event-loop-lag.js";
|
|
33
33
|
import { aliasBaselineRefreshState, refreshAliasBaseline } from "./alias-baseline.js";
|
|
34
|
-
import { enqueueBroadcast } from "./broadcast-fanout.js";
|
|
34
|
+
import { broadcastRecentState, enqueueBroadcast } from "./broadcast-fanout.js";
|
|
35
35
|
import { assertNodeActive } from "./lifecycle-guard.js";
|
|
36
36
|
import { pendingInboxCount } from "./inbox-count.js";
|
|
37
37
|
import { notifyExpiredTasks, type ExpiredTaskRow } from "./task-expiry-notice.js";
|
|
@@ -77,6 +77,8 @@ import {
|
|
|
77
77
|
import { mkdirSync, writeFileSync, readFileSync, readdirSync, existsSync, statSync, unlinkSync, renameSync } from "fs";
|
|
78
78
|
import { dirname as pathDirname, join as pathJoin } from "path";
|
|
79
79
|
import { startRetentionSweeper } from "./retention.js";
|
|
80
|
+
import { sweepEmptyNodeRows } from "./empty-node-sweeper.js";
|
|
81
|
+
import { versionStats } from "./version-stats.js";
|
|
80
82
|
import { startStaleSessionSweeper } from "./stale-sweeper.js";
|
|
81
83
|
import {
|
|
82
84
|
AUDIT_LOG_REST_SELECT,
|
|
@@ -2813,6 +2815,9 @@ return Bun.serve({
|
|
|
2813
2815
|
// #2571 —— 告警阈值用的「每目标 p99 基线」最近一次刷新(只读聚合:时间/写了几行/现在共几行)。
|
|
2814
2816
|
// 没有它,「这条告警为什么没报」只能进机器 sqlite 查。
|
|
2815
2817
|
alert_baselines: aliasBaselineRefreshState(),
|
|
2818
|
+
// #2721 —— 最近一次广播(时间/收件人数/批数/耗时)。广播是唯一能一次性唤醒
|
|
2819
|
+
// 几百个 agent 的动作;没有这一格,事后只能把停顿记录和聊天记录对时间。
|
|
2820
|
+
broadcast: broadcastRecentState(),
|
|
2816
2821
|
auth: DEV_OPEN ? "dev-open" : "user-token",
|
|
2817
2822
|
security: DEV_OPEN ? "dev-open" : "secured",
|
|
2818
2823
|
tmux: TMUX_ENABLED ? "enabled" : "disabled",
|
|
@@ -3188,6 +3193,19 @@ return Bun.serve({
|
|
|
3188
3193
|
return withCors(req, Response.json({ ok: true, ...routeStats(minutes * 60_000, Date.now(), { byCaller }) }));
|
|
3189
3194
|
}
|
|
3190
3195
|
|
|
3196
|
+
// ── #2720 REST: 版本债务(只读聚合)──
|
|
3197
|
+
// 为什么要有这一条:370 个 session 里 159 个没有版本号,而「没有版本」不是一个桶 ——
|
|
3198
|
+
// 24 个是 claude-code(LLM 会话自己调工具,没有版本概念),9 个才是老 agent-node。
|
|
3199
|
+
// 把两者混算会得出“升级节点能修好 33 个”的错误结论(只能修好 9 个)。
|
|
3200
|
+
// 返回聚合数字,不含 alias 明细;节点令牌只看自己绑定的网络。
|
|
3201
|
+
if (url.pathname === "/api/stats/versions" && req.method === "GET") {
|
|
3202
|
+
if (restAuth && !isAdmin && !restAuth.networkId) {
|
|
3203
|
+
return withCors(req, Response.json({ ok: false, error: "admin or network token required" }, { status: 403 }));
|
|
3204
|
+
}
|
|
3205
|
+
const scopeNet = restAuth?.networkId ?? null;
|
|
3206
|
+
return withCors(req, Response.json({ ok: true, ...versionStats(scopeNet) }));
|
|
3207
|
+
}
|
|
3208
|
+
|
|
3191
3209
|
// ── #473 REST: SSE connection detail (ops-only) ──
|
|
3192
3210
|
// The per-key breakdown /health used to expose anonymously: keys are
|
|
3193
3211
|
// `{networkId}:{alias}` (+ `\0netobs:{networkId}` observer keys via
|
|
@@ -5639,6 +5657,22 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
|
|
|
5639
5657
|
: 60 * 60 * 1000;
|
|
5640
5658
|
const retentionSweeperTimer = startRetentionSweeper(sweepIntervalMs);
|
|
5641
5659
|
|
|
5660
|
+
// #2596 —— 空白 node 行(注册失败留下的空壳:runtime/hostname/config_path 全空、
|
|
5661
|
+
// 没有 session)会让组织架构页出现“重复名字”。启动跑一次 + 每天一次。
|
|
5662
|
+
// 判据写死在 empty-node-sweeper.ts:只删够旧、无身份、无 session 的行;删前把整行 JSON 打进日志。
|
|
5663
|
+
const sweepEmptyNodes = () => {
|
|
5664
|
+
try {
|
|
5665
|
+
const r = sweepEmptyNodeRows();
|
|
5666
|
+
if (r.deleted > 0) {
|
|
5667
|
+
console.log(`[patrol] removed ${r.deleted} empty placeholder node row(s) older than ${r.ageDays}d`);
|
|
5668
|
+
}
|
|
5669
|
+
} catch (e: any) {
|
|
5670
|
+
console.log(`[patrol] empty node sweep failed: ${e?.message ?? e}`);
|
|
5671
|
+
}
|
|
5672
|
+
};
|
|
5673
|
+
setImmediate(sweepEmptyNodes);
|
|
5674
|
+
const emptyNodeGcTimer = setInterval(sweepEmptyNodes, 24 * 60 * 60 * 1000);
|
|
5675
|
+
|
|
5642
5676
|
// #2614 —— 事件循环停顿监测。放这里是因为 startHub() 才是“真的在跑一个 Hub”
|
|
5643
5677
|
// 的入口(测试用 bootServer() 不会背这个 timer)。停顿日志与
|
|
5644
5678
|
// `[commhub retention]` / `[commhub slow-sql]` 共用前缀族,便于一次 grep 出
|
|
@@ -5707,6 +5741,7 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
|
|
|
5707
5741
|
// jobs must not — otherwise a test (or a future caller) that stops the
|
|
5708
5742
|
// server would still hang on live intervals it cannot reach.
|
|
5709
5743
|
(retentionSweeperTimer as any)?.unref?.();
|
|
5744
|
+
(emptyNodeGcTimer as any)?.unref?.();
|
|
5710
5745
|
(eventLoopLagTimer as any)?.unref?.();
|
|
5711
5746
|
(aliasBaselineTimer as any)?.unref?.();
|
|
5712
5747
|
(staleSweeperTimer as any)?.unref?.();
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
// #2720 —— 版本债务聚合:无版本要按 agent 拆,别把「没有版本概念」算成债务。
|
|
2
|
+
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
3
|
+
import { db } from "./db.js";
|
|
4
|
+
import { versionStats } from "./version-stats.js";
|
|
5
|
+
|
|
6
|
+
let networkId = "";
|
|
7
|
+
|
|
8
|
+
function insertSession(alias: string, agent: string, version: string | null, status: string) {
|
|
9
|
+
db.run(
|
|
10
|
+
`INSERT INTO sessions (resume_id, alias, agent, version, status, network_id, registered_at)
|
|
11
|
+
VALUES (?1, ?2, ?3, ?4, ?5, ?6, '2026-10-01 00:00:00')`,
|
|
12
|
+
[`res_${alias}`, alias, agent, version, status, networkId],
|
|
13
|
+
);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
beforeAll(async () => {
|
|
17
|
+
const { register } = await import("./auth.js");
|
|
18
|
+
const auth = register(`verstats_${Date.now()}`, "VerStats-Str0ng-1!", undefined, "watcher");
|
|
19
|
+
expect(auth.ok).toBe(true);
|
|
20
|
+
networkId = auth.network_id!;
|
|
21
|
+
|
|
22
|
+
insertSession("n-new", "agent-node:opencode", "2.3.0-preview.199", "idle");
|
|
23
|
+
insertSession("n-old", "agent-node:opencode", "2.3.0-preview.150", "idle");
|
|
24
|
+
insertSession("n-old-2", "agent-node:opencode", "2.3.0-preview.150", "working");
|
|
25
|
+
insertSession("n-gone", "agent-node:opencode", "2.3.0-preview.150", "offline");
|
|
26
|
+
// LLM 自己调工具的会话:没有版本是 n/a,不是债务
|
|
27
|
+
insertSession("cc-1", "claude-code", null, "idle");
|
|
28
|
+
insertSession("cc-2", "claude-code", null, "offline");
|
|
29
|
+
// 老 agent-node:该报没报,是真债务
|
|
30
|
+
insertSession("old-node-1", "agent-node:claude", null, "idle");
|
|
31
|
+
}, 30_000);
|
|
32
|
+
|
|
33
|
+
afterAll(() => {});
|
|
34
|
+
|
|
35
|
+
describe("versionStats (#2720)", () => {
|
|
36
|
+
test("counts sessions and active sessions for the network", () => {
|
|
37
|
+
const s = versionStats(networkId);
|
|
38
|
+
expect(s.sessions).toBeGreaterThanOrEqual(7);
|
|
39
|
+
expect(s.active).toBeGreaterThanOrEqual(5);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test("keeps 'no version' split by agent, with version_expected flag", () => {
|
|
43
|
+
const s = versionStats(networkId);
|
|
44
|
+
const cc = s.no_version_by_agent.find((x) => x.agent === "claude-code");
|
|
45
|
+
const oldNode = s.no_version_by_agent.find((x) => x.agent === "agent-node:claude");
|
|
46
|
+
expect(cc?.version_expected).toBe(false); // LLM 工具调用:n/a
|
|
47
|
+
expect(oldNode?.version_expected).toBe(true); // 老节点:真债务
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test("active_missing_version excludes producers with no version concept", () => {
|
|
51
|
+
const s = versionStats(networkId);
|
|
52
|
+
// 只有 old-node-1 计入(cc-1 是 n/a)
|
|
53
|
+
expect(s.active_missing_version).toBe(1);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test("active_behind_same_agent counts actives on a non-top version", () => {
|
|
57
|
+
const s = versionStats(networkId);
|
|
58
|
+
// .150 有 2 个在线(idle+working),.199 是最高的一个
|
|
59
|
+
expect(s.active_behind_same_agent).toBeGreaterThanOrEqual(2);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test("scoping by an unknown network yields nothing", () => {
|
|
63
|
+
const s = versionStats("net_does_not_exist");
|
|
64
|
+
expect(s.sessions).toBe(0);
|
|
65
|
+
});
|
|
66
|
+
});
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
// #2720 —— 让「版本债务」自己更新,不靠人跑 SQL。
|
|
2
|
+
//
|
|
3
|
+
// 背景(2026-10-27 生产只读统计):370 个 session 里 159 个没有版本号,
|
|
4
|
+
// 其中 33 个**还在线**。继续拆下去才看清真相:
|
|
5
|
+
// · 24 个的 `agent` 是 `claude-code` —— 那是 Claude 会话自己调
|
|
6
|
+
// `commhub_report_status(status=…, task=…)` 写的,**它没有 version 这个概念**;
|
|
7
|
+
// · 剩下 9 个才是老 agent-node(#1809 之前每次上报不带 version)。
|
|
8
|
+
// ⇒ 「无版本」不是一个桶:对 `claude-code` 是 n/a,对 agent-node 才是债务。
|
|
9
|
+
// 把两者混在一起算,会得出「升级节点能修好 33 个」的错误结论(只能修好 9 个)。
|
|
10
|
+
//
|
|
11
|
+
// 这个模块只做聚合(只读),返回:
|
|
12
|
+
// by_version —— (agent, version) → 总数/在线数
|
|
13
|
+
// no_version —— 按 agent 拆开的「无版本」,让调用方自己判 n/a 还是债务
|
|
14
|
+
// agent_kind —— 该 agent 的上报方式(`llm-tool-call` = 不自带版本)
|
|
15
|
+
|
|
16
|
+
import { db } from "./db.js";
|
|
17
|
+
|
|
18
|
+
/** 这些 agent 的上报者是 LLM 会话自己调工具,没有打包版本号可报(不是缺字段)。 */
|
|
19
|
+
const LLM_TOOL_CALL_AGENTS = new Set(["claude-code"]);
|
|
20
|
+
|
|
21
|
+
export type VersionBucket = {
|
|
22
|
+
agent: string;
|
|
23
|
+
version: string | null;
|
|
24
|
+
sessions: number;
|
|
25
|
+
active: number;
|
|
26
|
+
/** false = 这个 agent 没有版本概念(n/a),而不是“该报未报”。 */
|
|
27
|
+
version_expected: boolean;
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
export type VersionStats = {
|
|
31
|
+
network_id: string | null;
|
|
32
|
+
sessions: number;
|
|
33
|
+
active: number;
|
|
34
|
+
by_version: VersionBucket[];
|
|
35
|
+
no_version_by_agent: Array<{ agent: string; sessions: number; version_expected: boolean }>;
|
|
36
|
+
/** 在线且「该报版本却没报」的总数 —— 这才是真正的债务口径。 */
|
|
37
|
+
active_missing_version: number;
|
|
38
|
+
/** 在线且版本落后于「在线里最高版本」的数量(只对字符串完全可比的同 agent 生效)。 */
|
|
39
|
+
active_behind_same_agent: number;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
export function versionStats(networkId: string | null = null): VersionStats {
|
|
43
|
+
const rows = db.all<{ agent: string | null; version: string | null; status: string | null; c: number }>(
|
|
44
|
+
`SELECT COALESCE(agent, '') AS agent,
|
|
45
|
+
COALESCE(NULLIF(version, ''), '') AS version,
|
|
46
|
+
COALESCE(status, '') AS status,
|
|
47
|
+
COUNT(*) AS c
|
|
48
|
+
FROM sessions
|
|
49
|
+
WHERE (?1 IS NULL OR network_id = ?1)
|
|
50
|
+
GROUP BY 1, 2, 3`,
|
|
51
|
+
networkId,
|
|
52
|
+
);
|
|
53
|
+
|
|
54
|
+
const buckets = new Map<string, VersionBucket>();
|
|
55
|
+
let sessions = 0;
|
|
56
|
+
let active = 0;
|
|
57
|
+
for (const r of rows) {
|
|
58
|
+
const isActive = r.status !== "offline";
|
|
59
|
+
sessions += r.c;
|
|
60
|
+
if (isActive) active += r.c;
|
|
61
|
+
const version = r.version === "" ? null : r.version;
|
|
62
|
+
const key = `${r.agent}\u0000${version ?? ""}`;
|
|
63
|
+
const cur = buckets.get(key) ?? {
|
|
64
|
+
agent: r.agent,
|
|
65
|
+
version,
|
|
66
|
+
sessions: 0,
|
|
67
|
+
active: 0,
|
|
68
|
+
version_expected: !LLM_TOOL_CALL_AGENTS.has(r.agent),
|
|
69
|
+
};
|
|
70
|
+
cur.sessions += r.c;
|
|
71
|
+
if (isActive) cur.active += r.c;
|
|
72
|
+
buckets.set(key, cur);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const all = [...buckets.values()].sort((a, b) => b.sessions - a.sessions || a.agent.localeCompare(b.agent));
|
|
76
|
+
|
|
77
|
+
const noVersion = all
|
|
78
|
+
.filter((b) => b.version === null)
|
|
79
|
+
.map((b) => ({ agent: b.agent, sessions: b.sessions, version_expected: b.version_expected }));
|
|
80
|
+
|
|
81
|
+
// 每个 agent 在「在线」行里出现过的最高版本(仅用于粗判落后;不做语义版本比较,
|
|
82
|
+
// 只在字符串完全一致时才算“同一个版本”)。
|
|
83
|
+
const highest = new Map<string, string>();
|
|
84
|
+
for (const b of all) {
|
|
85
|
+
if (!b.version || b.active === 0) continue;
|
|
86
|
+
const cur = highest.get(b.agent);
|
|
87
|
+
if (!cur || b.version.localeCompare(cur) > 0) highest.set(b.agent, b.version);
|
|
88
|
+
}
|
|
89
|
+
let behind = 0;
|
|
90
|
+
for (const b of all) {
|
|
91
|
+
if (!b.version || b.active === 0) continue;
|
|
92
|
+
const top = highest.get(b.agent);
|
|
93
|
+
if (top && b.version !== top) behind += b.active;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const activeMissingVersion = all
|
|
97
|
+
.filter((b) => b.version === null && b.version_expected)
|
|
98
|
+
.reduce((n, b) => n + b.active, 0);
|
|
99
|
+
|
|
100
|
+
return {
|
|
101
|
+
network_id: networkId,
|
|
102
|
+
sessions,
|
|
103
|
+
active,
|
|
104
|
+
by_version: all,
|
|
105
|
+
no_version_by_agent: noVersion,
|
|
106
|
+
active_missing_version: activeMissingVersion,
|
|
107
|
+
active_behind_same_agent: behind,
|
|
108
|
+
};
|
|
109
|
+
}
|