@sleep2agi/commhub-server 0.9.0-preview.161 → 0.9.0-preview.162
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/db-adapter.ts +54 -4
- package/src/db.ts +7 -0
- package/src/event-loop-lag.test.ts +127 -0
- package/src/event-loop-lag.ts +122 -0
- package/src/requirements-people-mcp-http.test.ts +9 -3
- package/src/requirements-people.ts +26 -4
- package/src/server.ts +14 -0
- package/src/task-lifecycle-watcher-busy.test.ts +14 -14
- package/src/task-lifecycle-watcher.test.ts +24 -24
- package/src/task-lifecycle-watcher.ts +25 -7
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sleep2agi/commhub-server",
|
|
3
|
-
"version": "0.9.0-preview.
|
|
3
|
+
"version": "0.9.0-preview.162",
|
|
4
4
|
"description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
package/src/db-adapter.ts
CHANGED
|
@@ -61,6 +61,36 @@ export const PG_EXPERIMENTAL_ENV = "COMMHUB_PG_EXPERIMENTAL";
|
|
|
61
61
|
// SQLite Adapter (bun:sqlite, sync)
|
|
62
62
|
// ════════════════════════════════════════════
|
|
63
63
|
|
|
64
|
+
// #2614 —— 慢 SQL 观测。SQLite 是**同步**的:一条慢查询会直接卡住整个事件循环
|
|
65
|
+
// (生产实测 2026-10-26 每小时一次的 8–11 秒停顿就是 retention 的全表扫描,
|
|
66
|
+
// 症状是日志静默 + 在飞数暴涨 + RSS 冲到 4.5 GB)。之前没有任何一条日志能指出
|
|
67
|
+
// “是这条 SQL 卡了 11 秒”,只能靠事后比对日志静默的时间窗手工推断。
|
|
68
|
+
//
|
|
69
|
+
// 这里只在**超过阈值**时打一行日志,不含参数值(避免把令牌/正文写进日志)。
|
|
70
|
+
// `COMMHUB_SLOW_SQL_MS` 控制阈值(默认 1000 ms,设 0 关闭)。
|
|
71
|
+
const DEFAULT_SLOW_SQL_MS = 1000;
|
|
72
|
+
|
|
73
|
+
export function slowSqlThresholdMs(): number {
|
|
74
|
+
const raw = Number(process.env.COMMHUB_SLOW_SQL_MS);
|
|
75
|
+
return Number.isFinite(raw) && raw >= 0 ? raw : DEFAULT_SLOW_SQL_MS;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** 截断 + 压平 SQL,只保留足够定位的头部(不包含绑定参数)。 */
|
|
79
|
+
export function formatSlowSql(ms: number, sql: string): string {
|
|
80
|
+
const oneLine = sql.replace(/\s+/g, " ").trim().slice(0, 160);
|
|
81
|
+
return `[commhub slow-sql] ${ms.toFixed(0)}ms ${oneLine}`;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
let sqlTimingDisabled: boolean | null = null;
|
|
85
|
+
|
|
86
|
+
function logIfSlow(started: number, sql: string): void {
|
|
87
|
+
if (sqlTimingDisabled === null) sqlTimingDisabled = slowSqlThresholdMs() <= 0;
|
|
88
|
+
if (sqlTimingDisabled) return;
|
|
89
|
+
const ms = performance.now() - started;
|
|
90
|
+
if (ms < slowSqlThresholdMs()) return;
|
|
91
|
+
console.log(formatSlowSql(ms, sql));
|
|
92
|
+
}
|
|
93
|
+
|
|
64
94
|
export class SQLiteAdapter implements DbAdapter {
|
|
65
95
|
readonly dialect = "sqlite" as const;
|
|
66
96
|
readonly atomicTransactions = true;
|
|
@@ -70,19 +100,39 @@ export class SQLiteAdapter implements DbAdapter {
|
|
|
70
100
|
constructor(private readonly rawDb: Database) {}
|
|
71
101
|
|
|
72
102
|
run(sql: string, params?: any[]): QueryResult {
|
|
73
|
-
|
|
103
|
+
const t0 = performance.now();
|
|
104
|
+
try {
|
|
105
|
+
return params ? this.rawDb.run(sql, params as any) : this.rawDb.run(sql);
|
|
106
|
+
} finally {
|
|
107
|
+
logIfSlow(t0, sql);
|
|
108
|
+
}
|
|
74
109
|
}
|
|
75
110
|
|
|
76
111
|
get<T = any>(sql: string, ...params: any[]): T | null {
|
|
77
|
-
|
|
112
|
+
const t0 = performance.now();
|
|
113
|
+
try {
|
|
114
|
+
return this.rawDb.query<T, any[]>(sql).get(...params) ?? null;
|
|
115
|
+
} finally {
|
|
116
|
+
logIfSlow(t0, sql);
|
|
117
|
+
}
|
|
78
118
|
}
|
|
79
119
|
|
|
80
120
|
all<T = any>(sql: string, ...params: any[]): T[] {
|
|
81
|
-
|
|
121
|
+
const t0 = performance.now();
|
|
122
|
+
try {
|
|
123
|
+
return this.rawDb.query<T, any[]>(sql).all(...params);
|
|
124
|
+
} finally {
|
|
125
|
+
logIfSlow(t0, sql);
|
|
126
|
+
}
|
|
82
127
|
}
|
|
83
128
|
|
|
84
129
|
exec(sql: string): void {
|
|
85
|
-
|
|
130
|
+
const t0 = performance.now();
|
|
131
|
+
try {
|
|
132
|
+
this.rawDb.exec(sql);
|
|
133
|
+
} finally {
|
|
134
|
+
logIfSlow(t0, sql);
|
|
135
|
+
}
|
|
86
136
|
}
|
|
87
137
|
|
|
88
138
|
transaction<T>(fn: () => T): T {
|
package/src/db.ts
CHANGED
|
@@ -174,6 +174,13 @@ db.exec(`
|
|
|
174
174
|
|
|
175
175
|
CREATE INDEX IF NOT EXISTS idx_agent_telemetry_alias_time
|
|
176
176
|
ON agent_telemetry(network_id, alias, created_at);
|
|
177
|
+
|
|
178
|
+
-- #2614 —— retention 每小时跑一次 DELETE FROM agent_telemetry WHERE created_at < ?。
|
|
179
|
+
-- 上面三个复合索引都以 network_id 打头,**这个查询用不上**(没有 network_id 等值条件),
|
|
180
|
+
-- 于是每次扫全表 62 万行 → 生产实测每小时出现 **8–11 秒的事件循环阻塞**
|
|
181
|
+
-- ([commhub retention] swept in 8331ms / 9863ms / 10588ms,同时日志静默 11 秒)。
|
|
182
|
+
-- 单独给 created_at 建索引,让这个扫描变成范围 seek。
|
|
183
|
+
CREATE INDEX IF NOT EXISTS idx_agent_telemetry_created ON agent_telemetry(created_at);
|
|
177
184
|
`);
|
|
178
185
|
|
|
179
186
|
// Round-2/4 review ② — server-health index split.
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
// #2614 —— 事件循环停顿监测 + 慢 SQL 日志的单元测试。
|
|
2
|
+
//
|
|
3
|
+
// 这两个东西的价值不在“能测出停顿”,而在**停顿发生时留下可对齐的证据**:
|
|
4
|
+
// 2026-10-26 生产两次 RSS 3.6/4.5 GB 的尖峰,事后只能靠“日志静默几十秒”
|
|
5
|
+
// 手工推断,`/api/stats/routes` 只给出受害者请求(含排队时间)。
|
|
6
|
+
// 所以测试要钉住的是语义:怎么算一次停顿、留什么档、日志阈值怎么生效、关闭后还计数。
|
|
7
|
+
|
|
8
|
+
import { describe, expect, test } from "bun:test";
|
|
9
|
+
import {
|
|
10
|
+
eventLoopStats,
|
|
11
|
+
recordGap,
|
|
12
|
+
resetEventLoopLagForTests,
|
|
13
|
+
startEventLoopLagMonitor,
|
|
14
|
+
} from "./event-loop-lag.js";
|
|
15
|
+
import { formatSlowSql, slowSqlThresholdMs } from "./db-adapter.js";
|
|
16
|
+
|
|
17
|
+
describe("event-loop-lag: recordGap", () => {
|
|
18
|
+
test("正常 tick(≈interval)不算停顿", () => {
|
|
19
|
+
resetEventLoopLagForTests();
|
|
20
|
+
expect(recordGap(260, { intervalMs: 250, thresholdMs: 2000 })).toBe(10);
|
|
21
|
+
const s = eventLoopStats();
|
|
22
|
+
expect(s.stall_count).toBe(0);
|
|
23
|
+
expect(s.max_gap_ms).toBe(260);
|
|
24
|
+
expect(s.recent_stalls).toEqual([]);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("超过阈值的 tick 记一条停顿,带 stall_ms / rss_mb / at", () => {
|
|
28
|
+
resetEventLoopLagForTests();
|
|
29
|
+
const now = Date.parse("2026-10-26T22:10:41Z");
|
|
30
|
+
const stall = recordGap(28_000, { intervalMs: 250, thresholdMs: 2000, rssMb: 4502, now });
|
|
31
|
+
expect(stall).toBe(27_750);
|
|
32
|
+
const s = eventLoopStats();
|
|
33
|
+
expect(s.stall_count).toBe(1);
|
|
34
|
+
expect(s.last_stall_ms).toBe(27_750);
|
|
35
|
+
expect(s.last_stall_at).toBe("2026-10-26T22:10:41.000Z");
|
|
36
|
+
expect(s.max_gap_ms).toBe(28_000);
|
|
37
|
+
expect(s.recent_stalls[0]).toMatchObject({ gap_ms: 28_000, stall_ms: 27_750, rss_mb: 4502 });
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test("recent_stalls 最新在前,且只保留 20 条", () => {
|
|
41
|
+
resetEventLoopLagForTests();
|
|
42
|
+
for (let i = 0; i < 25; i++) {
|
|
43
|
+
recordGap(5_000 + i, { intervalMs: 250, thresholdMs: 2_000, rssMb: 300, now: 1_700_000_000_000 + i * 1_000 });
|
|
44
|
+
}
|
|
45
|
+
const s = eventLoopStats();
|
|
46
|
+
expect(s.stall_count).toBe(25);
|
|
47
|
+
expect(s.recent_stalls.length).toBe(20);
|
|
48
|
+
// 最新一条是 i=24
|
|
49
|
+
expect(s.recent_stalls[0].gap_ms).toBe(5_024);
|
|
50
|
+
expect(s.recent_stalls[19].gap_ms).toBe(5_005);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("max_gap_ms 保留历史最大值(回落后不被覆盖)", () => {
|
|
54
|
+
resetEventLoopLagForTests();
|
|
55
|
+
recordGap(47_000, { intervalMs: 250, thresholdMs: 2_000 });
|
|
56
|
+
recordGap(260, { intervalMs: 250, thresholdMs: 2_000 });
|
|
57
|
+
const s = eventLoopStats();
|
|
58
|
+
expect(s.max_gap_ms).toBe(47_000);
|
|
59
|
+
expect(s.last_gap_ms).toBe(260);
|
|
60
|
+
});
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
describe("event-loop-lag: startEventLoopLagMonitor", () => {
|
|
64
|
+
test("阈值 0 时不打日志,但仍然计数", async () => {
|
|
65
|
+
resetEventLoopLagForTests();
|
|
66
|
+
const logs: string[] = [];
|
|
67
|
+
const timer = startEventLoopLagMonitor({
|
|
68
|
+
intervalMs: 20,
|
|
69
|
+
thresholdMs: 0,
|
|
70
|
+
log: (m) => logs.push(m),
|
|
71
|
+
rssMb: () => 123,
|
|
72
|
+
});
|
|
73
|
+
// 同步阻塞 80ms,让下一次 tick 一定超阈值
|
|
74
|
+
const until = performance.now() + 80;
|
|
75
|
+
while (performance.now() < until) { /* spin */ }
|
|
76
|
+
await Bun.sleep(60);
|
|
77
|
+
clearInterval(timer);
|
|
78
|
+
expect(logs).toEqual([]);
|
|
79
|
+
expect(eventLoopStats().max_gap_ms).toBeGreaterThan(0);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test("真实停顿会打一行 [commhub stall] 且带 rss", async () => {
|
|
83
|
+
resetEventLoopLagForTests();
|
|
84
|
+
const logs: string[] = [];
|
|
85
|
+
const timer = startEventLoopLagMonitor({
|
|
86
|
+
intervalMs: 20,
|
|
87
|
+
thresholdMs: 40,
|
|
88
|
+
log: (m) => logs.push(m),
|
|
89
|
+
rssMb: () => 4502,
|
|
90
|
+
});
|
|
91
|
+
const until = performance.now() + 120;
|
|
92
|
+
while (performance.now() < until) { /* spin 阻塞事件循环 */ }
|
|
93
|
+
await Bun.sleep(60);
|
|
94
|
+
clearInterval(timer);
|
|
95
|
+
expect(logs.length).toBeGreaterThan(0);
|
|
96
|
+
expect(logs[0]).toStartWith("[commhub stall] loop blocked ~");
|
|
97
|
+
expect(logs[0]).toContain("rss 4502 MB");
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
describe("db-adapter: 慢 SQL 日志", () => {
|
|
102
|
+
test("默认阈值 1000ms;环境变量可调,0 表示关闭", () => {
|
|
103
|
+
const prev = process.env.COMMHUB_SLOW_SQL_MS;
|
|
104
|
+
try {
|
|
105
|
+
delete process.env.COMMHUB_SLOW_SQL_MS;
|
|
106
|
+
expect(slowSqlThresholdMs()).toBe(1000);
|
|
107
|
+
process.env.COMMHUB_SLOW_SQL_MS = "250";
|
|
108
|
+
expect(slowSqlThresholdMs()).toBe(250);
|
|
109
|
+
process.env.COMMHUB_SLOW_SQL_MS = "0";
|
|
110
|
+
expect(slowSqlThresholdMs()).toBe(0);
|
|
111
|
+
process.env.COMMHUB_SLOW_SQL_MS = "not-a-number";
|
|
112
|
+
expect(slowSqlThresholdMs()).toBe(1000);
|
|
113
|
+
} finally {
|
|
114
|
+
if (prev === undefined) delete process.env.COMMHUB_SLOW_SQL_MS;
|
|
115
|
+
else process.env.COMMHUB_SLOW_SQL_MS = prev;
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
test("日志压平换行并截断,且不含绑定参数", () => {
|
|
120
|
+
const sql = "DELETE FROM agent_telemetry\n WHERE created_at < datetime('now', ?1)";
|
|
121
|
+
const line = formatSlowSql(10_588.4, sql);
|
|
122
|
+
expect(line).toStartWith("[commhub slow-sql] 10588ms DELETE FROM agent_telemetry WHERE created_at < datetime('now', ?1)");
|
|
123
|
+
expect(line).not.toContain("\n");
|
|
124
|
+
const long = formatSlowSql(1_200, "SELECT " + "x".repeat(500));
|
|
125
|
+
expect(long.length).toBeLessThanOrEqual("[commhub slow-sql] 1200ms ".length + 160);
|
|
126
|
+
});
|
|
127
|
+
});
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
// #2614 —— 事件循环停顿监测(只观测,不改任何行为)。
|
|
2
|
+
//
|
|
3
|
+
// 背景:2026-10-26 21:55–22:12 生产 Hub 出现两次内存尖峰(RSS 3.6 → 4.5 GB),
|
|
4
|
+
// 伴随机 `mcp_inflight` 387 → 807。事后看日志才发现**同一时刻日志静默 14–47 秒**
|
|
5
|
+
// ——Hub 是单线程(Bun + 同步 SQLite),日志静默就等于事件循环被阻塞。
|
|
6
|
+
//
|
|
7
|
+
// 现有 `/api/stats/routes` 只能告诉我们“哪个请求**受害**了”(它的耗时包含了排队时间),
|
|
8
|
+
// 不能告诉我们“当时**谁**在阻塞”。缺的就是停顿本身的时间序列。
|
|
9
|
+
//
|
|
10
|
+
// 这个模块补上这一格:每 `intervalMs` 醒一次,量 `now - lastTick`;
|
|
11
|
+
// 正常情况 ≈ interval,超出 `thresholdMs` 的部分就是一次真实停顿。
|
|
12
|
+
// 停顿会:① 计数/留档(`/health` 的 `event_loop` 字段可读);
|
|
13
|
+
// ② 打一行日志(带当时的 RSS,方便和内存曲线对齐)。
|
|
14
|
+
//
|
|
15
|
+
// 注意:`setInterval` 自己也会被阻塞 —— 这正是我们要测量的东西。
|
|
16
|
+
// 停顿结束后第一次 tick 的 gap 就是停顿长度,不需要外部进程配合。
|
|
17
|
+
|
|
18
|
+
export type EventLoopStall = {
|
|
19
|
+
/** ISO 时间(停顿**结束**的时刻,即这次 tick 的时间)。 */
|
|
20
|
+
at: string;
|
|
21
|
+
/** 距上次 tick 的实际间隔(毫秒)。等于 interval + 停顿。 */
|
|
22
|
+
gap_ms: number;
|
|
23
|
+
/** 停顿量 = gap - interval。 */
|
|
24
|
+
stall_ms: number;
|
|
25
|
+
/** 停顿结束时的进程 RSS(MB),用于和内存尖峰对齐。 */
|
|
26
|
+
rss_mb: number;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
const KEEP_STALLS = 20;
|
|
30
|
+
|
|
31
|
+
let maxGapMs = 0;
|
|
32
|
+
let lastGapMs = 0;
|
|
33
|
+
let stallCount = 0;
|
|
34
|
+
let lastStallAt: string | null = null;
|
|
35
|
+
let lastStallMs = 0;
|
|
36
|
+
const stalls: EventLoopStall[] = [];
|
|
37
|
+
|
|
38
|
+
/** 记录一次 tick 间隔。返回这次 tick 的停顿量(毫秒,可为负→归零)。 */
|
|
39
|
+
export function recordGap(
|
|
40
|
+
gapMs: number,
|
|
41
|
+
opts: { intervalMs?: number; thresholdMs?: number; rssMb?: number; now?: number } = {},
|
|
42
|
+
): number {
|
|
43
|
+
const interval = opts.intervalMs ?? 250;
|
|
44
|
+
const threshold = opts.thresholdMs ?? 2000;
|
|
45
|
+
lastGapMs = gapMs;
|
|
46
|
+
if (gapMs > maxGapMs) maxGapMs = gapMs;
|
|
47
|
+
const stallMs = Math.max(0, gapMs - interval);
|
|
48
|
+
if (stallMs < threshold) return stallMs;
|
|
49
|
+
const at = new Date(opts.now ?? Date.now()).toISOString();
|
|
50
|
+
stallCount++;
|
|
51
|
+
lastStallAt = at;
|
|
52
|
+
lastStallMs = stallMs;
|
|
53
|
+
stalls.push({ at, gap_ms: Math.round(gapMs), stall_ms: Math.round(stallMs), rss_mb: opts.rssMb ?? 0 });
|
|
54
|
+
while (stalls.length > KEEP_STALLS) stalls.shift();
|
|
55
|
+
return stallMs;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function eventLoopStats(): {
|
|
59
|
+
max_gap_ms: number;
|
|
60
|
+
last_gap_ms: number;
|
|
61
|
+
stall_count: number;
|
|
62
|
+
last_stall_ms: number;
|
|
63
|
+
last_stall_at: string | null;
|
|
64
|
+
recent_stalls: EventLoopStall[];
|
|
65
|
+
} {
|
|
66
|
+
return {
|
|
67
|
+
max_gap_ms: Math.round(maxGapMs),
|
|
68
|
+
last_gap_ms: Math.round(lastGapMs),
|
|
69
|
+
stall_count: stallCount,
|
|
70
|
+
last_stall_ms: Math.round(lastStallMs),
|
|
71
|
+
last_stall_at: lastStallAt,
|
|
72
|
+
recent_stalls: [...stalls].reverse(),
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** 只给测试用:清空累计状态。 */
|
|
77
|
+
export function resetEventLoopLagForTests(): void {
|
|
78
|
+
maxGapMs = 0;
|
|
79
|
+
lastGapMs = 0;
|
|
80
|
+
stallCount = 0;
|
|
81
|
+
lastStallAt = null;
|
|
82
|
+
lastStallMs = 0;
|
|
83
|
+
stalls.length = 0;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function envMs(name: string, fallback: number): number {
|
|
87
|
+
const raw = Number(process.env[name]);
|
|
88
|
+
return Number.isFinite(raw) && raw >= 0 ? raw : fallback;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* 启动监测。返回 timer(调用方负责 clearInterval)。
|
|
93
|
+
* - `COMMHUB_STALL_LOG_MS`:停顿日志阈值,默认 2000 ms,设 0 关闭日志(仍计数);
|
|
94
|
+
* - `COMMHUB_STALL_TICK_MS`:采样间隔,默认 250 ms。
|
|
95
|
+
*/
|
|
96
|
+
export function startEventLoopLagMonitor(opts: {
|
|
97
|
+
intervalMs?: number;
|
|
98
|
+
thresholdMs?: number;
|
|
99
|
+
log?: (msg: string) => void;
|
|
100
|
+
rssMb?: () => number;
|
|
101
|
+
} = {}): ReturnType<typeof setInterval> {
|
|
102
|
+
const intervalMs = opts.intervalMs ?? Math.max(50, envMs("COMMHUB_STALL_TICK_MS", 250));
|
|
103
|
+
const thresholdMs = opts.thresholdMs ?? envMs("COMMHUB_STALL_LOG_MS", 2000);
|
|
104
|
+
const log = opts.log ?? ((m: string) => console.log(m));
|
|
105
|
+
const rssMb = opts.rssMb ?? (() => Math.round(process.memoryUsage().rss / 1048576));
|
|
106
|
+
let last = performance.now();
|
|
107
|
+
const timer = setInterval(() => {
|
|
108
|
+
const now = performance.now();
|
|
109
|
+
const gap = now - last;
|
|
110
|
+
last = now;
|
|
111
|
+
const rss = rssMb();
|
|
112
|
+
const stall = recordGap(gap, { intervalMs, thresholdMs, rssMb: rss });
|
|
113
|
+
if (thresholdMs > 0 && stall >= thresholdMs) {
|
|
114
|
+
// 和 retention/slow-sql 的日志同一前缀,方便 grep。
|
|
115
|
+
log(`[commhub stall] loop blocked ~${Math.round(stall)}ms (tick gap ${Math.round(gap)}ms, rss ${rss} MB)`);
|
|
116
|
+
}
|
|
117
|
+
}, intervalMs);
|
|
118
|
+
// 监测不该拖住进程退出。
|
|
119
|
+
(timer as unknown as { unref?: () => void }).unref?.();
|
|
120
|
+
last = performance.now();
|
|
121
|
+
return timer;
|
|
122
|
+
}
|
|
@@ -107,10 +107,16 @@ describe("#473 requirements_people", () => {
|
|
|
107
107
|
test("compact rows: username, display_name, role, department, owned agents; unowned agents listed once", async () => {
|
|
108
108
|
const r = await asBoss.call("requirements_people", { network_id: netA, q: "alice" });
|
|
109
109
|
expect(r.ok).toBe(true);
|
|
110
|
-
|
|
110
|
+
// #2596 —— Agent 行新增 updated_at / lifecycle_state(客户端用它做 alias 去重与“近 7 天”过滤)。
|
|
111
|
+
// 形状断言改为「关键字段匹配 + 允许新增键」,而不是整行严格相等。
|
|
112
|
+
expect(r.people).toHaveLength(1);
|
|
113
|
+
expect(r.people[0]).toMatchObject({
|
|
111
114
|
user_id: alice.id, username: alice.username, display_name: "爱丽丝", role: "member",
|
|
112
|
-
department: { id: deptId, name: `研发部${stamp % 1000}` },
|
|
113
|
-
}
|
|
115
|
+
department: { id: deptId, name: `研发部${stamp % 1000}` },
|
|
116
|
+
});
|
|
117
|
+
expect(r.people[0].agents).toHaveLength(1);
|
|
118
|
+
expect(r.people[0].agents[0]).toMatchObject({ node_id: node("alice"), alias: ALIAS.alice });
|
|
119
|
+
expect(r.people[0].agents[0]).toHaveProperty("updated_at");
|
|
114
120
|
const all = await asBoss.call("requirements_people", { network_id: netA, q: String(stamp) });
|
|
115
121
|
expect(all.agents_without_owner.map((a: any) => a.alias).sort()).toEqual([ALIAS.dup, ALIAS.dup, ALIAS.free].sort());
|
|
116
122
|
const bossRow = (await asBoss.call("requirements_people", { network_id: netA, q: boss.username })).people[0];
|
|
@@ -19,7 +19,22 @@ export const PEOPLE_MAX_LIMIT = 200;
|
|
|
19
19
|
|
|
20
20
|
/** `department_id`: Agent 的**显式**部门归属(可空)。空 = 仍按「负责人在哪个部门」推断,
|
|
21
21
|
* 客户端据此把 Agent 放对格子 —— 2026-10-19 前没有这一列,328 个 Agent 只能全算「未归属」。 */
|
|
22
|
-
export type DirectoryAgent = {
|
|
22
|
+
export type DirectoryAgent = {
|
|
23
|
+
node_id: string;
|
|
24
|
+
alias: string;
|
|
25
|
+
department_id?: string | null;
|
|
26
|
+
/**
|
|
27
|
+
* #2596 —— 这一行“最后一次被 Hub 看到”的时间(`nodes.updated_at`),以及生命周期状态。
|
|
28
|
+
*
|
|
29
|
+
* 为什么加:组织架构页把 368 个 agent 全塞进「未归属」,其中 125 个是 5–8 月的历史残留
|
|
30
|
+
* (如 `海报设计师1–5号` 注册于 2026-05-17、早已 offline)。客户端要能“默认只显示近 7 天活跃”
|
|
31
|
+
* 与“同一 alias 只保留最新的那条”,就必须拿到时间轴——否则只能靠 node_id 猜。
|
|
32
|
+
*
|
|
33
|
+
* 只增不改:老客户端忽略未知键,形状断言已在同批测试里放宽。
|
|
34
|
+
*/
|
|
35
|
+
updated_at?: string | null;
|
|
36
|
+
lifecycle_state?: string | null;
|
|
37
|
+
};
|
|
23
38
|
export type DirectoryPerson = {
|
|
24
39
|
user_id: string;
|
|
25
40
|
username: string;
|
|
@@ -64,8 +79,8 @@ export function peopleDirectory(networkId: string, hidden: HiddenNode, query: {
|
|
|
64
79
|
ORDER BY u.username, u.user_id`,
|
|
65
80
|
networkId,
|
|
66
81
|
);
|
|
67
|
-
const nodes = db.all<{ node_id: string; alias: string | null; node_name: string; owner_user_id: string | null; department_id: string | null }>(
|
|
68
|
-
"SELECT node_id, alias, node_name, owner_user_id, department_id FROM nodes WHERE network_id = ?1 ORDER BY COALESCE(alias, node_name), node_id",
|
|
82
|
+
const nodes = db.all<{ node_id: string; alias: string | null; node_name: string; owner_user_id: string | null; department_id: string | null; updated_at: string | null; lifecycle_state: string | null }>(
|
|
83
|
+
"SELECT node_id, alias, node_name, owner_user_id, department_id, updated_at, lifecycle_state FROM nodes WHERE network_id = ?1 ORDER BY COALESCE(alias, node_name), node_id",
|
|
69
84
|
networkId,
|
|
70
85
|
).filter(n => !hidden?.(n.node_id));
|
|
71
86
|
const memberIds = new Set(members.map(m => m.user_id));
|
|
@@ -75,7 +90,14 @@ export function peopleDirectory(networkId: string, hidden: HiddenNode, query: {
|
|
|
75
90
|
// 只在**显式**归属时带这一格:没设 = 没有显式归属(顶层 agent_department_supported 告诉
|
|
76
91
|
// 客户端这一点)。行形状保持不变 —— 旧客户端拿到的行还是老样子(多一个键会让严格要求
|
|
77
92
|
// 形状的测试/客户端意外),只是新客户端不再把「负责人在哪个部门」当成 Agent 的部门。
|
|
78
|
-
const agent = {
|
|
93
|
+
const agent = {
|
|
94
|
+
node_id: n.node_id,
|
|
95
|
+
alias: n.alias || n.node_name,
|
|
96
|
+
...(n.department_id ? { department_id: n.department_id } : {}),
|
|
97
|
+
// #2596 —— 客户端据此做「按 alias 去重(保留最新)」与「默认只看近 7 天」。
|
|
98
|
+
updated_at: n.updated_at ?? null,
|
|
99
|
+
lifecycle_state: n.lifecycle_state ?? null,
|
|
100
|
+
};
|
|
79
101
|
if (n.owner_user_id && memberIds.has(n.owner_user_id)) {
|
|
80
102
|
const list = byOwner.get(n.owner_user_id) ?? [];
|
|
81
103
|
list.push(agent);
|
package/src/server.ts
CHANGED
|
@@ -29,6 +29,7 @@ import { deriveTeamRulesView } from "./team-rules-status.js";
|
|
|
29
29
|
import { recentNodeIdentityConflicts } from "./node-identity-conflict.js";
|
|
30
30
|
import { createSSEStream, createNetworkObserverStream, createUserEventStream, pushEvent, pushNetworkObserverEvent, getSSEStats, PRINTABLE_OBSERVER_KEY_PREFIX, closeUserStreamsInNetwork, getUserPresence, onUserPresenceChange, pushUserEventToNetwork } from "./push.js";
|
|
31
31
|
import { enterMcpInflight, leaveMcpInflight, getMcpInflightStats } from "./mcp-inflight.js";
|
|
32
|
+
import { eventLoopStats, startEventLoopLagMonitor } from "./event-loop-lag.js";
|
|
32
33
|
import { assertNodeActive } from "./lifecycle-guard.js";
|
|
33
34
|
import { pendingInboxCount } from "./inbox-count.js";
|
|
34
35
|
import { notifyExpiredTasks, type ExpiredTaskRow } from "./task-expiry-notice.js";
|
|
@@ -2788,6 +2789,11 @@ return Bun.serve({
|
|
|
2788
2789
|
// 而 `sse_connections` 只反映长连接,看不到“同秒打进来的一批请求”。
|
|
2789
2790
|
mcp_inflight: getMcpInflightStats().inflight,
|
|
2790
2791
|
mcp_inflight_peak: getMcpInflightStats().peak,
|
|
2792
|
+
// #2614 —— 事件循环停顿。`mcp_inflight` 只能说明“有多少请求在排队”,
|
|
2793
|
+
// 说明不了“服务端为什么卡”。这里给出停顿本身:max_gap_ms 是进程内
|
|
2794
|
+
// 单次 tick 的最大间隔(正常 ≈250ms),recent_stalls 保留最近 20 次
|
|
2795
|
+
// 超过阈值的停顿(含当时 RSS),用于和内存曲线对齐。
|
|
2796
|
+
event_loop: eventLoopStats(),
|
|
2791
2797
|
auth: DEV_OPEN ? "dev-open" : "user-token",
|
|
2792
2798
|
security: DEV_OPEN ? "dev-open" : "secured",
|
|
2793
2799
|
tmux: TMUX_ENABLED ? "enabled" : "disabled",
|
|
@@ -5631,6 +5637,12 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
|
|
|
5631
5637
|
: 60 * 60 * 1000;
|
|
5632
5638
|
const retentionSweeperTimer = startRetentionSweeper(sweepIntervalMs);
|
|
5633
5639
|
|
|
5640
|
+
// #2614 —— 事件循环停顿监测。放这里是因为 startHub() 才是“真的在跑一个 Hub”
|
|
5641
|
+
// 的入口(测试用 bootServer() 不会背这个 timer)。停顿日志与
|
|
5642
|
+
// `[commhub retention]` / `[commhub slow-sql]` 共用前缀族,便于一次 grep 出
|
|
5643
|
+
// “同一秒里 retention 跑了 10 秒 + 有两百个请求在排队”。
|
|
5644
|
+
const eventLoopLagTimer = startEventLoopLagMonitor();
|
|
5645
|
+
|
|
5634
5646
|
// Round-2/4 review ③ — stale session sweeper (coexists with the
|
|
5635
5647
|
// retention sweeper above; different concerns + different cadences).
|
|
5636
5648
|
// Replaces the per-request UPDATE in GET /api/status (+ /api/servers,
|
|
@@ -5664,6 +5676,7 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
|
|
|
5664
5676
|
// jobs must not — otherwise a test (or a future caller) that stops the
|
|
5665
5677
|
// server would still hang on live intervals it cannot reach.
|
|
5666
5678
|
(retentionSweeperTimer as any)?.unref?.();
|
|
5679
|
+
(eventLoopLagTimer as any)?.unref?.();
|
|
5667
5680
|
(staleSweeperTimer as any)?.unref?.();
|
|
5668
5681
|
(rateLimitSweepTimer as any)?.unref?.();
|
|
5669
5682
|
(taskPatrolTimer as any)?.unref?.();
|
|
@@ -5675,6 +5688,7 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
|
|
|
5675
5688
|
function shutdown() {
|
|
5676
5689
|
console.log("[commhub] shutting down...");
|
|
5677
5690
|
clearInterval(retentionSweeperTimer);
|
|
5691
|
+
clearInterval(eventLoopLagTimer);
|
|
5678
5692
|
clearInterval(staleSweeperTimer);
|
|
5679
5693
|
clearInterval(rateLimitSweepTimer);
|
|
5680
5694
|
clearInterval(taskPatrolTimer);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// #2571 —— delivered_stale
|
|
1
|
+
// #2571 —— delivered_stale 的**分流**(阈值 10m/30m,见 task-lifecycle-watcher.ts 顶部说明):目标在跑别的任务时不再报“可能离线”。
|
|
2
2
|
//
|
|
3
3
|
// 生产数据(24h):出现过 delivered_stale 的任务里 83% 最终成功,等待 p50 2.9 分钟、
|
|
4
4
|
// p90 27 分钟 ⇒ 事件事实成立,但把“目标正忙”写成“可能离线”会制造告警疲劳。
|
|
@@ -11,7 +11,7 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
|
11
11
|
import { db } from "./db.js";
|
|
12
12
|
import { recordDeliveredStaleEvents } from "./task-lifecycle-watcher.js";
|
|
13
13
|
|
|
14
|
-
const NOW = new Date("2026-08-
|
|
14
|
+
const NOW = new Date("2026-08-10T01:00:01.000Z");
|
|
15
15
|
let networkId = "";
|
|
16
16
|
|
|
17
17
|
function sqliteUtcTimestamp(date: Date): string {
|
|
@@ -48,15 +48,15 @@ beforeAll(async () => {
|
|
|
48
48
|
|
|
49
49
|
// 目标在跑别的任务
|
|
50
50
|
insertSession("target-busy", "working");
|
|
51
|
-
insertTask("busy-
|
|
51
|
+
insertTask("busy-600", "target-busy", "2026-08-10 00:50:01");
|
|
52
52
|
// 目标空闲却没开始(真异常)
|
|
53
53
|
insertSession("target-idle", "idle");
|
|
54
|
-
insertTask("idle-
|
|
54
|
+
insertTask("idle-600", "target-idle", "2026-08-10 00:50:01");
|
|
55
55
|
// 目标掉线
|
|
56
56
|
insertSession("target-offline", "offline");
|
|
57
|
-
insertTask("offline-
|
|
57
|
+
insertTask("offline-600", "target-offline", "2026-08-10 00:50:01");
|
|
58
58
|
// 完全没有 session 行(未知)
|
|
59
|
-
insertTask("unknown-
|
|
59
|
+
insertTask("unknown-600", "target-without-session", "2026-08-10 00:50:01");
|
|
60
60
|
}, 30_000);
|
|
61
61
|
|
|
62
62
|
afterAll(() => {
|
|
@@ -70,17 +70,17 @@ afterAll(() => {
|
|
|
70
70
|
describe("#2571 stale vs queued-behind-busy", () => {
|
|
71
71
|
test("目标 working ⇒ 只写 queued_behind_busy_*,不写 delivered_stale_*", () => {
|
|
72
72
|
recordDeliveredStaleEvents(NOW);
|
|
73
|
-
const events = eventsFor("busy-
|
|
74
|
-
expect(events).toContain("task.warning.
|
|
75
|
-
expect(events).not.toContain("task.warning.
|
|
73
|
+
const events = eventsFor("busy-600");
|
|
74
|
+
expect(events).toContain("task.warning.queued_behind_busy_600s");
|
|
75
|
+
expect(events).not.toContain("task.warning.delivered_stale_600s");
|
|
76
76
|
});
|
|
77
77
|
|
|
78
78
|
test("目标 idle / offline / 无 session ⇒ 只写 delivered_stale_*", () => {
|
|
79
79
|
recordDeliveredStaleEvents(NOW);
|
|
80
|
-
for (const taskId of ["idle-
|
|
80
|
+
for (const taskId of ["idle-600", "offline-600", "unknown-600"]) {
|
|
81
81
|
const events = eventsFor(taskId);
|
|
82
|
-
expect(events).toContain("task.warning.
|
|
83
|
-
expect(events).not.toContain("task.warning.
|
|
82
|
+
expect(events).toContain("task.warning.delivered_stale_600s");
|
|
83
|
+
expect(events).not.toContain("task.warning.queued_behind_busy_600s");
|
|
84
84
|
}
|
|
85
85
|
});
|
|
86
86
|
|
|
@@ -100,10 +100,10 @@ describe("#2571 stale vs queued-behind-busy", () => {
|
|
|
100
100
|
|
|
101
101
|
test("detail 文案能区分“排队”和“没在跑”", () => {
|
|
102
102
|
const busy = db.get<{ detail: string }>(
|
|
103
|
-
`SELECT detail FROM task_events WHERE task_id='busy-
|
|
103
|
+
`SELECT detail FROM task_events WHERE task_id='busy-600' AND event_type LIKE 'task.warning.queued_behind_busy_%'`,
|
|
104
104
|
);
|
|
105
105
|
const stale = db.get<{ detail: string }>(
|
|
106
|
-
`SELECT detail FROM task_events WHERE task_id='idle-
|
|
106
|
+
`SELECT detail FROM task_events WHERE task_id='idle-600' AND event_type LIKE 'task.warning.delivered_stale_%'`,
|
|
107
107
|
);
|
|
108
108
|
expect(busy!.detail).toContain("queued after");
|
|
109
109
|
expect(stale!.detail).toContain("still delivered");
|
|
@@ -4,7 +4,7 @@ import { db, logTaskEvent } from "./db.js";
|
|
|
4
4
|
import { TASK_EVENT_REST_COLUMNS } from "./rest-projections.js";
|
|
5
5
|
import { recordDeliveredStaleEvents } from "./task-lifecycle-watcher.js";
|
|
6
6
|
|
|
7
|
-
const NOW = new Date("2026-08-
|
|
7
|
+
const NOW = new Date("2026-08-10T01:00:01.000Z");
|
|
8
8
|
let server: ReturnType<typeof Bun.serve>;
|
|
9
9
|
let base = "";
|
|
10
10
|
let token = "";
|
|
@@ -34,9 +34,9 @@ beforeAll(async () => {
|
|
|
34
34
|
token = auth.token!;
|
|
35
35
|
networkId = auth.network_id!;
|
|
36
36
|
|
|
37
|
-
insertTask("stale-
|
|
38
|
-
insertTask("stale-
|
|
39
|
-
insertTask("stale-
|
|
37
|
+
insertTask("stale-599", "delivered", "2026-08-10 00:50:02");
|
|
38
|
+
insertTask("stale-600", "delivered", "2026-08-10 00:50:01");
|
|
39
|
+
insertTask("stale-1800", "delivered", "2026-08-10 00:30:01");
|
|
40
40
|
insertTask("stale-acked", "acked", "2026-08-10 00:00:00");
|
|
41
41
|
insertTask("stale-no-delivery", "delivered", null);
|
|
42
42
|
|
|
@@ -70,21 +70,21 @@ async function waitUntil(ready: () => boolean, timeoutMs: number, what: string |
|
|
|
70
70
|
}
|
|
71
71
|
|
|
72
72
|
describe("#167 Hub delivered-stale lifecycle watcher", () => {
|
|
73
|
-
test("
|
|
73
|
+
test("10m/30m thresholds are exact and non-delivered tasks stay silent", () => {
|
|
74
74
|
const first = recordDeliveredStaleEvents(NOW);
|
|
75
75
|
expect(first.inserted).toBe(3);
|
|
76
|
-
expect(first.by_threshold_seconds).toEqual({
|
|
76
|
+
expect(first.by_threshold_seconds).toEqual({ 600: 2, 1800: 1 });
|
|
77
77
|
|
|
78
|
-
expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-
|
|
79
|
-
expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-
|
|
80
|
-
expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-
|
|
78
|
+
expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-599'")!.count).toBe(0);
|
|
79
|
+
expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-600'")!.count).toBe(1);
|
|
80
|
+
expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-1800'")!.count).toBe(2);
|
|
81
81
|
expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-acked'")!.count).toBe(0);
|
|
82
82
|
expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-no-delivery'")!.count).toBe(0);
|
|
83
83
|
|
|
84
|
-
const sixty = db.all<any>("SELECT * FROM task_events WHERE task_id = 'stale-
|
|
84
|
+
const sixty = db.all<any>("SELECT * FROM task_events WHERE task_id = 'stale-1800' ORDER BY event_type");
|
|
85
85
|
expect(sixty.map((row) => row.event_type)).toEqual([
|
|
86
|
-
"task.warning.
|
|
87
|
-
"task.warning.
|
|
86
|
+
"task.warning.delivered_stale_1800s",
|
|
87
|
+
"task.warning.delivered_stale_600s",
|
|
88
88
|
]);
|
|
89
89
|
expect(sixty.map((row) => row.event_key)).toEqual(sixty.map((row) => row.event_type));
|
|
90
90
|
expect(sixty.every((row) => row.from_status === "delivered" && row.to_status === "delivered")).toBe(true);
|
|
@@ -95,21 +95,21 @@ describe("#167 Hub delivered-stale lifecycle watcher", () => {
|
|
|
95
95
|
test("a second patrol is write-once for each task and threshold", () => {
|
|
96
96
|
const second = recordDeliveredStaleEvents(NOW);
|
|
97
97
|
expect(second.inserted).toBe(0);
|
|
98
|
-
expect(second.by_threshold_seconds).toEqual({
|
|
98
|
+
expect(second.by_threshold_seconds).toEqual({ 600: 0, 1800: 0 });
|
|
99
99
|
expect(db.get<{ count: number }>(
|
|
100
100
|
"SELECT COUNT(*) AS count FROM task_events WHERE event_type LIKE 'task.warning.delivered_stale_%'",
|
|
101
101
|
)!.count).toBe(3);
|
|
102
102
|
});
|
|
103
103
|
|
|
104
104
|
test("ordinary lifecycle events get standard event names without a dedupe key", () => {
|
|
105
|
-
logTaskEvent("stale-
|
|
106
|
-
logTaskEvent("stale-
|
|
107
|
-
logTaskEvent("stale-
|
|
108
|
-
logTaskEvent("stale-
|
|
109
|
-
logTaskEvent("stale-
|
|
110
|
-
logTaskEvent("stale-
|
|
105
|
+
logTaskEvent("stale-600", null, "delivered", "target");
|
|
106
|
+
logTaskEvent("stale-600", "delivered", "acked", "target");
|
|
107
|
+
logTaskEvent("stale-600", "acked", "running", "target");
|
|
108
|
+
logTaskEvent("stale-600", "running", "replied", "target");
|
|
109
|
+
logTaskEvent("stale-600", "running", "failed", "target");
|
|
110
|
+
logTaskEvent("stale-600", "delivered", "expired", "target");
|
|
111
111
|
const rows = db.all<any>(
|
|
112
|
-
"SELECT event_type, event_key FROM task_events WHERE task_id = 'stale-
|
|
112
|
+
"SELECT event_type, event_key FROM task_events WHERE task_id = 'stale-600' AND actor = 'target' ORDER BY id",
|
|
113
113
|
);
|
|
114
114
|
expect(rows).toEqual([
|
|
115
115
|
{ event_type: "task.send.delivered", event_key: null },
|
|
@@ -125,7 +125,7 @@ describe("#167 Hub delivered-stale lifecycle watcher", () => {
|
|
|
125
125
|
expect(TASK_EVENT_REST_COLUMNS).toContain("event_type");
|
|
126
126
|
expect(TASK_EVENT_REST_COLUMNS).not.toContain("event_key" as any);
|
|
127
127
|
const response = await fetch(
|
|
128
|
-
`${base}/api/task_events?network_id=${encodeURIComponent(networkId)}&task_id=stale-
|
|
128
|
+
`${base}/api/task_events?network_id=${encodeURIComponent(networkId)}&task_id=stale-1800`,
|
|
129
129
|
{ headers: { Authorization: `Bearer ${token}` } },
|
|
130
130
|
);
|
|
131
131
|
expect(response.status).toBe(200);
|
|
@@ -177,7 +177,7 @@ describe("#167 Hub delivered-stale lifecycle watcher", () => {
|
|
|
177
177
|
`INSERT INTO tasks
|
|
178
178
|
(task_id, from_name, to_name, status, content, delivered_at, network_id)
|
|
179
179
|
VALUES (?1, 'sender', 'target', 'delivered', 'live watcher fixture', ?2, ?3)`,
|
|
180
|
-
[taskId, sqliteUtcTimestamp(new Date(Date.now() -
|
|
180
|
+
[taskId, sqliteUtcTimestamp(new Date(Date.now() - 601_000)), networkId],
|
|
181
181
|
);
|
|
182
182
|
expect(childDb.query<{ count: number }, [string]>(
|
|
183
183
|
"SELECT COUNT(*) AS count FROM task_events WHERE task_id = ?1",
|
|
@@ -188,10 +188,10 @@ describe("#167 Hub delivered-stale lifecycle watcher", () => {
|
|
|
188
188
|
// 实测在 CI 上被这一条打红过(#798 让这个文件第一次进 CI 才暴露)。
|
|
189
189
|
// 改成「轮询到目标状态,或到期报错」:常态下快 ~60 倍,慢的时候等得起。
|
|
190
190
|
const countStale = () => childDb.query<{ count: number }, [string]>(
|
|
191
|
-
"SELECT COUNT(*) AS count FROM task_events WHERE task_id = ?1 AND event_type = 'task.warning.
|
|
191
|
+
"SELECT COUNT(*) AS count FROM task_events WHERE task_id = ?1 AND event_type = 'task.warning.delivered_stale_600s'",
|
|
192
192
|
).get(taskId)!.count;
|
|
193
193
|
await waitUntil(() => countStale() === 1, 20_000,
|
|
194
|
-
"watcher did not write the
|
|
194
|
+
"watcher did not write the delivered_stale_600s event");
|
|
195
195
|
expect(countStale()).toBe(1);
|
|
196
196
|
} finally {
|
|
197
197
|
childDb.close();
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { db } from "./db.js";
|
|
2
2
|
|
|
3
|
-
export const DELIVERED_STALE_THRESHOLDS = [
|
|
3
|
+
export const DELIVERED_STALE_THRESHOLDS = [600, 1800] as const;
|
|
4
4
|
|
|
5
5
|
export type DeliveredStalePatrolResult = {
|
|
6
6
|
inserted: number;
|
|
@@ -11,6 +11,19 @@ function sqliteUtcTimestamp(date: Date): string {
|
|
|
11
11
|
return date.toISOString().slice(0, 19).replace("T", " ");
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
+
// #2571(第二轮)—— 阈值从 30s/60s 提到 10min/30min。
|
|
15
|
+
//
|
|
16
|
+
// 第一轮只做了“按目标忙/闲分流”,上线后(19:45–22:30)实测**告警量反而翻倍**
|
|
17
|
+
// (33/h → 73/h,33/h → 104/h),因为同一个任务会先在忙时写 busy、目标转闲后
|
|
18
|
+
// 再写一次 stale(46 个任务两类都出现)。而被标 stale 的任务里仍有 73.5%
|
|
19
|
+
// 最终成功。真正的病因不是“忙 vs 闲”,是**阈值本身离真实分布太远**:
|
|
20
|
+
// 7 天 11877 个成功任务,delivered → 首次活动 p50 = 1 秒、p90 = 6 秒、
|
|
21
|
+
// p95 = 57 秒、p99 = 12 分钟。
|
|
22
|
+
// ⇒ 30 秒的判据会命中 6.4% 的**正常**任务,60 秒命中 4.9%,等于把 p95–p99
|
|
23
|
+
// 的长尾全报警。改成 10 分钟(p99 之下)才有意义:超过它的比例 1.2%,
|
|
24
|
+
// 30 分钟是 0.2%,只留真异常。
|
|
25
|
+
//
|
|
26
|
+
// event_key 里带秒数 ⇒ 改阈值会产生新的 key,历史事件不受影响、可并存对比。
|
|
14
27
|
/**
|
|
15
28
|
* Persist Hub-authoritative warnings for tasks which are still delivered.
|
|
16
29
|
*
|
|
@@ -27,13 +40,18 @@ function sqliteUtcTimestamp(date: Date): string {
|
|
|
27
40
|
* 命中次数变成 0 或让 mutation 只命中一半 —— 那就把“可被变异测试覆盖”这条性质弄丢了。
|
|
28
41
|
* 因此这里仍然只保留**一条** INSERT,用 CASE 选择事件名与文案。
|
|
29
42
|
*/
|
|
30
|
-
function
|
|
43
|
+
function thresholdLabel(thresholdSeconds: 600 | 1800): string {
|
|
44
|
+
return thresholdSeconds === 600 ? "10m" : "30m";
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function recordStale(thresholdSeconds: 600 | 1800, cutoff: string): number {
|
|
31
48
|
const busyType = `task.warning.queued_behind_busy_${thresholdSeconds}s`;
|
|
32
49
|
const staleType = `task.warning.delivered_stale_${thresholdSeconds}s`;
|
|
33
|
-
const
|
|
34
|
-
const
|
|
35
|
-
|
|
36
|
-
|
|
50
|
+
const label = thresholdLabel(thresholdSeconds);
|
|
51
|
+
const busyDetail = `queued after ${label}; target is working on another task`;
|
|
52
|
+
const staleDetail = thresholdSeconds === 600
|
|
53
|
+
? `still delivered after ${label}; target is not working on another task and has not started`
|
|
54
|
+
: `still delivered after ${label}; target is idle or offline and not consuming`;
|
|
37
55
|
const result = db.run(
|
|
38
56
|
`INSERT INTO task_events
|
|
39
57
|
(task_id, from_status, to_status, event_type, event_key, actor, detail, network_id)
|
|
@@ -62,7 +80,7 @@ function recordStale(thresholdSeconds: 30 | 60, cutoff: string): number {
|
|
|
62
80
|
export function recordDeliveredStaleEvents(now = new Date()): DeliveredStalePatrolResult {
|
|
63
81
|
if (!Number.isFinite(now.getTime())) throw new Error("invalid lifecycle patrol time");
|
|
64
82
|
|
|
65
|
-
const byThreshold = {
|
|
83
|
+
const byThreshold = { 600: 0, 1800: 0 };
|
|
66
84
|
let inserted = 0;
|
|
67
85
|
|
|
68
86
|
for (const thresholdSeconds of DELIVERED_STALE_THRESHOLDS) {
|