@sleep2agi/commhub-server 0.9.0-preview.176 → 0.9.0-preview.178
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sleep2agi/commhub-server",
|
|
3
|
-
"version": "0.9.0-preview.
|
|
3
|
+
"version": "0.9.0-preview.178",
|
|
4
4
|
"description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// #2614(第三轮)—— 首次 sweep 不是“等 N 秒”,而是“等循环安静”。
|
|
2
|
+
//
|
|
3
|
+
// 两次部署实测:60 秒后仍撞上风暴(max 1238ms);180 秒后撞得更狠(max 3919ms,风暴持续 5 分钟)。
|
|
4
|
+
// 这里钉住判据本身(纯函数),不看时钟、不看真实 hub。
|
|
5
|
+
import { describe, expect, test } from "bun:test";
|
|
6
|
+
import { firstSweepShouldWait, FIRST_SWEEP_QUIET_INFLIGHT } from "./retention.js";
|
|
7
|
+
|
|
8
|
+
const NOW = Date.parse("2026-10-27T12:00:00.000Z");
|
|
9
|
+
const iso = (s: number) => new Date(NOW - s * 1000).toISOString();
|
|
10
|
+
|
|
11
|
+
describe("firstSweepShouldWait (#2614)", () => {
|
|
12
|
+
test("waits while requests are queued", () => {
|
|
13
|
+
expect(firstSweepShouldWait({ now: NOW, inflight: FIRST_SWEEP_QUIET_INFLIGHT + 1, lastStallAtIso: null })).toBe(true);
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
test("waits while the loop stalled within the last minute", () => {
|
|
17
|
+
expect(firstSweepShouldWait({ now: NOW, inflight: 0, lastStallAtIso: iso(30) })).toBe(true);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
test("runs once inflight is low and the last stall is old enough", () => {
|
|
21
|
+
expect(firstSweepShouldWait({ now: NOW, inflight: 3, lastStallAtIso: iso(120) })).toBe(false);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
test("runs when there has never been a stall", () => {
|
|
25
|
+
expect(firstSweepShouldWait({ now: NOW, inflight: 0, lastStallAtIso: null })).toBe(false);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test("an unparseable stall timestamp does not block forever", () => {
|
|
29
|
+
expect(firstSweepShouldWait({ now: NOW, inflight: 0, lastStallAtIso: "not-a-date" })).toBe(false);
|
|
30
|
+
});
|
|
31
|
+
});
|
package/src/retention.ts
CHANGED
|
@@ -24,6 +24,8 @@
|
|
|
24
24
|
// COMMHUB_DB=/tmp/test-retention.db bun test src/retention.test.ts
|
|
25
25
|
|
|
26
26
|
import { db } from "./db.js";
|
|
27
|
+
import { getMcpInflightStats } from "./mcp-inflight.js";
|
|
28
|
+
import { eventLoopStats } from "./event-loop-lag.js";
|
|
27
29
|
import { sweepNodeRequestContent, type NodeRequestSweepResult } from "./node-request-retention.js";
|
|
28
30
|
|
|
29
31
|
export type RetentionConfig = {
|
|
@@ -131,6 +133,54 @@ const DEFAULT_CHUNK_ROWS = 2000;
|
|
|
131
133
|
const INCREMENTAL_VACUUM_PAGES = 200;
|
|
132
134
|
const MAX_TABLE_MS = 10 * 60 * 1000;
|
|
133
135
|
|
|
136
|
+
/**
|
|
137
|
+
* #2614(第三轮)—— 首次 sweep 要等**循环真的安静下来**,而不是固定等 N 秒。
|
|
138
|
+
*
|
|
139
|
+
* 2026-10-27 两次部署实测:
|
|
140
|
+
* ```
|
|
141
|
+
* 18:55 部署:60 秒后 sweep → 3894ms / max 1238ms(风暴 2 分钟后仍在)
|
|
142
|
+
* 19:51 部署:180 秒后 sweep → 9153ms / max 3919ms(风暴持续 5 分钟,RSS 3.6 GB)
|
|
143
|
+
* ```
|
|
144
|
+
* 固定延迟只是把「撞上风暴」变成「可能撞上风暴」——风暴长短取决于同时重连多少节点。
|
|
145
|
+
*
|
|
146
|
+
* 判据(都来自 `/health` 的同名指标):
|
|
147
|
+
* · `mcp_inflight < 20`:没有成批请求在排队;
|
|
148
|
+
* · 最近一次 stall 至少在 60 秒前:循环刚刚不抖。
|
|
149
|
+
* 每 15 秒看一次,最多等 15 分钟;等不到就照常跑(宁可慢,不可不删)。
|
|
150
|
+
*/
|
|
151
|
+
export const FIRST_SWEEP_QUIET_INFLIGHT = 20;
|
|
152
|
+
export const FIRST_SWEEP_QUIET_STALL_AGE_MS = 60_000;
|
|
153
|
+
export const FIRST_SWEEP_MAX_WAIT_MS = 15 * 60_000;
|
|
154
|
+
export const FIRST_SWEEP_POLL_MS = 15_000;
|
|
155
|
+
|
|
156
|
+
/** 纯函数,便于单测:把「此刻该不该继续等」与时钟解耦。 */
|
|
157
|
+
export function firstSweepShouldWait(input: {
|
|
158
|
+
now: number;
|
|
159
|
+
inflight: number;
|
|
160
|
+
lastStallAtIso: string | null;
|
|
161
|
+
}): boolean {
|
|
162
|
+
if (input.inflight >= FIRST_SWEEP_QUIET_INFLIGHT) return true;
|
|
163
|
+
if (!input.lastStallAtIso) return false;
|
|
164
|
+
const last = Date.parse(input.lastStallAtIso);
|
|
165
|
+
if (!Number.isFinite(last)) return false;
|
|
166
|
+
return input.now - last < FIRST_SWEEP_QUIET_STALL_AGE_MS;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
async function waitForQuietLoop(): Promise<void> {
|
|
170
|
+
const startedMs = Date.now();
|
|
171
|
+
while (Date.now() - startedMs < FIRST_SWEEP_MAX_WAIT_MS) {
|
|
172
|
+
const stats = eventLoopStats();
|
|
173
|
+
if (!firstSweepShouldWait({
|
|
174
|
+
now: Date.now(),
|
|
175
|
+
inflight: getMcpInflightStats().inflight,
|
|
176
|
+
lastStallAtIso: stats.last_stall_at,
|
|
177
|
+
})) {
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
await Bun.sleep(FIRST_SWEEP_POLL_MS);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
134
184
|
/** 启动后第一次 sweep 的延迟;默认 180 秒(见 startRetentionSweeper 里的实测)。 */
|
|
135
185
|
function readFirstSweepDelayMs(): number {
|
|
136
186
|
const raw = process.env.COMMHUB_RETENTION_FIRST_DELAY_MS;
|
|
@@ -526,7 +576,13 @@ export function startRetentionSweeper(
|
|
|
526
576
|
// 18:56:26 stall 2.7s · 18:57:28 stall 3.4s · 18:57:31 stall 2.6s ← 风暴 2 分钟还没排空
|
|
527
577
|
// ```
|
|
528
578
|
// 稳态同一批删除是 0.24ms/行 ⇒ 60 秒时风暴仍在,max 单批被推到 1.2 秒(越过 1 秒慢 SQL 阈值)。
|
|
529
|
-
|
|
530
|
-
|
|
579
|
+
const firstSweep = async () => {
|
|
580
|
+
// 先等固定延迟越过重连风暴的**起点**,再等循环真的安静(见 waitForQuietLoop)。
|
|
581
|
+
if (firstDelayMs > 0) await Bun.sleep(firstDelayMs);
|
|
582
|
+
await waitForQuietLoop();
|
|
583
|
+
await sweep();
|
|
584
|
+
};
|
|
585
|
+
if (firstDelayMs > 0) setTimeout(() => { void firstSweep(); }, 0).unref?.();
|
|
586
|
+
else setImmediate(() => { void firstSweep(); });
|
|
531
587
|
return setInterval(() => { void sweep(); }, intervalMs);
|
|
532
588
|
}
|
|
@@ -242,6 +242,8 @@ export function applyFailureBackoff(scheduleId: string, now = new Date()): { str
|
|
|
242
242
|
* · 与 `no_change_backoff=1` 的排程互斥(那套逻辑自己拥有 backoff_* 三列)。
|
|
243
243
|
*/
|
|
244
244
|
const NO_PROGRESS_CODES = ["previous_run_active", "target_offline"] as const;
|
|
245
|
+
/** 统计窗口:看最近多少次 run 里同因跳过出现了几次(默认 10)。 */
|
|
246
|
+
const NO_PROGRESS_WINDOW = 10;
|
|
245
247
|
export type NoProgressCause = (typeof NO_PROGRESS_CODES)[number];
|
|
246
248
|
|
|
247
249
|
function noProgressFromRuns(runs: Array<{ status: string; error_code: string | null }>): { cause: NoProgressCause | null; streak: number } {
|
|
@@ -253,10 +255,16 @@ function noProgressFromRuns(runs: Array<{ status: string; error_code: string | n
|
|
|
253
255
|
cause = first.error_code as NoProgressCause;
|
|
254
256
|
}
|
|
255
257
|
if (!cause) return { cause: null, streak: 0 };
|
|
258
|
+
// #2622(实测修正)—— **按窗口计数,不按连续链**。
|
|
259
|
+
// 生产 4.3 小时实测:一批排程的模式是「跳过 → 跳过 → 成功 → 跳过…」,
|
|
260
|
+
// 成功会把连续链打断,于是 streak 永远攒不到阈值,退避一次都没触发
|
|
261
|
+
// (previous_run_active 24h 从 2397/天 只降到 2258/天)。
|
|
262
|
+
// 改成「最近 N 次里同因跳过出现几次」:仍然要求最新一次是同因跳过(不干扰成功后的恢复),
|
|
263
|
+
// 但如果它在这 N 次里反复出现,就按窗口值放慢。
|
|
264
|
+
const window = runs.slice(0, NO_PROGRESS_WINDOW);
|
|
256
265
|
let streak = 0;
|
|
257
|
-
for (const r of
|
|
258
|
-
if (r.status
|
|
259
|
-
streak++;
|
|
266
|
+
for (const r of window) {
|
|
267
|
+
if (r.status === "skipped" && r.error_code === cause) streak++;
|
|
260
268
|
}
|
|
261
269
|
return { cause, streak };
|
|
262
270
|
}
|
|
@@ -108,6 +108,22 @@ describe("applyNoProgressBackoff (#2622)", () => {
|
|
|
108
108
|
expect(applyNoProgressBackoff(id)).toBeNull();
|
|
109
109
|
});
|
|
110
110
|
|
|
111
|
+
|
|
112
|
+
test("alternating skip/success pattern still backs off (window, not strict chain)", () => {
|
|
113
|
+
// 生产实测:一批排程是「跳过 → 跳过 → 成功 → 跳过…」,成功会把连续链打断,
|
|
114
|
+
// 按严格链计数永远攒不到阈值。窗口计数必须能抓住它。
|
|
115
|
+
const id = seedSchedule(180);
|
|
116
|
+
addRun(id, "skipped", "previous_run_active");
|
|
117
|
+
addRun(id, "skipped", "previous_run_active");
|
|
118
|
+
addRun(id, "replied", null); // 成功打断链
|
|
119
|
+
addRun(id, "skipped", "previous_run_active");
|
|
120
|
+
const r = applyNoProgressBackoff(id);
|
|
121
|
+
expect(r).not.toBeNull();
|
|
122
|
+
expect(r!.cause).toBe("previous_run_active");
|
|
123
|
+
expect(r!.streak).toBe(3);
|
|
124
|
+
expect(r!.effective_seconds).toBeGreaterThan(180);
|
|
125
|
+
});
|
|
126
|
+
|
|
111
127
|
test("schedules that opt into no_change_backoff are left to the other mechanism", () => {
|
|
112
128
|
const id = seedSchedule(180, { no_change_backoff: 1 });
|
|
113
129
|
for (let i = 0; i < 5; i++) addRun(id, "skipped", "target_offline");
|