@sleep2agi/commhub-server 0.9.0-preview.161 → 0.9.0-preview.163

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,150 @@
1
+ // #2614 —— 事件循环停顿监测(只观测,不改任何行为)。
2
+ //
3
+ // 背景:2026-10-26 21:55–22:12 生产 Hub 出现两次内存尖峰(RSS 3.6 → 4.5 GB),
4
+ // 伴随机 `mcp_inflight` 387 → 807。事后看日志才发现**同一时刻日志静默 14–47 秒**
5
+ // ——Hub 是单线程(Bun + 同步 SQLite),日志静默就等于事件循环被阻塞。
6
+ //
7
+ // 现有 `/api/stats/routes` 只能告诉我们“哪个请求**受害**了”(它的耗时包含了排队时间),
8
+ // 不能告诉我们“当时**谁**在阻塞”。缺的就是停顿本身的时间序列。
9
+ //
10
+ // 这个模块补上这一格:每 `intervalMs` 醒一次,量 `now - lastTick`;
11
+ // 正常情况 ≈ interval,超出 `thresholdMs` 的部分就是一次真实停顿。
12
+ // 停顿会:① 计数/留档(`/health` 的 `event_loop` 字段可读);
13
+ // ② 打一行日志(带当时的 RSS,方便和内存曲线对齐)。
14
+ //
15
+ // 注意:`setInterval` 自己也会被阻塞 —— 这正是我们要测量的东西。
16
+ // 停顿结束后第一次 tick 的 gap 就是停顿长度,不需要外部进程配合。
17
+
18
+ export type EventLoopStall = {
19
+ /** 这个间隔里处理完的请求数:0 = 真的没推进;>0 = 循环在干活,只是 timer 被饿住。 */
20
+ /** ISO 时间(停顿**结束**的时刻,即这次 tick 的时间)。 */
21
+ at: string;
22
+ /** 距上次 tick 的实际间隔(毫秒)。等于 interval + 停顿。 */
23
+ gap_ms: number;
24
+ /** 停顿量 = gap - interval。 */
25
+ stall_ms: number;
26
+ /** 停顿结束时的进程 RSS(MB),用于和内存尖峰对齐。 */
27
+ rss_mb: number;
28
+ requests_since_last_tick?: number;
29
+ };
30
+
31
+ const KEEP_STALLS = 20;
32
+
33
+ /**
34
+ * #2614(第二轮)—— 2026-10-27 00:39 线上出现 gap 84887ms,但**同一时段日志在正常写**
35
+ * (00:38:52 / 00:38:58 / 00:39:08 …),而且 `/api/stats/routes` 里最慢的请求只有 2.4s。
36
+ * 也就是说:事件循环**没有被阻塞 84 秒**,是 **timer 回调被 I/O 队列饿住**了。
37
+ * 只看 tick 间隔会把这两种情况混成一条“卡了 84 秒”的假结论。
38
+ *
39
+ * 区分办法:数一下这个间隔里**服务了多少请求**。
40
+ * requests_since_last_tick > 0 ⇒ 循环在干活 ⇒ 这是 timer 饥饿(高负载/大批回调)
41
+ * requests_since_last_tick = 0 ⇒ 真的没有推进任何请求 ⇒ 才可能是同步阻塞
42
+ */
43
+ let servedSinceTick = 0;
44
+
45
+ /** 请求处理完成时调用(route-timing 里)。只计数,不参与任何决策。 */
46
+ export function noteRequestServed(): void {
47
+ servedSinceTick++;
48
+ }
49
+
50
+ let maxGapMs = 0;
51
+ let lastGapMs = 0;
52
+ let stallCount = 0;
53
+ let lastStallAt: string | null = null;
54
+ let lastStallMs = 0;
55
+ const stalls: EventLoopStall[] = [];
56
+
57
+ /** 记录一次 tick 间隔。返回这次 tick 的停顿量(毫秒,可为负→归零)。 */
58
+ export function recordGap(
59
+ gapMs: number,
60
+ opts: { intervalMs?: number; thresholdMs?: number; rssMb?: number; now?: number; served?: number } = {},
61
+ ): number {
62
+ const interval = opts.intervalMs ?? 250;
63
+ const threshold = opts.thresholdMs ?? 2000;
64
+ lastGapMs = gapMs;
65
+ if (gapMs > maxGapMs) maxGapMs = gapMs;
66
+ const stallMs = Math.max(0, gapMs - interval);
67
+ if (stallMs < threshold) return stallMs;
68
+ const at = new Date(opts.now ?? Date.now()).toISOString();
69
+ stallCount++;
70
+ lastStallAt = at;
71
+ lastStallMs = stallMs;
72
+ stalls.push({
73
+ at,
74
+ gap_ms: Math.round(gapMs),
75
+ stall_ms: Math.round(stallMs),
76
+ rss_mb: opts.rssMb ?? 0,
77
+ requests_since_last_tick: opts.served ?? 0,
78
+ });
79
+ while (stalls.length > KEEP_STALLS) stalls.shift();
80
+ return stallMs;
81
+ }
82
+
83
+ export function eventLoopStats(): {
84
+ max_gap_ms: number;
85
+ last_gap_ms: number;
86
+ stall_count: number;
87
+ last_stall_ms: number;
88
+ last_stall_at: string | null;
89
+ recent_stalls: EventLoopStall[];
90
+ } {
91
+ return {
92
+ max_gap_ms: Math.round(maxGapMs),
93
+ last_gap_ms: Math.round(lastGapMs),
94
+ stall_count: stallCount,
95
+ last_stall_ms: Math.round(lastStallMs),
96
+ last_stall_at: lastStallAt,
97
+ recent_stalls: [...stalls].reverse(),
98
+ };
99
+ }
100
+
101
+ /** 只给测试用:清空累计状态。 */
102
+ export function resetEventLoopLagForTests(): void {
103
+ maxGapMs = 0;
104
+ lastGapMs = 0;
105
+ stallCount = 0;
106
+ lastStallAt = null;
107
+ lastStallMs = 0;
108
+ stalls.length = 0;
109
+ }
110
+
111
+ function envMs(name: string, fallback: number): number {
112
+ const raw = Number(process.env[name]);
113
+ return Number.isFinite(raw) && raw >= 0 ? raw : fallback;
114
+ }
115
+
116
+ /**
117
+ * 启动监测。返回 timer(调用方负责 clearInterval)。
118
+ * - `COMMHUB_STALL_LOG_MS`:停顿日志阈值,默认 2000 ms,设 0 关闭日志(仍计数);
119
+ * - `COMMHUB_STALL_TICK_MS`:采样间隔,默认 250 ms。
120
+ */
121
+ export function startEventLoopLagMonitor(opts: {
122
+ intervalMs?: number;
123
+ thresholdMs?: number;
124
+ log?: (msg: string) => void;
125
+ rssMb?: () => number;
126
+ } = {}): ReturnType<typeof setInterval> {
127
+ const intervalMs = opts.intervalMs ?? Math.max(50, envMs("COMMHUB_STALL_TICK_MS", 250));
128
+ const thresholdMs = opts.thresholdMs ?? envMs("COMMHUB_STALL_LOG_MS", 2000);
129
+ const log = opts.log ?? ((m: string) => console.log(m));
130
+ const rssMb = opts.rssMb ?? (() => Math.round(process.memoryUsage().rss / 1048576));
131
+ let last = performance.now();
132
+ const timer = setInterval(() => {
133
+ const now = performance.now();
134
+ const gap = now - last;
135
+ last = now;
136
+ const rss = rssMb();
137
+ const served = servedSinceTick;
138
+ servedSinceTick = 0;
139
+ const stall = recordGap(gap, { intervalMs, thresholdMs, rssMb: rss, served });
140
+ if (thresholdMs > 0 && stall >= thresholdMs) {
141
+ // 和 retention/slow-sql 的日志同一前缀,方便 grep。
142
+ // `served` 是判据的关键:它非零就说明循环在干活,这段“gap”是 timer 被饿住,不是阻塞。
143
+ log(`[commhub stall] loop idle ~${Math.round(stall)}ms (tick gap ${Math.round(gap)}ms, rss ${rss} MB, served ${served} req)`);
144
+ }
145
+ }, intervalMs);
146
+ // 监测不该拖住进程退出。
147
+ (timer as unknown as { unref?: () => void }).unref?.();
148
+ last = performance.now();
149
+ return timer;
150
+ }
@@ -107,10 +107,16 @@ describe("#473 requirements_people", () => {
107
107
  test("compact rows: username, display_name, role, department, owned agents; unowned agents listed once", async () => {
108
108
  const r = await asBoss.call("requirements_people", { network_id: netA, q: "alice" });
109
109
  expect(r.ok).toBe(true);
110
- expect(r.people).toEqual([{
110
+ // #2596 —— Agent 行新增 updated_at / lifecycle_state(客户端用它做 alias 去重与“近 7 天”过滤)。
111
+ // 形状断言改为「关键字段匹配 + 允许新增键」,而不是整行严格相等。
112
+ expect(r.people).toHaveLength(1);
113
+ expect(r.people[0]).toMatchObject({
111
114
  user_id: alice.id, username: alice.username, display_name: "爱丽丝", role: "member",
112
- department: { id: deptId, name: `研发部${stamp % 1000}` }, agents: [{ node_id: node("alice"), alias: ALIAS.alice }],
113
- }]);
115
+ department: { id: deptId, name: `研发部${stamp % 1000}` },
116
+ });
117
+ expect(r.people[0].agents).toHaveLength(1);
118
+ expect(r.people[0].agents[0]).toMatchObject({ node_id: node("alice"), alias: ALIAS.alice });
119
+ expect(r.people[0].agents[0]).toHaveProperty("updated_at");
114
120
  const all = await asBoss.call("requirements_people", { network_id: netA, q: String(stamp) });
115
121
  expect(all.agents_without_owner.map((a: any) => a.alias).sort()).toEqual([ALIAS.dup, ALIAS.dup, ALIAS.free].sort());
116
122
  const bossRow = (await asBoss.call("requirements_people", { network_id: netA, q: boss.username })).people[0];
@@ -19,7 +19,22 @@ export const PEOPLE_MAX_LIMIT = 200;
19
19
 
20
20
  /** `department_id`: Agent 的**显式**部门归属(可空)。空 = 仍按「负责人在哪个部门」推断,
21
21
  * 客户端据此把 Agent 放对格子 —— 2026-10-19 前没有这一列,328 个 Agent 只能全算「未归属」。 */
22
- export type DirectoryAgent = { node_id: string; alias: string; department_id?: string | null };
22
+ export type DirectoryAgent = {
23
+ node_id: string;
24
+ alias: string;
25
+ department_id?: string | null;
26
+ /**
27
+ * #2596 —— 这一行“最后一次被 Hub 看到”的时间(`nodes.updated_at`),以及生命周期状态。
28
+ *
29
+ * 为什么加:组织架构页把 368 个 agent 全塞进「未归属」,其中 125 个是 5–8 月的历史残留
30
+ * (如 `海报设计师1–5号` 注册于 2026-05-17、早已 offline)。客户端要能“默认只显示近 7 天活跃”
31
+ * 与“同一 alias 只保留最新的那条”,就必须拿到时间轴——否则只能靠 node_id 猜。
32
+ *
33
+ * 只增不改:老客户端忽略未知键,形状断言已在同批测试里放宽。
34
+ */
35
+ updated_at?: string | null;
36
+ lifecycle_state?: string | null;
37
+ };
23
38
  export type DirectoryPerson = {
24
39
  user_id: string;
25
40
  username: string;
@@ -64,8 +79,8 @@ export function peopleDirectory(networkId: string, hidden: HiddenNode, query: {
64
79
  ORDER BY u.username, u.user_id`,
65
80
  networkId,
66
81
  );
67
- const nodes = db.all<{ node_id: string; alias: string | null; node_name: string; owner_user_id: string | null; department_id: string | null }>(
68
- "SELECT node_id, alias, node_name, owner_user_id, department_id FROM nodes WHERE network_id = ?1 ORDER BY COALESCE(alias, node_name), node_id",
82
+ const nodes = db.all<{ node_id: string; alias: string | null; node_name: string; owner_user_id: string | null; department_id: string | null; updated_at: string | null; lifecycle_state: string | null }>(
83
+ "SELECT node_id, alias, node_name, owner_user_id, department_id, updated_at, lifecycle_state FROM nodes WHERE network_id = ?1 ORDER BY COALESCE(alias, node_name), node_id",
69
84
  networkId,
70
85
  ).filter(n => !hidden?.(n.node_id));
71
86
  const memberIds = new Set(members.map(m => m.user_id));
@@ -75,7 +90,14 @@ export function peopleDirectory(networkId: string, hidden: HiddenNode, query: {
75
90
  // 只在**显式**归属时带这一格:没设 = 没有显式归属(顶层 agent_department_supported 告诉
76
91
  // 客户端这一点)。行形状保持不变 —— 旧客户端拿到的行还是老样子(多一个键会让严格要求
77
92
  // 形状的测试/客户端意外),只是新客户端不再把「负责人在哪个部门」当成 Agent 的部门。
78
- const agent = { node_id: n.node_id, alias: n.alias || n.node_name, ...(n.department_id ? { department_id: n.department_id } : {}) };
93
+ const agent = {
94
+ node_id: n.node_id,
95
+ alias: n.alias || n.node_name,
96
+ ...(n.department_id ? { department_id: n.department_id } : {}),
97
+ // #2596 —— 客户端据此做「按 alias 去重(保留最新)」与「默认只看近 7 天」。
98
+ updated_at: n.updated_at ?? null,
99
+ lifecycle_state: n.lifecycle_state ?? null,
100
+ };
79
101
  if (n.owner_user_id && memberIds.has(n.owner_user_id)) {
80
102
  const list = byOwner.get(n.owner_user_id) ?? [];
81
103
  list.push(agent);
@@ -5,6 +5,8 @@
5
5
  // 2026-10-01 —— 原来是 4096 条样本的环:生产上 212 s 就写满,?minutes=10 实际只覆盖三分半钟。
6
6
  // 改成按分钟分桶的累计器:每个桶里每个路由一行(次数 / 总耗时 / 最大值 / 字节 / 5xx / 耗时直方图),
7
7
  // 保留 ROUTE_BUCKET_RETENTION 个桶(24 小时),窗口多长就真覆盖多长。p95 从直方图取(桶上沿,不超过 max)。
8
+ import { noteRequestServed } from "./event-loop-lag.js";
9
+
8
10
  export const ROUTE_BUCKET_MS = 60_000;
9
11
  export const ROUTE_BUCKET_RETENTION = 1440;
10
12
  /** 每个分钟桶里最多这么多个不同的路由;再多的记到 ROUTE_OVERFLOW_KEY,防止怪路径 / 怪工具名把内存撑大。 */
@@ -142,6 +144,8 @@ export function recordRouteTiming<T extends Response | undefined>(req: Request,
142
144
  const length = Number(res.headers.get("content-length"));
143
145
  const bytes = Number.isFinite(length) && res.headers.has("content-length") ? length : 0;
144
146
  addSample(route, performance.now() - started, res.status, bytes, now, callerClass(req));
147
+ // #2614 —— 让停顿监测能区分「循环被阻塞」与「timer 被饿住」:见 event-loop-lag.ts 顶部说明。
148
+ noteRequestServed();
145
149
  return res;
146
150
  }
147
151
 
@@ -0,0 +1,145 @@
1
+ // #2622 —— 定时任务「连续失败 → 间隔翻倍 → 成功即恢复」。
2
+ //
3
+ // 为什么需要它(生产 24h,2026-10-27):
4
+ // 一个 `every_seconds=180` 的排程 351 次失败 / 24 次成功 —— 失败与成功交替,
5
+ // 所以「连续失败」永远攒不到 #523 的自动暂停阈值,而**每一次失败都按 3 分钟原速重试**。
6
+ // `#523` 的机制(连续 5 次通知、可选自动暂停)都在,缺的只是「放慢」这一档。
7
+ //
8
+ // 这个文件钉住四件事:
9
+ // 1. 连续失败达到阈值 ⇒ `backoff_effective_seconds` 按 2 倍递增,且有上限;
10
+ // 2. `next_run_at` 只往后推,不会被“放大后又缩短”的抖动来回拉;
11
+ // 3. 成功一次 ⇒ 立刻清空退避,`next_run_at` 拉回正常节奏;
12
+ // 4. 与 `no_change_backoff` 互斥(那套逻辑自己拥有这一列),关掉开关时完全不介入。
13
+
14
+ import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
15
+ import { register } from "./auth.js";
16
+ import { db } from "./db.js";
17
+ import { applyFailureBackoff, failureBackoffRuns } from "./scheduled-failures.js";
18
+
19
+ const stamp = Date.now();
20
+ let NET = "";
21
+ let creatorId = "";
22
+ const ENV_KEY = "COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS";
23
+ const savedEnv = process.env[ENV_KEY];
24
+ let slot = 0;
25
+
26
+ function seedSchedule(everySeconds = 180, patch: Record<string, unknown> = {}) {
27
+ const scheduleId = `sched_bo_${crypto.randomUUID()}`;
28
+ db.run(
29
+ `INSERT INTO scheduled_tasks
30
+ (schedule_id, network_id, created_by, name, target_node_id, target_alias, task_content, priority,
31
+ schedule_type, schedule_json, timezone, status, next_run_at, overlap_policy)
32
+ VALUES (?1, ?2, ?3, 'backoff fixture', 'n_bo', 'bo-node', 'x', 'normal', 'interval', ?4,
33
+ 'Asia/Shanghai', 'active', ?5, 'skip')`,
34
+ [scheduleId, NET, creatorId, JSON.stringify({ type: "interval", every_seconds: everySeconds }),
35
+ new Date(Date.now() + everySeconds * 1000).toISOString()],
36
+ );
37
+ for (const [k, v] of Object.entries(patch)) db.run(`UPDATE scheduled_tasks SET ${k} = ?1 WHERE schedule_id = ?2`, [v as any, scheduleId]);
38
+ return scheduleId;
39
+ }
40
+
41
+ function addRun(scheduleId: string, status: string, minutesFromNow: number) {
42
+ const runId = `srun_bo_${crypto.randomUUID()}`;
43
+ db.run(
44
+ `INSERT INTO scheduled_task_runs (run_id, schedule_id, network_id, scheduled_for, status, created_at)
45
+ VALUES (?1, ?2, ?3, ?4, ?5, ?4)`,
46
+ [runId, scheduleId, NET, new Date(Date.now() + minutesFromNow * 60_000).toISOString(), status],
47
+ );
48
+ return runId;
49
+ }
50
+
51
+ const reload = (id: string) => db.get<any>("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", id)!;
52
+
53
+ beforeAll(async () => {
54
+ const auth = register(`bo_owner_${stamp}`, "Bo-Owner-Strong-1!", undefined, "tester");
55
+ expect(auth.ok).toBe(true);
56
+ NET = auth.network_id!;
57
+ creatorId = auth.user_id!;
58
+ process.env[ENV_KEY] = "6";
59
+ });
60
+
61
+ afterEach(() => {
62
+ process.env[ENV_KEY] = "6";
63
+ });
64
+
65
+ afterAll(() => {
66
+ try { db.run(`DELETE FROM scheduled_task_runs WHERE network_id = ?1`, [NET]); } catch {}
67
+ try { db.run(`DELETE FROM scheduled_tasks WHERE network_id = ?1`, [NET]); } catch {}
68
+ if (savedEnv === undefined) delete process.env[ENV_KEY]; else process.env[ENV_KEY] = savedEnv;
69
+ });
70
+
71
+ describe("#2622 failure backoff", () => {
72
+ test("默认阈值是 6,且可以关掉", () => {
73
+ delete process.env[ENV_KEY];
74
+ expect(failureBackoffRuns()).toBe(6);
75
+ process.env[ENV_KEY] = "0";
76
+ expect(failureBackoffRuns()).toBe(0);
77
+ });
78
+
79
+ test("失败次数未到阈值:不写退避", () => {
80
+ const id = seedSchedule();
81
+ for (let i = 0; i < 5; i++) addRun(id, "failed", -60 + i);
82
+ const out = applyFailureBackoff(id);
83
+ expect(out).toEqual({ streak: 5, effective_seconds: null });
84
+ expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
85
+ });
86
+
87
+ test("达到阈值后间隔翻倍,且 next_run_at 只会往后推", () => {
88
+ const id = seedSchedule(180);
89
+ for (let i = 0; i < 6; i++) addRun(id, "failed", -60 + i);
90
+ const before = reload(id).next_run_at;
91
+ const out = applyFailureBackoff(id);
92
+ expect(out!.effective_seconds).toBe(360); // 180 × 2
93
+ const after = reload(id);
94
+ expect(after.backoff_effective_seconds).toBe(360);
95
+ expect(Date.parse(after.next_run_at)).toBeGreaterThanOrEqual(Date.parse(before));
96
+ });
97
+
98
+ test("失败继续累积:指数放大且有 1 小时上限", () => {
99
+ const id = seedSchedule(180);
100
+ for (let i = 0; i < 12; i++) addRun(id, "failed", -120 + i);
101
+ const out = applyFailureBackoff(id);
102
+ // 180 × 2^k 会被 3600 截断
103
+ expect(out!.effective_seconds).toBeLessThanOrEqual(3600);
104
+ expect(out!.streak).toBe(12);
105
+ });
106
+
107
+ test("成功一次 ⇒ 立刻清空退避并把 next_run_at 拉回正常节奏", () => {
108
+ const id = seedSchedule(180);
109
+ for (let i = 0; i < 8; i++) addRun(id, "failed", -120 + i);
110
+ applyFailureBackoff(id);
111
+ expect(reload(id).backoff_effective_seconds).toBeGreaterThan(180);
112
+ addRun(id, "replied", -1);
113
+ const out = applyFailureBackoff(id);
114
+ expect(out!.streak).toBe(0);
115
+ expect(out!.effective_seconds).toBeNull();
116
+ expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
117
+ });
118
+
119
+ test("no_change_backoff 的排程不被这条逻辑改写", () => {
120
+ const id = seedSchedule(180, { no_change_backoff: 1 });
121
+ for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
122
+ expect(applyFailureBackoff(id)).toBeNull();
123
+ expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
124
+ });
125
+
126
+ test("非 interval 排程不参与", () => {
127
+ const id = seedSchedule(180, { schedule_type: "daily", schedule_json: JSON.stringify({ type: "daily", time: "09:00" }) });
128
+ for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
129
+ expect(applyFailureBackoff(id)).toBeNull();
130
+ });
131
+
132
+ test("paused 的排程不参与(暂停已经把它停住了)", () => {
133
+ const id = seedSchedule(180, { status: "paused", next_run_at: null });
134
+ for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
135
+ expect(applyFailureBackoff(id)).toBeNull();
136
+ });
137
+
138
+ test("开关设为 0 时完全不介入", () => {
139
+ const id = seedSchedule(180);
140
+ for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
141
+ process.env[ENV_KEY] = "0";
142
+ expect(applyFailureBackoff(id)).toBeNull();
143
+ expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
144
+ });
145
+ });
@@ -167,6 +167,65 @@ export function evaluateScheduleFailures(scheduleId: string, now = new Date()):
167
167
  return notice;
168
168
  }
169
169
 
170
+ /**
171
+ * #2622 —— 失败退避:连续失败达到 N 次后把**间隔翻倍**(上限 1 小时),成功一次立刻恢复原节奏。
172
+ *
173
+ * 为什么不是「暂停」:`#523` 已经有 `COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS`(默认关),
174
+ * 但它会把排程**停下来**,需要人来恢复。对“上游额度暂时打满”这类外部原因,暂停太久同样是损失;
175
+ * 退避只是**放慢重试**,外部原因一恢复就会自己跑回原节奏。
176
+ *
177
+ * 现场(2026-10-27 生产 24h):一个 `every_seconds=180` 的排程失败 351 次、成功 24 次 ——
178
+ * 失败与成功交替出现,所以“连续失败”很少攒到暂停阈值,而**每次失败都在按 3 分钟原速重试**。
179
+ *
180
+ * 存储复用 `backoff_effective_seconds`(派发时按 `max(every_seconds, backoff_effective_seconds)` 生效),
181
+ * 与 `no_change_backoff` 互斥:那套逻辑自己会写这一列,这里只在 `no_change_backoff = 0` 时介入。
182
+ *
183
+ * 环境变量:`COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS`(默认 6;0 = 关闭)。
184
+ */
185
+ export function failureBackoffRuns(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS", 6, 0); }
186
+
187
+ const FAILURE_BACKOFF_CAP_SECONDS = 3600;
188
+
189
+ export function applyFailureBackoff(scheduleId: string, now = new Date()): { streak: number; effective_seconds: number | null } | null {
190
+ const from = failureBackoffRuns();
191
+ if (from <= 0) return null;
192
+ const row = db.get<any>(
193
+ `SELECT schedule_id, revision, status, schedule_type, schedule_json, no_change_backoff,
194
+ backoff_effective_seconds, next_run_at
195
+ FROM scheduled_tasks WHERE schedule_id = ?1`,
196
+ scheduleId,
197
+ );
198
+ if (!row || row.status !== "active" || row.no_change_backoff === 1) return null;
199
+ if (row.schedule_type !== "interval") return null;
200
+ let spec: any = null;
201
+ try { spec = JSON.parse(row.schedule_json || "{}"); } catch { spec = null; }
202
+ const base = spec && spec.type === "interval" ? Number(spec.every_seconds) : NaN;
203
+ if (!Number.isFinite(base) || base <= 0) return null;
204
+
205
+ const streak = failureStreak(scheduleId).consecutiveFailures;
206
+ const effective = streak >= from
207
+ ? Math.min(FAILURE_BACKOFF_CAP_SECONDS, base * 2 ** Math.min(streak - from + 1, 6))
208
+ : null;
209
+ const target = effective ?? base;
210
+ const previous = row.backoff_effective_seconds ?? base;
211
+ let next = row.next_run_at;
212
+ if (target !== previous && next) {
213
+ // 放大:至少推到 now + target(不把已经更远的 next 拉近);
214
+ // 恢复:至少拉回 now + target(不把已经更近的 next 推远)。
215
+ const candidate = new Date(now.getTime() + target * 1000).toISOString();
216
+ const existing = Date.parse(next);
217
+ if (Number.isFinite(existing)) {
218
+ next = new Date(target > previous ? Math.max(existing, Date.parse(candidate)) : Math.min(existing, Date.parse(candidate))).toISOString();
219
+ }
220
+ }
221
+ db.run(
222
+ `UPDATE scheduled_tasks SET backoff_streak = ?1, backoff_effective_seconds = ?2, next_run_at = ?3,
223
+ updated_at = datetime('now') WHERE schedule_id = ?4 AND revision = ?5 AND status = 'active'`,
224
+ [streak, effective, next, scheduleId, row.revision],
225
+ );
226
+ return { streak, effective_seconds: effective };
227
+ }
228
+
170
229
  /** GET /runs 的附加字段。 */
171
230
  export function failureSummary(scheduleId: string): { consecutive_failures: number; failure_alert_threshold: number; last_failure_alert_at: string | null } {
172
231
  const streak = failureStreak(scheduleId);
@@ -176,4 +235,10 @@ export function failureSummary(scheduleId: string): { consecutive_failures: numb
176
235
 
177
236
  setScheduledRunTerminalHook((scheduleId, status) => {
178
237
  if (status === "failed") evaluateScheduleFailures(scheduleId);
238
+ // #2622 —— 成功一次就把间隔恢复原样;失败则按连续失败次数放大间隔。
239
+ try {
240
+ applyFailureBackoff(scheduleId);
241
+ } catch (e: any) {
242
+ console.error(`[scheduled-tasks] failure backoff failed schedule=${scheduleId}: ${e?.message || e}`);
243
+ }
179
244
  });
package/src/server.ts CHANGED
@@ -29,6 +29,7 @@ import { deriveTeamRulesView } from "./team-rules-status.js";
29
29
  import { recentNodeIdentityConflicts } from "./node-identity-conflict.js";
30
30
  import { createSSEStream, createNetworkObserverStream, createUserEventStream, pushEvent, pushNetworkObserverEvent, getSSEStats, PRINTABLE_OBSERVER_KEY_PREFIX, closeUserStreamsInNetwork, getUserPresence, onUserPresenceChange, pushUserEventToNetwork } from "./push.js";
31
31
  import { enterMcpInflight, leaveMcpInflight, getMcpInflightStats } from "./mcp-inflight.js";
32
+ import { eventLoopStats, startEventLoopLagMonitor } from "./event-loop-lag.js";
32
33
  import { assertNodeActive } from "./lifecycle-guard.js";
33
34
  import { pendingInboxCount } from "./inbox-count.js";
34
35
  import { notifyExpiredTasks, type ExpiredTaskRow } from "./task-expiry-notice.js";
@@ -2788,6 +2789,11 @@ return Bun.serve({
2788
2789
  // 而 `sse_connections` 只反映长连接,看不到“同秒打进来的一批请求”。
2789
2790
  mcp_inflight: getMcpInflightStats().inflight,
2790
2791
  mcp_inflight_peak: getMcpInflightStats().peak,
2792
+ // #2614 —— 事件循环停顿。`mcp_inflight` 只能说明“有多少请求在排队”,
2793
+ // 说明不了“服务端为什么卡”。这里给出停顿本身:max_gap_ms 是进程内
2794
+ // 单次 tick 的最大间隔(正常 ≈250ms),recent_stalls 保留最近 20 次
2795
+ // 超过阈值的停顿(含当时 RSS),用于和内存曲线对齐。
2796
+ event_loop: eventLoopStats(),
2791
2797
  auth: DEV_OPEN ? "dev-open" : "user-token",
2792
2798
  security: DEV_OPEN ? "dev-open" : "secured",
2793
2799
  tmux: TMUX_ENABLED ? "enabled" : "disabled",
@@ -5631,6 +5637,12 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5631
5637
  : 60 * 60 * 1000;
5632
5638
  const retentionSweeperTimer = startRetentionSweeper(sweepIntervalMs);
5633
5639
 
5640
+ // #2614 —— 事件循环停顿监测。放这里是因为 startHub() 才是“真的在跑一个 Hub”
5641
+ // 的入口(测试用 bootServer() 不会背这个 timer)。停顿日志与
5642
+ // `[commhub retention]` / `[commhub slow-sql]` 共用前缀族,便于一次 grep 出
5643
+ // “同一秒里 retention 跑了 10 秒 + 有两百个请求在排队”。
5644
+ const eventLoopLagTimer = startEventLoopLagMonitor();
5645
+
5634
5646
  // Round-2/4 review ③ — stale session sweeper (coexists with the
5635
5647
  // retention sweeper above; different concerns + different cadences).
5636
5648
  // Replaces the per-request UPDATE in GET /api/status (+ /api/servers,
@@ -5664,6 +5676,7 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5664
5676
  // jobs must not — otherwise a test (or a future caller) that stops the
5665
5677
  // server would still hang on live intervals it cannot reach.
5666
5678
  (retentionSweeperTimer as any)?.unref?.();
5679
+ (eventLoopLagTimer as any)?.unref?.();
5667
5680
  (staleSweeperTimer as any)?.unref?.();
5668
5681
  (rateLimitSweepTimer as any)?.unref?.();
5669
5682
  (taskPatrolTimer as any)?.unref?.();
@@ -5675,6 +5688,7 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5675
5688
  function shutdown() {
5676
5689
  console.log("[commhub] shutting down...");
5677
5690
  clearInterval(retentionSweeperTimer);
5691
+ clearInterval(eventLoopLagTimer);
5678
5692
  clearInterval(staleSweeperTimer);
5679
5693
  clearInterval(rateLimitSweepTimer);
5680
5694
  clearInterval(taskPatrolTimer);
@@ -1,4 +1,4 @@
1
- // #2571 —— delivered_stale 的**分流**:目标在跑别的任务时不再报“可能离线”。
1
+ // #2571 —— delivered_stale 的**分流**(阈值 10m/30m,见 task-lifecycle-watcher.ts 顶部说明):目标在跑别的任务时不再报“可能离线”。
2
2
  //
3
3
  // 生产数据(24h):出现过 delivered_stale 的任务里 83% 最终成功,等待 p50 2.9 分钟、
4
4
  // p90 27 分钟 ⇒ 事件事实成立,但把“目标正忙”写成“可能离线”会制造告警疲劳。
@@ -11,7 +11,7 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test";
11
11
  import { db } from "./db.js";
12
12
  import { recordDeliveredStaleEvents } from "./task-lifecycle-watcher.js";
13
13
 
14
- const NOW = new Date("2026-08-10T00:01:01.000Z");
14
+ const NOW = new Date("2026-08-10T01:00:01.000Z");
15
15
  let networkId = "";
16
16
 
17
17
  function sqliteUtcTimestamp(date: Date): string {
@@ -48,15 +48,15 @@ beforeAll(async () => {
48
48
 
49
49
  // 目标在跑别的任务
50
50
  insertSession("target-busy", "working");
51
- insertTask("busy-60", "target-busy", "2026-08-10 00:00:01");
51
+ insertTask("busy-600", "target-busy", "2026-08-10 00:50:01");
52
52
  // 目标空闲却没开始(真异常)
53
53
  insertSession("target-idle", "idle");
54
- insertTask("idle-60", "target-idle", "2026-08-10 00:00:01");
54
+ insertTask("idle-600", "target-idle", "2026-08-10 00:50:01");
55
55
  // 目标掉线
56
56
  insertSession("target-offline", "offline");
57
- insertTask("offline-60", "target-offline", "2026-08-10 00:00:01");
57
+ insertTask("offline-600", "target-offline", "2026-08-10 00:50:01");
58
58
  // 完全没有 session 行(未知)
59
- insertTask("unknown-60", "target-without-session", "2026-08-10 00:00:01");
59
+ insertTask("unknown-600", "target-without-session", "2026-08-10 00:50:01");
60
60
  }, 30_000);
61
61
 
62
62
  afterAll(() => {
@@ -70,17 +70,17 @@ afterAll(() => {
70
70
  describe("#2571 stale vs queued-behind-busy", () => {
71
71
  test("目标 working ⇒ 只写 queued_behind_busy_*,不写 delivered_stale_*", () => {
72
72
  recordDeliveredStaleEvents(NOW);
73
- const events = eventsFor("busy-60");
74
- expect(events).toContain("task.warning.queued_behind_busy_60s");
75
- expect(events).not.toContain("task.warning.delivered_stale_60s");
73
+ const events = eventsFor("busy-600");
74
+ expect(events).toContain("task.warning.queued_behind_busy_600s");
75
+ expect(events).not.toContain("task.warning.delivered_stale_600s");
76
76
  });
77
77
 
78
78
  test("目标 idle / offline / 无 session ⇒ 只写 delivered_stale_*", () => {
79
79
  recordDeliveredStaleEvents(NOW);
80
- for (const taskId of ["idle-60", "offline-60", "unknown-60"]) {
80
+ for (const taskId of ["idle-600", "offline-600", "unknown-600"]) {
81
81
  const events = eventsFor(taskId);
82
- expect(events).toContain("task.warning.delivered_stale_60s");
83
- expect(events).not.toContain("task.warning.queued_behind_busy_60s");
82
+ expect(events).toContain("task.warning.delivered_stale_600s");
83
+ expect(events).not.toContain("task.warning.queued_behind_busy_600s");
84
84
  }
85
85
  });
86
86
 
@@ -100,10 +100,10 @@ describe("#2571 stale vs queued-behind-busy", () => {
100
100
 
101
101
  test("detail 文案能区分“排队”和“没在跑”", () => {
102
102
  const busy = db.get<{ detail: string }>(
103
- `SELECT detail FROM task_events WHERE task_id='busy-60' AND event_type LIKE 'task.warning.queued_behind_busy_%'`,
103
+ `SELECT detail FROM task_events WHERE task_id='busy-600' AND event_type LIKE 'task.warning.queued_behind_busy_%'`,
104
104
  );
105
105
  const stale = db.get<{ detail: string }>(
106
- `SELECT detail FROM task_events WHERE task_id='idle-60' AND event_type LIKE 'task.warning.delivered_stale_%'`,
106
+ `SELECT detail FROM task_events WHERE task_id='idle-600' AND event_type LIKE 'task.warning.delivered_stale_%'`,
107
107
  );
108
108
  expect(busy!.detail).toContain("queued after");
109
109
  expect(stale!.detail).toContain("still delivered");