@sleep2agi/commhub-server 0.9.0-preview.162 → 0.9.0-preview.164

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,6 +8,7 @@
8
8
  import { describe, expect, test } from "bun:test";
9
9
  import {
10
10
  eventLoopStats,
11
+ noteRequestServed,
11
12
  recordGap,
12
13
  resetEventLoopLagForTests,
13
14
  startEventLoopLagMonitor,
@@ -50,6 +51,16 @@ describe("event-loop-lag: recordGap", () => {
50
51
  expect(s.recent_stalls[19].gap_ms).toBe(5_005);
51
52
  });
52
53
 
54
+ test("记录这段间隔里服务了多少请求(区分“阻塞”与“timer 饥饿”)", () => {
55
+ resetEventLoopLagForTests();
56
+ recordGap(85_000, { intervalMs: 250, thresholdMs: 2_000, rssMb: 2076, served: 137 });
57
+ const s = eventLoopStats();
58
+ expect(s.recent_stalls[0].requests_since_last_tick).toBe(137);
59
+ // 0 服务数 = 这段时间真的什么都没推进
60
+ recordGap(30_000, { intervalMs: 250, thresholdMs: 2_000, rssMb: 500, served: 0 });
61
+ expect(eventLoopStats().recent_stalls[0].requests_since_last_tick).toBe(0);
62
+ });
63
+
53
64
  test("max_gap_ms 保留历史最大值(回落后不被覆盖)", () => {
54
65
  resetEventLoopLagForTests();
55
66
  recordGap(47_000, { intervalMs: 250, thresholdMs: 2_000 });
@@ -93,8 +104,9 @@ describe("event-loop-lag: startEventLoopLagMonitor", () => {
93
104
  await Bun.sleep(60);
94
105
  clearInterval(timer);
95
106
  expect(logs.length).toBeGreaterThan(0);
96
- expect(logs[0]).toStartWith("[commhub stall] loop blocked ~");
107
+ expect(logs[0]).toStartWith("[commhub stall] loop idle ~");
97
108
  expect(logs[0]).toContain("rss 4502 MB");
109
+ expect(logs[0]).toContain("served 0 req");
98
110
  });
99
111
  });
100
112
 
@@ -16,6 +16,7 @@
16
16
  // 停顿结束后第一次 tick 的 gap 就是停顿长度,不需要外部进程配合。
17
17
 
18
18
  export type EventLoopStall = {
19
+ /** 这个间隔里处理完的请求数:0 = 真的没推进;>0 = 循环在干活,只是 timer 被饿住。 */
19
20
  /** ISO 时间(停顿**结束**的时刻,即这次 tick 的时间)。 */
20
21
  at: string;
21
22
  /** 距上次 tick 的实际间隔(毫秒)。等于 interval + 停顿。 */
@@ -24,10 +25,28 @@ export type EventLoopStall = {
24
25
  stall_ms: number;
25
26
  /** 停顿结束时的进程 RSS(MB),用于和内存尖峰对齐。 */
26
27
  rss_mb: number;
28
+ requests_since_last_tick?: number;
27
29
  };
28
30
 
29
31
  const KEEP_STALLS = 20;
30
32
 
33
+ /**
34
+ * #2614(第二轮)—— 2026-10-27 00:39 线上出现 gap 84887ms,但**同一时段日志在正常写**
35
+ * (00:38:52 / 00:38:58 / 00:39:08 …),而且 `/api/stats/routes` 里最慢的请求只有 2.4s。
36
+ * 也就是说:事件循环**没有被阻塞 84 秒**,是 **timer 回调被 I/O 队列饿住**了。
37
+ * 只看 tick 间隔会把这两种情况混成一条“卡了 84 秒”的假结论。
38
+ *
39
+ * 区分办法:数一下这个间隔里**服务了多少请求**。
40
+ * requests_since_last_tick > 0 ⇒ 循环在干活 ⇒ 这是 timer 饥饿(高负载/大批回调)
41
+ * requests_since_last_tick = 0 ⇒ 真的没有推进任何请求 ⇒ 才可能是同步阻塞
42
+ */
43
+ let servedSinceTick = 0;
44
+
45
+ /** 请求处理完成时调用(route-timing 里)。只计数,不参与任何决策。 */
46
+ export function noteRequestServed(): void {
47
+ servedSinceTick++;
48
+ }
49
+
31
50
  let maxGapMs = 0;
32
51
  let lastGapMs = 0;
33
52
  let stallCount = 0;
@@ -38,7 +57,7 @@ const stalls: EventLoopStall[] = [];
38
57
  /** 记录一次 tick 间隔。返回这次 tick 的停顿量(毫秒,可为负→归零)。 */
39
58
  export function recordGap(
40
59
  gapMs: number,
41
- opts: { intervalMs?: number; thresholdMs?: number; rssMb?: number; now?: number } = {},
60
+ opts: { intervalMs?: number; thresholdMs?: number; rssMb?: number; now?: number; served?: number } = {},
42
61
  ): number {
43
62
  const interval = opts.intervalMs ?? 250;
44
63
  const threshold = opts.thresholdMs ?? 2000;
@@ -50,7 +69,13 @@ export function recordGap(
50
69
  stallCount++;
51
70
  lastStallAt = at;
52
71
  lastStallMs = stallMs;
53
- stalls.push({ at, gap_ms: Math.round(gapMs), stall_ms: Math.round(stallMs), rss_mb: opts.rssMb ?? 0 });
72
+ stalls.push({
73
+ at,
74
+ gap_ms: Math.round(gapMs),
75
+ stall_ms: Math.round(stallMs),
76
+ rss_mb: opts.rssMb ?? 0,
77
+ requests_since_last_tick: opts.served ?? 0,
78
+ });
54
79
  while (stalls.length > KEEP_STALLS) stalls.shift();
55
80
  return stallMs;
56
81
  }
@@ -109,10 +134,13 @@ export function startEventLoopLagMonitor(opts: {
109
134
  const gap = now - last;
110
135
  last = now;
111
136
  const rss = rssMb();
112
- const stall = recordGap(gap, { intervalMs, thresholdMs, rssMb: rss });
137
+ const served = servedSinceTick;
138
+ servedSinceTick = 0;
139
+ const stall = recordGap(gap, { intervalMs, thresholdMs, rssMb: rss, served });
113
140
  if (thresholdMs > 0 && stall >= thresholdMs) {
114
141
  // 和 retention/slow-sql 的日志同一前缀,方便 grep。
115
- log(`[commhub stall] loop blocked ~${Math.round(stall)}ms (tick gap ${Math.round(gap)}ms, rss ${rss} MB)`);
142
+ // `served` 是判据的关键:它非零就说明循环在干活,这段“gap”是 timer 被饿住,不是阻塞。
143
+ log(`[commhub stall] loop idle ~${Math.round(stall)}ms (tick gap ${Math.round(gap)}ms, rss ${rss} MB, served ${served} req)`);
116
144
  }
117
145
  }, intervalMs);
118
146
  // 监测不该拖住进程退出。
@@ -5,6 +5,8 @@
5
5
  // 2026-10-01 —— 原来是 4096 条样本的环:生产上 212 s 就写满,?minutes=10 实际只覆盖三分半钟。
6
6
  // 改成按分钟分桶的累计器:每个桶里每个路由一行(次数 / 总耗时 / 最大值 / 字节 / 5xx / 耗时直方图),
7
7
  // 保留 ROUTE_BUCKET_RETENTION 个桶(24 小时),窗口多长就真覆盖多长。p95 从直方图取(桶上沿,不超过 max)。
8
+ import { noteRequestServed } from "./event-loop-lag.js";
9
+
8
10
  export const ROUTE_BUCKET_MS = 60_000;
9
11
  export const ROUTE_BUCKET_RETENTION = 1440;
10
12
  /** 每个分钟桶里最多这么多个不同的路由;再多的记到 ROUTE_OVERFLOW_KEY,防止怪路径 / 怪工具名把内存撑大。 */
@@ -142,6 +144,8 @@ export function recordRouteTiming<T extends Response | undefined>(req: Request,
142
144
  const length = Number(res.headers.get("content-length"));
143
145
  const bytes = Number.isFinite(length) && res.headers.has("content-length") ? length : 0;
144
146
  addSample(route, performance.now() - started, res.status, bytes, now, callerClass(req));
147
+ // #2614 —— 让停顿监测能区分「循环被阻塞」与「timer 被饿住」:见 event-loop-lag.ts 顶部说明。
148
+ noteRequestServed();
145
149
  return res;
146
150
  }
147
151
 
@@ -0,0 +1,145 @@
1
+ // #2622 —— 定时任务「连续失败 → 间隔翻倍 → 成功即恢复」。
2
+ //
3
+ // 为什么需要它(生产 24h,2026-10-27):
4
+ // 一个 `every_seconds=180` 的排程 351 次失败 / 24 次成功 —— 失败与成功交替,
5
+ // 所以「连续失败」永远攒不到 #523 的自动暂停阈值,而**每一次失败都按 3 分钟原速重试**。
6
+ // `#523` 的机制(连续 5 次通知、可选自动暂停)都在,缺的只是「放慢」这一档。
7
+ //
8
+ // 这个文件钉住四件事:
9
+ // 1. 连续失败达到阈值 ⇒ `backoff_effective_seconds` 按 2 倍递增,且有上限;
10
+ // 2. `next_run_at` 只往后推,不会被“放大后又缩短”的抖动来回拉;
11
+ // 3. 成功一次 ⇒ 立刻清空退避,`next_run_at` 拉回正常节奏;
12
+ // 4. 与 `no_change_backoff` 互斥(那套逻辑自己拥有这一列),关掉开关时完全不介入。
13
+
14
+ import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
15
+ import { register } from "./auth.js";
16
+ import { db } from "./db.js";
17
+ import { applyFailureBackoff, failureBackoffRuns } from "./scheduled-failures.js";
18
+
19
+ const stamp = Date.now();
20
+ let NET = "";
21
+ let creatorId = "";
22
+ const ENV_KEY = "COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS";
23
+ const savedEnv = process.env[ENV_KEY];
24
+ let slot = 0;
25
+
26
+ function seedSchedule(everySeconds = 180, patch: Record<string, unknown> = {}) {
27
+ const scheduleId = `sched_bo_${crypto.randomUUID()}`;
28
+ db.run(
29
+ `INSERT INTO scheduled_tasks
30
+ (schedule_id, network_id, created_by, name, target_node_id, target_alias, task_content, priority,
31
+ schedule_type, schedule_json, timezone, status, next_run_at, overlap_policy)
32
+ VALUES (?1, ?2, ?3, 'backoff fixture', 'n_bo', 'bo-node', 'x', 'normal', 'interval', ?4,
33
+ 'Asia/Shanghai', 'active', ?5, 'skip')`,
34
+ [scheduleId, NET, creatorId, JSON.stringify({ type: "interval", every_seconds: everySeconds }),
35
+ new Date(Date.now() + everySeconds * 1000).toISOString()],
36
+ );
37
+ for (const [k, v] of Object.entries(patch)) db.run(`UPDATE scheduled_tasks SET ${k} = ?1 WHERE schedule_id = ?2`, [v as any, scheduleId]);
38
+ return scheduleId;
39
+ }
40
+
41
+ function addRun(scheduleId: string, status: string, minutesFromNow: number) {
42
+ const runId = `srun_bo_${crypto.randomUUID()}`;
43
+ db.run(
44
+ `INSERT INTO scheduled_task_runs (run_id, schedule_id, network_id, scheduled_for, status, created_at)
45
+ VALUES (?1, ?2, ?3, ?4, ?5, ?4)`,
46
+ [runId, scheduleId, NET, new Date(Date.now() + minutesFromNow * 60_000).toISOString(), status],
47
+ );
48
+ return runId;
49
+ }
50
+
51
+ const reload = (id: string) => db.get<any>("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", id)!;
52
+
53
+ beforeAll(async () => {
54
+ const auth = register(`bo_owner_${stamp}`, "Bo-Owner-Strong-1!", undefined, "tester");
55
+ expect(auth.ok).toBe(true);
56
+ NET = auth.network_id!;
57
+ creatorId = auth.user_id!;
58
+ process.env[ENV_KEY] = "6";
59
+ });
60
+
61
+ afterEach(() => {
62
+ process.env[ENV_KEY] = "6";
63
+ });
64
+
65
+ afterAll(() => {
66
+ try { db.run(`DELETE FROM scheduled_task_runs WHERE network_id = ?1`, [NET]); } catch {}
67
+ try { db.run(`DELETE FROM scheduled_tasks WHERE network_id = ?1`, [NET]); } catch {}
68
+ if (savedEnv === undefined) delete process.env[ENV_KEY]; else process.env[ENV_KEY] = savedEnv;
69
+ });
70
+
71
+ describe("#2622 failure backoff", () => {
72
+ test("默认阈值是 6,且可以关掉", () => {
73
+ delete process.env[ENV_KEY];
74
+ expect(failureBackoffRuns()).toBe(6);
75
+ process.env[ENV_KEY] = "0";
76
+ expect(failureBackoffRuns()).toBe(0);
77
+ });
78
+
79
+ test("失败次数未到阈值:不写退避", () => {
80
+ const id = seedSchedule();
81
+ for (let i = 0; i < 5; i++) addRun(id, "failed", -60 + i);
82
+ const out = applyFailureBackoff(id);
83
+ expect(out).toEqual({ streak: 5, effective_seconds: null });
84
+ expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
85
+ });
86
+
87
+ test("达到阈值后间隔翻倍,且 next_run_at 只会往后推", () => {
88
+ const id = seedSchedule(180);
89
+ for (let i = 0; i < 6; i++) addRun(id, "failed", -60 + i);
90
+ const before = reload(id).next_run_at;
91
+ const out = applyFailureBackoff(id);
92
+ expect(out!.effective_seconds).toBe(360); // 180 × 2
93
+ const after = reload(id);
94
+ expect(after.backoff_effective_seconds).toBe(360);
95
+ expect(Date.parse(after.next_run_at)).toBeGreaterThanOrEqual(Date.parse(before));
96
+ });
97
+
98
+ test("失败继续累积:指数放大且有 1 小时上限", () => {
99
+ const id = seedSchedule(180);
100
+ for (let i = 0; i < 12; i++) addRun(id, "failed", -120 + i);
101
+ const out = applyFailureBackoff(id);
102
+ // 180 × 2^k 会被 3600 截断
103
+ expect(out!.effective_seconds).toBeLessThanOrEqual(3600);
104
+ expect(out!.streak).toBe(12);
105
+ });
106
+
107
+ test("成功一次 ⇒ 立刻清空退避并把 next_run_at 拉回正常节奏", () => {
108
+ const id = seedSchedule(180);
109
+ for (let i = 0; i < 8; i++) addRun(id, "failed", -120 + i);
110
+ applyFailureBackoff(id);
111
+ expect(reload(id).backoff_effective_seconds).toBeGreaterThan(180);
112
+ addRun(id, "replied", -1);
113
+ const out = applyFailureBackoff(id);
114
+ expect(out!.streak).toBe(0);
115
+ expect(out!.effective_seconds).toBeNull();
116
+ expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
117
+ });
118
+
119
+ test("no_change_backoff 的排程不被这条逻辑改写", () => {
120
+ const id = seedSchedule(180, { no_change_backoff: 1 });
121
+ for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
122
+ expect(applyFailureBackoff(id)).toBeNull();
123
+ expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
124
+ });
125
+
126
+ test("非 interval 排程不参与", () => {
127
+ const id = seedSchedule(180, { schedule_type: "daily", schedule_json: JSON.stringify({ type: "daily", time: "09:00" }) });
128
+ for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
129
+ expect(applyFailureBackoff(id)).toBeNull();
130
+ });
131
+
132
+ test("paused 的排程不参与(暂停已经把它停住了)", () => {
133
+ const id = seedSchedule(180, { status: "paused", next_run_at: null });
134
+ for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
135
+ expect(applyFailureBackoff(id)).toBeNull();
136
+ });
137
+
138
+ test("开关设为 0 时完全不介入", () => {
139
+ const id = seedSchedule(180);
140
+ for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
141
+ process.env[ENV_KEY] = "0";
142
+ expect(applyFailureBackoff(id)).toBeNull();
143
+ expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
144
+ });
145
+ });
@@ -167,6 +167,65 @@ export function evaluateScheduleFailures(scheduleId: string, now = new Date()):
167
167
  return notice;
168
168
  }
169
169
 
170
+ /**
171
+ * #2622 —— 失败退避:连续失败达到 N 次后把**间隔翻倍**(上限 1 小时),成功一次立刻恢复原节奏。
172
+ *
173
+ * 为什么不是「暂停」:`#523` 已经有 `COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS`(默认关),
174
+ * 但它会把排程**停下来**,需要人来恢复。对“上游额度暂时打满”这类外部原因,暂停太久同样是损失;
175
+ * 退避只是**放慢重试**,外部原因一恢复就会自己跑回原节奏。
176
+ *
177
+ * 现场(2026-10-27 生产 24h):一个 `every_seconds=180` 的排程失败 351 次、成功 24 次 ——
178
+ * 失败与成功交替出现,所以“连续失败”很少攒到暂停阈值,而**每次失败都在按 3 分钟原速重试**。
179
+ *
180
+ * 存储复用 `backoff_effective_seconds`(派发时按 `max(every_seconds, backoff_effective_seconds)` 生效),
181
+ * 与 `no_change_backoff` 互斥:那套逻辑自己会写这一列,这里只在 `no_change_backoff = 0` 时介入。
182
+ *
183
+ * 环境变量:`COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS`(默认 6;0 = 关闭)。
184
+ */
185
+ export function failureBackoffRuns(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS", 6, 0); }
186
+
187
+ const FAILURE_BACKOFF_CAP_SECONDS = 3600;
188
+
189
+ export function applyFailureBackoff(scheduleId: string, now = new Date()): { streak: number; effective_seconds: number | null } | null {
190
+ const from = failureBackoffRuns();
191
+ if (from <= 0) return null;
192
+ const row = db.get<any>(
193
+ `SELECT schedule_id, revision, status, schedule_type, schedule_json, no_change_backoff,
194
+ backoff_effective_seconds, next_run_at
195
+ FROM scheduled_tasks WHERE schedule_id = ?1`,
196
+ scheduleId,
197
+ );
198
+ if (!row || row.status !== "active" || row.no_change_backoff === 1) return null;
199
+ if (row.schedule_type !== "interval") return null;
200
+ let spec: any = null;
201
+ try { spec = JSON.parse(row.schedule_json || "{}"); } catch { spec = null; }
202
+ const base = spec && spec.type === "interval" ? Number(spec.every_seconds) : NaN;
203
+ if (!Number.isFinite(base) || base <= 0) return null;
204
+
205
+ const streak = failureStreak(scheduleId).consecutiveFailures;
206
+ const effective = streak >= from
207
+ ? Math.min(FAILURE_BACKOFF_CAP_SECONDS, base * 2 ** Math.min(streak - from + 1, 6))
208
+ : null;
209
+ const target = effective ?? base;
210
+ const previous = row.backoff_effective_seconds ?? base;
211
+ let next = row.next_run_at;
212
+ if (target !== previous && next) {
213
+ // 放大:至少推到 now + target(不把已经更远的 next 拉近);
214
+ // 恢复:至少拉回 now + target(不把已经更近的 next 推远)。
215
+ const candidate = new Date(now.getTime() + target * 1000).toISOString();
216
+ const existing = Date.parse(next);
217
+ if (Number.isFinite(existing)) {
218
+ next = new Date(target > previous ? Math.max(existing, Date.parse(candidate)) : Math.min(existing, Date.parse(candidate))).toISOString();
219
+ }
220
+ }
221
+ db.run(
222
+ `UPDATE scheduled_tasks SET backoff_streak = ?1, backoff_effective_seconds = ?2, next_run_at = ?3,
223
+ updated_at = datetime('now') WHERE schedule_id = ?4 AND revision = ?5 AND status = 'active'`,
224
+ [streak, effective, next, scheduleId, row.revision],
225
+ );
226
+ return { streak, effective_seconds: effective };
227
+ }
228
+
170
229
  /** GET /runs 的附加字段。 */
171
230
  export function failureSummary(scheduleId: string): { consecutive_failures: number; failure_alert_threshold: number; last_failure_alert_at: string | null } {
172
231
  const streak = failureStreak(scheduleId);
@@ -176,4 +235,10 @@ export function failureSummary(scheduleId: string): { consecutive_failures: numb
176
235
 
177
236
  setScheduledRunTerminalHook((scheduleId, status) => {
178
237
  if (status === "failed") evaluateScheduleFailures(scheduleId);
238
+ // #2622 —— 成功一次就把间隔恢复原样;失败则按连续失败次数放大间隔。
239
+ try {
240
+ applyFailureBackoff(scheduleId);
241
+ } catch (e: any) {
242
+ console.error(`[scheduled-tasks] failure backoff failed schedule=${scheduleId}: ${e?.message || e}`);
243
+ }
179
244
  });
package/src/server.ts CHANGED
@@ -30,10 +30,12 @@ import { recentNodeIdentityConflicts } from "./node-identity-conflict.js";
30
30
  import { createSSEStream, createNetworkObserverStream, createUserEventStream, pushEvent, pushNetworkObserverEvent, getSSEStats, PRINTABLE_OBSERVER_KEY_PREFIX, closeUserStreamsInNetwork, getUserPresence, onUserPresenceChange, pushUserEventToNetwork } from "./push.js";
31
31
  import { enterMcpInflight, leaveMcpInflight, getMcpInflightStats } from "./mcp-inflight.js";
32
32
  import { eventLoopStats, startEventLoopLagMonitor } from "./event-loop-lag.js";
33
+ import { refreshAliasBaseline } from "./alias-baseline.js";
33
34
  import { assertNodeActive } from "./lifecycle-guard.js";
34
35
  import { pendingInboxCount } from "./inbox-count.js";
35
36
  import { notifyExpiredTasks, type ExpiredTaskRow } from "./task-expiry-notice.js";
36
37
  import { expireStaleOpenTasks } from "./task-stale-open.js";
38
+ import { expireDeliveredToOfflineTargets } from "./task-offline-delivery.js";
37
39
  import { flagOrphanTasks } from "./task-orphan.js";
38
40
  import { addAgentNetworkScope, addHumanNetworkScope, addNetworkScope, addOwnTrafficScope, addAgentTimelineScope, canRestWriteNetwork, canRestWriteNetworkAsHuman, getUserNetworkIds, resolveRestNetworkScope, resolveRestWriteNetworkId, singleNetworkId, type RestNetworkScope } from "./network-scope.js";
39
41
  import { pickAliasNetwork } from "./rest-alias-network-fallback.js";
@@ -983,6 +985,19 @@ export function patrolExpiredTasks(): void {
983
985
  if (notices.length > 0) console.log(`[patrol] sent ${notices.length} expiry notice(s)`);
984
986
  }
985
987
  } catch {}
988
+ // #2668 —— 投给「已离线」目标的陈年投递:超过 TTL(默认 30 分钟)就过期,并走与 TTL 相同的通知路径。
989
+ // 只处理“目标明确不在”这一种;忙 ⇒ queued_behind_busy 告警,不在这里处置。
990
+ try {
991
+ const offlineDropped = expireDeliveredToOfflineTargets();
992
+ if (offlineDropped.length > 0) {
993
+ console.log(`[patrol] expired ${offlineDropped.length} task(s) queued to an offline target (#2668)`);
994
+ for (const task of offlineDropped) logTaskEvent(task.task_id, "delivered", "expired", "patrol");
995
+ const notices = notifyExpiredTasks(offlineDropped as any);
996
+ if (notices.length > 0) console.log(`[patrol] sent ${notices.length} offline-expiry notice(s)`);
997
+ }
998
+ } catch (e: any) {
999
+ console.error(`[patrol] offline-delivery phase failed: ${e?.message || e}`);
1000
+ }
986
1001
  // #519 — acked/running with no activity for COMMHUB_STALE_OPEN_TASK_HOURS
987
1002
  // (default 0 = off) → expired, at most 500 per pass, no sender notice
988
1003
  // (task-stale-open.ts). Its own try: a failure here must not hide the TTL phase.
@@ -5643,6 +5658,20 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5643
5658
  // “同一秒里 retention 跑了 10 秒 + 有两百个请求在排队”。
5644
5659
  const eventLoopLagTimer = startEventLoopLagMonitor();
5645
5660
 
5661
+ // #2571(第三轮)—— 每个目标自己的投递延迟 p99 基线,供 delivered_stale 判据使用。
5662
+ // 生产库上重算一次约 0.46s(116k tasks / 7 天窗口),所以只每小时一次、且与 retention 错开。
5663
+ // 失败不影响任何请求路径:基线表只是缓存,读不到就退回全局阈值。
5664
+ const refreshBaseline = () => {
5665
+ try {
5666
+ const r = refreshAliasBaseline();
5667
+ if (r.rows > 0) console.log(`[commhub alias-baseline] refreshed ${r.rows} alias(es)`);
5668
+ } catch (e: any) {
5669
+ console.error(`[commhub alias-baseline] refresh failed: ${e?.message || e}`);
5670
+ }
5671
+ };
5672
+ refreshBaseline();
5673
+ const aliasBaselineTimer = setInterval(refreshBaseline, 60 * 60 * 1000);
5674
+
5646
5675
  // Round-2/4 review ③ — stale session sweeper (coexists with the
5647
5676
  // retention sweeper above; different concerns + different cadences).
5648
5677
  // Replaces the per-request UPDATE in GET /api/status (+ /api/servers,
@@ -5677,6 +5706,7 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5677
5706
  // server would still hang on live intervals it cannot reach.
5678
5707
  (retentionSweeperTimer as any)?.unref?.();
5679
5708
  (eventLoopLagTimer as any)?.unref?.();
5709
+ (aliasBaselineTimer as any)?.unref?.();
5680
5710
  (staleSweeperTimer as any)?.unref?.();
5681
5711
  (rateLimitSweepTimer as any)?.unref?.();
5682
5712
  (taskPatrolTimer as any)?.unref?.();
@@ -5689,6 +5719,7 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5689
5719
  console.log("[commhub] shutting down...");
5690
5720
  clearInterval(retentionSweeperTimer);
5691
5721
  clearInterval(eventLoopLagTimer);
5722
+ clearInterval(aliasBaselineTimer);
5692
5723
  clearInterval(staleSweeperTimer);
5693
5724
  clearInterval(rateLimitSweepTimer);
5694
5725
  clearInterval(taskPatrolTimer);
@@ -1,4 +1,5 @@
1
1
  import { db } from "./db.js";
2
+ import { ALIAS_BASELINE_TABLE, BASELINE_MAX_SECONDS, BASELINE_P99_FACTOR, ensureAliasBaselineTable } from "./alias-baseline.js";
2
3
 
3
4
  export const DELIVERED_STALE_THRESHOLDS = [600, 1800] as const;
4
5
 
@@ -44,7 +45,7 @@ function thresholdLabel(thresholdSeconds: 600 | 1800): string {
44
45
  return thresholdSeconds === 600 ? "10m" : "30m";
45
46
  }
46
47
 
47
- function recordStale(thresholdSeconds: 600 | 1800, cutoff: string): number {
48
+ function recordStale(thresholdSeconds: 600 | 1800, cutoff: string, referenceIso: string): number {
48
49
  const busyType = `task.warning.queued_behind_busy_${thresholdSeconds}s`;
49
50
  const staleType = `task.warning.delivered_stale_${thresholdSeconds}s`;
50
51
  const label = thresholdLabel(thresholdSeconds);
@@ -68,11 +69,24 @@ function recordStale(thresholdSeconds: 600 | 1800, cutoff: string): number {
68
69
  THEN ?6 ELSE ?7 END,
69
70
  network_id
70
71
  FROM tasks
72
+ -- 这个子查询把列名改成 b_* 前缀:SELECT 列表里的 network_id 必须继续是 tasks 的
73
+ -- (test683 的 L9 变异门用 network_id + FROM tasks 相邻两行当锚点,裸列名一改锚点就失效)。
74
+ LEFT JOIN (SELECT alias AS b_alias, network_id AS b_network, p99_seconds AS b_p99
75
+ FROM alias_latency_baseline) b
76
+ ON b.b_alias = tasks.to_name AND b.b_network = tasks.network_id
71
77
  WHERE status = 'delivered'
72
78
  AND delivered_at IS NOT NULL
73
79
  AND delivered_at <= ?3
80
+ -- #2571 第三轮:目标自己的 p99 基线比全局下限更宽时,用它(全队变慢不炸屏,
81
+ -- 单节点卡住仍会显形)。没有基线行 ⇒ 与 .163 行为完全一致。
82
+ AND (
83
+ b.b_p99 IS NULL
84
+ OR delivered_at <= datetime(?8, '-' || CAST(b.b_p99 * 1.5 AS INTEGER) || ' seconds')
85
+ -- 上限:再慢的节点,等满 30 分钟也必须报(否则 p99=50min 的节点永远不响)。
86
+ OR delivered_at <= datetime(?8, '-' || CAST(?9 AS INTEGER) || ' seconds')
87
+ )
74
88
  ON CONFLICT(task_id, event_key) DO NOTHING`,
75
- [staleType, staleDetail, cutoff, busyType, staleType, busyDetail, staleDetail],
89
+ [staleType, staleDetail, cutoff, busyType, staleType, busyDetail, staleDetail, referenceIso, BASELINE_MAX_SECONDS],
76
90
  );
77
91
  return result.changes;
78
92
  }
@@ -80,12 +94,13 @@ function recordStale(thresholdSeconds: 600 | 1800, cutoff: string): number {
80
94
  export function recordDeliveredStaleEvents(now = new Date()): DeliveredStalePatrolResult {
81
95
  if (!Number.isFinite(now.getTime())) throw new Error("invalid lifecycle patrol time");
82
96
 
97
+ ensureAliasBaselineTable();
83
98
  const byThreshold = { 600: 0, 1800: 0 };
84
99
  let inserted = 0;
85
100
 
86
101
  for (const thresholdSeconds of DELIVERED_STALE_THRESHOLDS) {
87
102
  const cutoff = sqliteUtcTimestamp(new Date(now.getTime() - thresholdSeconds * 1000));
88
- const changes = recordStale(thresholdSeconds, cutoff);
103
+ const changes = recordStale(thresholdSeconds, cutoff, now.toISOString());
89
104
  byThreshold[thresholdSeconds] = changes;
90
105
  inserted += changes;
91
106
  }
@@ -0,0 +1,119 @@
1
+ // #2668 —— 投给已离线节点的任务,超过 TTL(默认 30 分钟)就过期,而不是躺满 24 小时的 TTL。
2
+ //
3
+ // 现场:任务 3b3c63e0 投给 TM基建鲸 后目标立刻 offline,`delivered` 状态挂了 **2 小时 56 分**,
4
+ // 10/30 分钟两档告警都报了但没有处置;`expires_at` 是创建后 24 小时。
5
+ //
6
+ // 这个文件钉住五件事:
7
+ // 1. 目标 offline ⇒ 超时后过期;
8
+ // 2. 目标没有 session 行(未知)⇒ 同样按 offline 处理;
9
+ // 3. 目标忙(working)⇒ **不处置**(那是 queued_behind_busy 的语义);
10
+ // 4. 节点已经取走(consumed_at 有值)⇒ 不处置(交给 #758 orphan 判据);
11
+ // 5. TTL 可配且可关(COMMHUB_OFFLINE_DELIVERY_TTL_MINUTES,0 = 关闭)。
12
+
13
+ import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
14
+ import { db } from "./db.js";
15
+ import {
16
+ OFFLINE_DELIVERY_TTL_ENV,
17
+ expireDeliveredToOfflineTargets,
18
+ offlineDeliveryTtlMinutes,
19
+ } from "./task-offline-delivery.js";
20
+
21
+ const stamp = `${Date.now()}`;
22
+ let NET = "";
23
+ const NOW = new Date("2026-10-27T03:30:00.000Z");
24
+ const savedEnv = process.env[OFFLINE_DELIVERY_TTL_ENV];
25
+
26
+ function deliveredAgo(minutes: number): string {
27
+ return new Date(NOW.getTime() - minutes * 60_000).toISOString().slice(0, 19).replace("T", " ");
28
+ }
29
+
30
+ function seedSession(alias: string, status: string) {
31
+ db.run(
32
+ `INSERT INTO sessions (resume_id, alias, node_id, status, network_id, registered_at)
33
+ VALUES (?1, ?2, ?3, ?4, ?5, datetime('now'))`,
34
+ [`res_${alias}_${stamp}`, alias, `n_${alias}_${stamp}`, status, NET],
35
+ );
36
+ }
37
+
38
+ function seedTask(id: string, toName: string, deliveredMinutesAgo: number, extra: { consumed?: string } = {}) {
39
+ db.run(
40
+ `INSERT INTO tasks (task_id, from_name, to_name, status, content, delivered_at, consumed_at, expires_at, network_id)
41
+ VALUES (?1, 'sender', ?2, 'delivered', 'offline fixture', ?3, ?4, datetime('now','+1 day'), ?5)`,
42
+ [id, toName, deliveredAgo(deliveredMinutesAgo), extra.consumed ?? null, NET],
43
+ );
44
+ }
45
+
46
+ const statusOf = (id: string) => db.get<{ status: string }>("SELECT status FROM tasks WHERE task_id = ?1", [id])!.status;
47
+
48
+ beforeAll(async () => {
49
+ const { register } = await import("./auth.js");
50
+ const auth = register(`offl_owner_${stamp}`, "Offl-Owner-Strong-1!", undefined, "tester");
51
+ expect(auth.ok).toBe(true);
52
+ NET = auth.network_id!;
53
+ process.env[OFFLINE_DELIVERY_TTL_ENV] = "30";
54
+
55
+ seedSession(`offl-off-${stamp}`, "offline");
56
+ seedSession(`offl-busy-${stamp}`, "working");
57
+
58
+ seedTask(`offl-1-${stamp}`, `offl-off-${stamp}`, 45); // 离线 + 45 分钟 ⇒ 过期
59
+ seedTask(`offl-2-${stamp}`, `offl-off-${stamp}`, 10); // 离线但只 10 分钟 ⇒ 不动
60
+ seedTask(`offl-3-${stamp}`, `offl-busy-${stamp}`, 45); // 目标忙 ⇒ 不动
61
+ seedTask(`offl-4-${stamp}`, `offl-missing-${stamp}`, 45); // 没有 session 行 ⇒ 视为离线 ⇒ 过期
62
+ seedTask(`offl-5-${stamp}`, `offl-off-${stamp}`, 45, { consumed: deliveredAgo(46) }); // 已被取走 ⇒ 不动
63
+ });
64
+
65
+ afterEach(() => { process.env[OFFLINE_DELIVERY_TTL_ENV] = "30"; });
66
+
67
+ afterAll(() => {
68
+ try { db.run(`DELETE FROM tasks WHERE task_id LIKE ?1`, [`offl-%-${stamp}`]); } catch {}
69
+ try { db.run(`DELETE FROM sessions WHERE alias LIKE ?1`, [`offl-%-${stamp}`]); } catch {}
70
+ if (savedEnv === undefined) delete process.env[OFFLINE_DELIVERY_TTL_ENV];
71
+ else process.env[OFFLINE_DELIVERY_TTL_ENV] = savedEnv;
72
+ });
73
+
74
+ describe("#2668 delivered-to-offline expiry", () => {
75
+ test("默认 TTL 30 分钟,可配、可关", () => {
76
+ delete process.env[OFFLINE_DELIVERY_TTL_ENV];
77
+ expect(offlineDeliveryTtlMinutes()).toBe(30);
78
+ process.env[OFFLINE_DELIVERY_TTL_ENV] = "10";
79
+ expect(offlineDeliveryTtlMinutes()).toBe(10);
80
+ process.env[OFFLINE_DELIVERY_TTL_ENV] = "0";
81
+ expect(offlineDeliveryTtlMinutes()).toBe(0);
82
+ });
83
+
84
+ test("目标离线且超过 TTL ⇒ 过期;未超时/目标忙/已取走 ⇒ 都不动", () => {
85
+ const dropped = expireDeliveredToOfflineTargets(NOW);
86
+ const ids = dropped.map(d => d.task_id);
87
+ expect(ids).toContain(`offl-1-${stamp}`);
88
+ expect(ids).toContain(`offl-4-${stamp}`);
89
+ expect(ids).not.toContain(`offl-2-${stamp}`);
90
+ expect(ids).not.toContain(`offl-3-${stamp}`);
91
+ expect(ids).not.toContain(`offl-5-${stamp}`);
92
+
93
+ expect(statusOf(`offl-1-${stamp}`)).toBe("expired");
94
+ expect(statusOf(`offl-4-${stamp}`)).toBe("expired");
95
+ expect(statusOf(`offl-2-${stamp}`)).toBe("delivered");
96
+ expect(statusOf(`offl-3-${stamp}`)).toBe("delivered");
97
+ expect(statusOf(`offl-5-${stamp}`)).toBe("delivered");
98
+ });
99
+
100
+ test("返回行带 waited_minutes(通知文案要用)", () => {
101
+ seedTask(`offl-6-${stamp}`, `offl-off-${stamp}`, 90);
102
+ const dropped = expireDeliveredToOfflineTargets(NOW);
103
+ const row = dropped.find(d => d.task_id === `offl-6-${stamp}`);
104
+ expect(row).toBeTruthy();
105
+ expect(row!.waited_minutes).toBeGreaterThanOrEqual(89);
106
+ });
107
+
108
+ test("第二次调用幂等(已过期的不再出现)", () => {
109
+ const again = expireDeliveredToOfflineTargets(NOW);
110
+ expect(again.map(d => d.task_id)).not.toContain(`offl-1-${stamp}`);
111
+ });
112
+
113
+ test("TTL=0 时整个阶段关闭", () => {
114
+ seedTask(`offl-7-${stamp}`, `offl-off-${stamp}`, 999);
115
+ process.env[OFFLINE_DELIVERY_TTL_ENV] = "0";
116
+ expect(expireDeliveredToOfflineTargets(NOW)).toEqual([]);
117
+ expect(statusOf(`offl-7-${stamp}`)).toBe("delivered");
118
+ });
119
+ });