@sleep2agi/commhub-server 0.9.0-preview.163 → 0.9.0-preview.165

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sleep2agi/commhub-server",
3
- "version": "0.9.0-preview.163",
3
+ "version": "0.9.0-preview.165",
4
4
  "description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
5
5
  "type": "module",
6
6
  "main": "src/index.ts",
@@ -0,0 +1,152 @@
1
+ // #2571(第三轮)—— 告警阈值从「全局常数」改成「每个目标自己的 p99 基线」。
2
+ //
3
+ // 现场(2026-10-27):同一个 `10 分钟没消费就报警` 的判据,在两种节奏下表现相反 ——
4
+ // 7 天基线:delivered→首次活动 p90=6s、p99=12min ⇒ 超过 600s 只占 1.2%
5
+ // 某一晚: p90=411s、p99=1573s ⇒ 超过 600s 占 5.4%,其中 43% 最终成功
6
+ // 全队一起变慢时,全局常数会炸一屏;而单个节点卡死时,又必须能显形。
7
+ //
8
+ // 这个文件钉住四件事:
9
+ // 1. 没有基线(样本不足/老库)⇒ **与 .163 完全一致**(沿用全局下限,不漏报不误报);
10
+ // 2. 基线比全局更宽(慢节点)⇒ 更晚才报;
11
+ // 3. 基线比全局更窄(快节点)⇒ **仍然不早于全局下限**(不允许比 600s 更敏感);
12
+ // 4. 基线只由**成功且已进 runtime** 的样本算出来,且样本不足不写行。
13
+
14
+ import { afterAll, beforeAll, describe, expect, test } from "bun:test";
15
+ import { db } from "./db.js";
16
+ import {
17
+ ALIAS_BASELINE_TABLE,
18
+ BASELINE_MIN_SAMPLES,
19
+ aliasBaseline,
20
+ ensureAliasBaselineTable,
21
+ refreshAliasBaseline,
22
+ } from "./alias-baseline.js";
23
+ import { recordDeliveredStaleEvents } from "./task-lifecycle-watcher.js";
24
+
25
+ const stamp = `${Date.now()}`;
26
+ let NET = "";
27
+ const NOW = new Date("2026-10-27T03:00:00.000Z");
28
+
29
+ function iso(secondsAgo: number): string {
30
+ return new Date(NOW.getTime() - secondsAgo * 1000).toISOString().slice(0, 19).replace("T", " ");
31
+ }
32
+
33
+ function seedTask(id: string, toName: string, deliveredSecondsAgo: number) {
34
+ db.run(
35
+ `INSERT INTO tasks (task_id, from_name, to_name, status, content, delivered_at, network_id)
36
+ VALUES (?1, 'sender', ?2, 'delivered', 'baseline fixture', ?3, ?4)`,
37
+ [id, toName, iso(deliveredSecondsAgo), NET],
38
+ );
39
+ }
40
+
41
+ function seedSession(alias: string, status: string) {
42
+ db.run(
43
+ `INSERT INTO sessions (resume_id, alias, node_id, status, network_id, registered_at)
44
+ VALUES (?1, ?2, ?3, ?4, ?5, datetime('now'))`,
45
+ [`res_${alias}_${stamp}`, alias, `n_${alias}_${stamp}`, status, NET],
46
+ );
47
+ }
48
+
49
+ function setBaseline(alias: string, p99: number, samples = 100) {
50
+ ensureAliasBaselineTable();
51
+ db.run(
52
+ `INSERT INTO ${ALIAS_BASELINE_TABLE} (alias, network_id, p99_seconds, samples, computed_at)
53
+ VALUES (?1, ?2, ?3, ?4, ?5)
54
+ ON CONFLICT(alias, network_id) DO UPDATE SET p99_seconds = excluded.p99_seconds, samples = excluded.samples`,
55
+ [alias, NET, p99, samples, NOW.toISOString()],
56
+ );
57
+ }
58
+
59
+ const eventsFor = (taskId: string) =>
60
+ db.all<{ event_type: string }>("SELECT event_type FROM task_events WHERE task_id = ?1", [taskId]).map(r => r.event_type);
61
+
62
+ beforeAll(async () => {
63
+ const { register } = await import("./auth.js");
64
+ const auth = register(`baseline_owner_${stamp}`, "Baseline-Owner-Strong-1!", undefined, "tester");
65
+ expect(auth.ok).toBe(true);
66
+ NET = auth.network_id!;
67
+ ensureAliasBaselineTable();
68
+ });
69
+
70
+ afterAll(() => {
71
+ try {
72
+ db.run(`DELETE FROM task_events WHERE task_id LIKE ?1`, [`bl-%-${stamp}`]);
73
+ db.run(`DELETE FROM tasks WHERE task_id LIKE ?1`, [`bl-%-${stamp}`]);
74
+ db.run(`DELETE FROM sessions WHERE alias LIKE ?1`, [`bl-%-${stamp}`]);
75
+ db.run(`DELETE FROM ${ALIAS_BASELINE_TABLE} WHERE alias LIKE ?1`, [`bl-%-${stamp}`]);
76
+ } catch {}
77
+ });
78
+
79
+ describe("#2571 alias p99 baseline", () => {
80
+ test("没有基线 ⇒ 沿用全局下限(与 .163 行为一致)", () => {
81
+ seedSession(`bl-nobase-${stamp}`, "idle");
82
+ seedTask(`bl-nobase-task-${stamp}`, `bl-nobase-${stamp}`, 700);
83
+ recordDeliveredStaleEvents(NOW);
84
+ expect(eventsFor(`bl-nobase-task-${stamp}`)).toContain("task.warning.delivered_stale_600s");
85
+ });
86
+
87
+ test("基线更宽(慢节点):700 秒不报,超过 p99×1.5 才报", () => {
88
+ seedSession(`bl-slow-${stamp}`, "idle");
89
+ setBaseline(`bl-slow-${stamp}`, 3600); // 阈值 5400s
90
+ seedTask(`bl-slow-700-${stamp}`, `bl-slow-${stamp}`, 700);
91
+ seedTask(`bl-slow-6000-${stamp}`, `bl-slow-${stamp}`, 6000);
92
+ recordDeliveredStaleEvents(NOW);
93
+ expect(eventsFor(`bl-slow-700-${stamp}`)).toEqual([]);
94
+ expect(eventsFor(`bl-slow-6000-${stamp}`)).toContain("task.warning.delivered_stale_600s");
95
+ });
96
+
97
+ test("基线极大(p99=50min)时,等满 30 分钟仍然报(上限兜底)", () => {
98
+ seedSession(`bl-very-slow-${stamp}`, "idle");
99
+ setBaseline(`bl-very-slow-${stamp}`, 3017); // ×1.5 = 4525s,若无上限就永远不报
100
+ seedTask(`bl-very-slow-1900-${stamp}`, `bl-very-slow-${stamp}`, 1900);
101
+ seedTask(`bl-very-slow-1700-${stamp}`, `bl-very-slow-${stamp}`, 1700);
102
+ recordDeliveredStaleEvents(NOW);
103
+ expect(eventsFor(`bl-very-slow-1900-${stamp}`)).toContain("task.warning.delivered_stale_600s");
104
+ expect(eventsFor(`bl-very-slow-1700-${stamp}`)).toEqual([]);
105
+ });
106
+
107
+ test("基线更窄(快节点):不允许比全局下限更敏感", () => {
108
+ seedSession(`bl-fast-${stamp}`, "idle");
109
+ setBaseline(`bl-fast-${stamp}`, 2); // 阈值 max(600, 3) = 600
110
+ seedTask(`bl-fast-120-${stamp}`, `bl-fast-${stamp}`, 120);
111
+ recordDeliveredStaleEvents(NOW);
112
+ expect(eventsFor(`bl-fast-120-${stamp}`)).toEqual([]);
113
+ });
114
+
115
+ test("refresh 只用成功且已进 runtime 的样本", () => {
116
+ const alias = `bl-refresh-${stamp}`;
117
+ // 20 条成功的(delivered→started 各 10s),2 条失败的(不应计入)
118
+ for (let i = 0; i < BASELINE_MIN_SAMPLES; i++) {
119
+ db.run(
120
+ `INSERT INTO tasks (task_id, from_name, to_name, status, content, delivered_at, started_at, completed_at, network_id)
121
+ VALUES (?1, 'sender', ?2, 'replied', 'x', ?3, ?4, ?5, ?6)`,
122
+ [`bl-ref-ok-${i}-${stamp}`, alias, iso(1000 - i), iso(990 - i), iso(985 - i), NET],
123
+ );
124
+ }
125
+ for (let i = 0; i < 5; i++) {
126
+ db.run(
127
+ `INSERT INTO tasks (task_id, from_name, to_name, status, content, delivered_at, completed_at, network_id)
128
+ VALUES (?1, 'sender', ?2, 'failed', 'x', ?3, ?4, ?5)`,
129
+ [`bl-ref-bad-${i}-${stamp}`, alias, iso(5000 - i), iso(1000), NET],
130
+ );
131
+ }
132
+ refreshAliasBaseline(NOW);
133
+ const row = aliasBaseline(alias, NET);
134
+ expect(row).not.toBeNull();
135
+ expect(row!.samples).toBe(BASELINE_MIN_SAMPLES); // 失败的不算
136
+ expect(row!.p99Seconds).toBeGreaterThanOrEqual(9); // 10s 左右的样本
137
+ expect(row!.p99Seconds).toBeLessThanOrEqual(11);
138
+ });
139
+
140
+ test("样本不足不写基线行(退回全局)", () => {
141
+ const alias = `bl-thin-${stamp}`;
142
+ for (let i = 0; i < BASELINE_MIN_SAMPLES - 1; i++) {
143
+ db.run(
144
+ `INSERT INTO tasks (task_id, from_name, to_name, status, content, delivered_at, started_at, completed_at, network_id)
145
+ VALUES (?1, 'sender', ?2, 'replied', 'x', ?3, ?4, ?5, ?6)`,
146
+ [`bl-thin-${i}-${stamp}`, alias, iso(1000 - i), iso(995 - i), iso(990 - i), NET],
147
+ );
148
+ }
149
+ refreshAliasBaseline(NOW);
150
+ expect(aliasBaseline(alias, NET)).toBeNull();
151
+ });
152
+ });
@@ -0,0 +1,102 @@
1
+ // #2571(第三轮)—— 每个目标自己的「投递→首次活动」p99 基线。
2
+ //
3
+ // 为什么需要(2026-10-27 实测):
4
+ // 同一个「10 分钟没消费就报警」的判据,在两种节奏下表现完全相反 ——
5
+ // 7 天基线:delivered→首次活动 p50=1s p90=6s p99=12min ⇒ >600s 只占 1.2%
6
+ // 某一晚: p50=32s p90=411s p99=1573s ⇒ >600s 占 5.4%,其中 43% 最终成功
7
+ // 全局常数不可能同时适配“全员都在跑长 turn”和“某个节点单独卡死”。
8
+ // ⇒ 判据改成 **age > max(全局下限, 该 alias 自己的 p99 × 1.5)**:
9
+ // 全队一起变慢 ⇒ 各自基线一起抬高,不炸屏;单个节点变慢 ⇒ 它自己的基线没高,立刻显形。
10
+ //
11
+ // 这张表只是**缓存**(派生数据):算错了重算即可,不影响投递/回复语义。
12
+ // 样本不足(< MIN_SAMPLES)的 alias 不写行 ⇒ 巡逻退回全局阈值(行为与 .163 完全一致)。
13
+
14
+ import { db } from "./db.js";
15
+
16
+ export const ALIAS_BASELINE_TABLE = "alias_latency_baseline";
17
+ export const BASELINE_WINDOW_DAYS = 7;
18
+ export const BASELINE_MIN_SAMPLES = 20;
19
+ /** 阈值 = max(全局下限, p99 × 这个倍数)。1.5 是“比它自己最慢的 1% 再宽 50%”。 */
20
+ export const BASELINE_P99_FACTOR = 1.5;
21
+
22
+ /**
23
+ * 基线的**上限**:再慢的节点,等满 30 分钟也必须报。
24
+ *
25
+ * 没有这条上限时,p99=3017s(50 分钟)的节点会把自己的阈值推到 75 分钟 —— 它卡死时永远不报。
26
+ * 生产实测(2026-10-27 用最近 24h 数据回溯模拟):加基线后 46 条 → 8 条(-83%),
27
+ * 但被抑制的样本里出现了 `TMA设计师` 两条最终 **failed** 的任务(它的 p99 是 50 分钟)
28
+ * ⇒ 慢节点需要一条“无论如何都要响”的底线。
29
+ */
30
+ export const BASELINE_MAX_SECONDS = 1800;
31
+
32
+ export function ensureAliasBaselineTable(): void {
33
+ db.exec(`
34
+ CREATE TABLE IF NOT EXISTS ${ALIAS_BASELINE_TABLE} (
35
+ alias TEXT NOT NULL,
36
+ network_id TEXT NOT NULL,
37
+ p99_seconds INTEGER NOT NULL,
38
+ samples INTEGER NOT NULL,
39
+ computed_at TEXT NOT NULL,
40
+ PRIMARY KEY (alias, network_id)
41
+ );
42
+ CREATE INDEX IF NOT EXISTS idx_alias_baseline_computed
43
+ ON ${ALIAS_BASELINE_TABLE}(computed_at);
44
+ `);
45
+ }
46
+
47
+ /**
48
+ * 重新计算基线。只统计**成功任务**(replied)且已经进过 runtime 的样本:
49
+ * 失败/未开始的任务会把“没消费”混进“消费慢”,两者不是一回事。
50
+ */
51
+ export function refreshAliasBaseline(now = new Date()): { rows: number } {
52
+ ensureAliasBaselineTable();
53
+ const iso = now.toISOString();
54
+ const since = new Date(now.getTime() - BASELINE_WINDOW_DAYS * 86_400_000).toISOString();
55
+ const result = db.run(
56
+ `INSERT INTO ${ALIAS_BASELINE_TABLE} (alias, network_id, p99_seconds, samples, computed_at)
57
+ SELECT alias, network_id, CAST(seconds AS INTEGER), samples, ?2
58
+ FROM (
59
+ SELECT to_name AS alias,
60
+ network_id,
61
+ seconds,
62
+ samples,
63
+ ROW_NUMBER() OVER (PARTITION BY to_name, network_id ORDER BY seconds) AS rn,
64
+ CAST(samples * 0.99 AS INTEGER) AS p99_rank
65
+ FROM (
66
+ SELECT to_name,
67
+ network_id,
68
+ (julianday(COALESCE(started_at, runtime_submitted_at, completed_at)) - julianday(delivered_at)) * 86400.0 AS seconds,
69
+ COUNT(*) OVER (PARTITION BY to_name, network_id) AS samples
70
+ FROM tasks
71
+ WHERE status = 'replied'
72
+ AND delivered_at IS NOT NULL
73
+ AND COALESCE(started_at, runtime_submitted_at, completed_at) IS NOT NULL
74
+ AND created_at >= ?1
75
+ )
76
+ )
77
+ WHERE samples >= ?3
78
+ AND rn = MAX(1, p99_rank)
79
+ ON CONFLICT(alias, network_id) DO UPDATE SET
80
+ p99_seconds = excluded.p99_seconds,
81
+ samples = excluded.samples,
82
+ computed_at = excluded.computed_at`,
83
+ [since, iso, BASELINE_MIN_SAMPLES],
84
+ );
85
+ // 窗口内没有样本的 alias(长期不活跃)不该继续用旧基线兜着。
86
+ db.run(`DELETE FROM ${ALIAS_BASELINE_TABLE} WHERE computed_at < ?1`, [since]);
87
+ return { rows: result.changes };
88
+ }
89
+
90
+ /** 只读查询(排障 / 测试用)。 */
91
+ export function aliasBaseline(alias: string, networkId: string): { p99Seconds: number; samples: number } | null {
92
+ try {
93
+ const row = db.get<{ p99_seconds: number; samples: number }>(
94
+ `SELECT p99_seconds, samples FROM ${ALIAS_BASELINE_TABLE} WHERE alias = ?1 AND network_id = ?2`,
95
+ alias, networkId,
96
+ );
97
+ return row ? { p99Seconds: Number(row.p99_seconds), samples: Number(row.samples) } : null;
98
+ } catch {
99
+ // 表还没建(老库/单测里没跑过 refresh)⇒ 当作没有基线。
100
+ return null;
101
+ }
102
+ }
package/src/db.ts CHANGED
@@ -1467,7 +1467,41 @@ export function backfillMessageNodeIds() {
1467
1467
  console.log(`[commhub] node_id backfill: inbox=${inboxDirect + inboxRenamed}, tasks.to=${tasksToDirect + tasksToRenamed}, tasks.from=${tasksFromDirect + tasksFromRenamed}, total=${total}, took=${Math.round(performance.now() - t0)}ms`);
1468
1468
  }
1469
1469
 
1470
- backfillMessageNodeIds();
1470
+ // #2664 —— 不在 import 时跑(见 runDeferredBootBackfills 的说明)。这里留一行**准确**的日志:
1471
+ // · 有测试/运维会 grep `node_id backfill` 来确认迁移路径被走到过;
1472
+ // · 真正的汇总行(带 took=)在 runDeferredBootBackfills() 里,启动后 15 秒出现。
1473
+ console.log("[commhub] node_id backfill: deferred until after start (#2664)");
1474
+
1475
+ /**
1476
+ * #2664 —— 两个 boot 期的历史回填,挪到「服务已开始监听之后」再跑。
1477
+ *
1478
+ * 为什么:这两条语句在生产上合计约 3.5 秒(慢 SQL 日志实测:`UPDATE tasks … FROM` 1511ms +
1479
+ * `INSERT OR IGNORE INTO task_terminal_events` 2137ms),而它们只修**历史缺口**——
1480
+ * 新行由触发器实时写入。把 3.5 秒挡在启动路径上,等于让 369 个会话的重连风暴多等 3.5 秒,
1481
+ * 而这段时间里 Hub 本来就最脆弱。
1482
+ *
1483
+ * 调用点:startHub() 监听成功后延迟数秒执行(见 server.ts)。
1484
+ */
1485
+ export function runDeferredBootBackfills(): { nodeIdsMs: number; terminalEventsMs: number } {
1486
+ const t0 = performance.now();
1487
+ try { backfillMessageNodeIds(); } catch (e: any) {
1488
+ console.error(`[commhub] node_id backfill failed: ${e?.message || e}`);
1489
+ }
1490
+ const t1 = performance.now();
1491
+ try {
1492
+ if (db.dialect === "sqlite") {
1493
+ db.run(`INSERT OR IGNORE INTO task_terminal_events (task_id, network_id, from_node_id, completed_at)
1494
+ SELECT task_id, network_id, from_node_id, COALESCE(completed_at, created_at)
1495
+ FROM tasks
1496
+ WHERE status IN ('replied', 'failed', 'cancelled', 'expired')
1497
+ ORDER BY COALESCE(completed_at, created_at), task_id`);
1498
+ }
1499
+ } catch (e: any) {
1500
+ console.error(`[commhub] task_terminal_events backfill failed: ${e?.message || e}`);
1501
+ }
1502
+ const t2 = performance.now();
1503
+ return { nodeIdsMs: Math.round(t1 - t0), terminalEventsMs: Math.round(t2 - t1) };
1504
+ }
1471
1505
 
1472
1506
  // ── P0: sessions alias uniqueness is network-scoped.
1473
1507
  // Older SQLite databases created `alias TEXT UNIQUE`, which prevents the same
@@ -1656,11 +1690,8 @@ if (db.dialect === "sqlite") {
1656
1690
  VALUES (NEW.task_id, NEW.network_id, NEW.from_node_id, COALESCE(NEW.completed_at, datetime('now')));
1657
1691
  END;
1658
1692
  `);
1659
- db.run(`INSERT OR IGNORE INTO task_terminal_events (task_id, network_id, from_node_id, completed_at)
1660
- SELECT task_id, network_id, from_node_id, COALESCE(completed_at, created_at)
1661
- FROM tasks
1662
- WHERE status IN ('replied', 'failed', 'cancelled', 'expired')
1663
- ORDER BY COALESCE(completed_at, created_at), task_id`);
1693
+ // #2664 —— 回填本身挪到 runDeferredBootBackfills()(启动后再跑)。这里只留触发器定义:
1694
+ // 触发器负责**新**行的写入,回填只修历史缺口,不该占启动的关键路径。
1664
1695
  } else {
1665
1696
  db.exec(`
1666
1697
  CREATE OR REPLACE FUNCTION record_task_terminal_event() RETURNS trigger AS $$
package/src/server.ts CHANGED
@@ -23,17 +23,19 @@ import { normalizeSessionRuntime } from "./runtime-label.js";
23
23
  import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
24
24
  import { z } from "zod/v4";
25
25
  import { registerTools } from "./tools.js";
26
- import { db, logTaskEvent, logAudit, syncScheduledRunForTask } from "./db.js";
26
+ import { db, logTaskEvent, logAudit, syncScheduledRunForTask, runDeferredBootBackfills } from "./db.js";
27
27
  import { ensureNetworkRulesOnce, readNetworkRules, writeNetworkRules } from "./network-rules.js";
28
28
  import { deriveTeamRulesView } from "./team-rules-status.js";
29
29
  import { recentNodeIdentityConflicts } from "./node-identity-conflict.js";
30
30
  import { createSSEStream, createNetworkObserverStream, createUserEventStream, pushEvent, pushNetworkObserverEvent, getSSEStats, PRINTABLE_OBSERVER_KEY_PREFIX, closeUserStreamsInNetwork, getUserPresence, onUserPresenceChange, pushUserEventToNetwork } from "./push.js";
31
31
  import { enterMcpInflight, leaveMcpInflight, getMcpInflightStats } from "./mcp-inflight.js";
32
32
  import { eventLoopStats, startEventLoopLagMonitor } from "./event-loop-lag.js";
33
+ import { refreshAliasBaseline } from "./alias-baseline.js";
33
34
  import { assertNodeActive } from "./lifecycle-guard.js";
34
35
  import { pendingInboxCount } from "./inbox-count.js";
35
36
  import { notifyExpiredTasks, type ExpiredTaskRow } from "./task-expiry-notice.js";
36
37
  import { expireStaleOpenTasks } from "./task-stale-open.js";
38
+ import { expireDeliveredToOfflineTargets } from "./task-offline-delivery.js";
37
39
  import { flagOrphanTasks } from "./task-orphan.js";
38
40
  import { addAgentNetworkScope, addHumanNetworkScope, addNetworkScope, addOwnTrafficScope, addAgentTimelineScope, canRestWriteNetwork, canRestWriteNetworkAsHuman, getUserNetworkIds, resolveRestNetworkScope, resolveRestWriteNetworkId, singleNetworkId, type RestNetworkScope } from "./network-scope.js";
39
41
  import { pickAliasNetwork } from "./rest-alias-network-fallback.js";
@@ -983,6 +985,19 @@ export function patrolExpiredTasks(): void {
983
985
  if (notices.length > 0) console.log(`[patrol] sent ${notices.length} expiry notice(s)`);
984
986
  }
985
987
  } catch {}
988
+ // #2668 —— 投给「已离线」目标的陈年投递:超过 TTL(默认 30 分钟)就过期,并走与 TTL 相同的通知路径。
989
+ // 只处理“目标明确不在”这一种;忙 ⇒ queued_behind_busy 告警,不在这里处置。
990
+ try {
991
+ const offlineDropped = expireDeliveredToOfflineTargets();
992
+ if (offlineDropped.length > 0) {
993
+ console.log(`[patrol] expired ${offlineDropped.length} task(s) queued to an offline target (#2668)`);
994
+ for (const task of offlineDropped) logTaskEvent(task.task_id, "delivered", "expired", "patrol");
995
+ const notices = notifyExpiredTasks(offlineDropped as any);
996
+ if (notices.length > 0) console.log(`[patrol] sent ${notices.length} offline-expiry notice(s)`);
997
+ }
998
+ } catch (e: any) {
999
+ console.error(`[patrol] offline-delivery phase failed: ${e?.message || e}`);
1000
+ }
986
1001
  // #519 — acked/running with no activity for COMMHUB_STALE_OPEN_TASK_HOURS
987
1002
  // (default 0 = off) → expired, at most 500 per pass, no sender notice
988
1003
  // (task-stale-open.ts). Its own try: a failure here must not hide the TTL phase.
@@ -5643,6 +5658,33 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5643
5658
  // “同一秒里 retention 跑了 10 秒 + 有两百个请求在排队”。
5644
5659
  const eventLoopLagTimer = startEventLoopLagMonitor();
5645
5660
 
5661
+ // #2664 —— 两个历史回填(node_id 映射 / task_terminal_events)原来在 import 阶段跑,
5662
+ // 生产实测合计约 3.5 秒,正好落在会话重连风暴窗口里。现在等监听稳定后再跑:
5663
+ // 它们只修历史缺口,新行由触发器实时写入。
5664
+ const deferredBackfillTimer = setTimeout(() => {
5665
+ try {
5666
+ const r = runDeferredBootBackfills();
5667
+ console.log(`[commhub] deferred boot backfills: node_id=${r.nodeIdsMs}ms terminal_events=${r.terminalEventsMs}ms`);
5668
+ } catch (e: any) {
5669
+ console.error(`[commhub] deferred boot backfills failed: ${e?.message || e}`);
5670
+ }
5671
+ }, Number(process.env.COMMHUB_BOOT_BACKFILL_DELAY_MS) >= 0 ? Number(process.env.COMMHUB_BOOT_BACKFILL_DELAY_MS) : 15_000);
5672
+ (deferredBackfillTimer as any)?.unref?.();
5673
+
5674
+ // #2571(第三轮)—— 每个目标自己的投递延迟 p99 基线,供 delivered_stale 判据使用。
5675
+ // 生产库上重算一次约 0.46s(116k tasks / 7 天窗口),所以只每小时一次、且与 retention 错开。
5676
+ // 失败不影响任何请求路径:基线表只是缓存,读不到就退回全局阈值。
5677
+ const refreshBaseline = () => {
5678
+ try {
5679
+ const r = refreshAliasBaseline();
5680
+ if (r.rows > 0) console.log(`[commhub alias-baseline] refreshed ${r.rows} alias(es)`);
5681
+ } catch (e: any) {
5682
+ console.error(`[commhub alias-baseline] refresh failed: ${e?.message || e}`);
5683
+ }
5684
+ };
5685
+ refreshBaseline();
5686
+ const aliasBaselineTimer = setInterval(refreshBaseline, 60 * 60 * 1000);
5687
+
5646
5688
  // Round-2/4 review ③ — stale session sweeper (coexists with the
5647
5689
  // retention sweeper above; different concerns + different cadences).
5648
5690
  // Replaces the per-request UPDATE in GET /api/status (+ /api/servers,
@@ -5677,6 +5719,7 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5677
5719
  // server would still hang on live intervals it cannot reach.
5678
5720
  (retentionSweeperTimer as any)?.unref?.();
5679
5721
  (eventLoopLagTimer as any)?.unref?.();
5722
+ (aliasBaselineTimer as any)?.unref?.();
5680
5723
  (staleSweeperTimer as any)?.unref?.();
5681
5724
  (rateLimitSweepTimer as any)?.unref?.();
5682
5725
  (taskPatrolTimer as any)?.unref?.();
@@ -5689,6 +5732,8 @@ export function startHub(opts?: { port?: number; hostname?: string }): ReturnTyp
5689
5732
  console.log("[commhub] shutting down...");
5690
5733
  clearInterval(retentionSweeperTimer);
5691
5734
  clearInterval(eventLoopLagTimer);
5735
+ clearInterval(aliasBaselineTimer);
5736
+ clearTimeout(deferredBackfillTimer);
5692
5737
  clearInterval(staleSweeperTimer);
5693
5738
  clearInterval(rateLimitSweepTimer);
5694
5739
  clearInterval(taskPatrolTimer);
@@ -1,4 +1,5 @@
1
1
  import { db } from "./db.js";
2
+ import { ALIAS_BASELINE_TABLE, BASELINE_MAX_SECONDS, BASELINE_P99_FACTOR, ensureAliasBaselineTable } from "./alias-baseline.js";
2
3
 
3
4
  export const DELIVERED_STALE_THRESHOLDS = [600, 1800] as const;
4
5
 
@@ -44,7 +45,7 @@ function thresholdLabel(thresholdSeconds: 600 | 1800): string {
44
45
  return thresholdSeconds === 600 ? "10m" : "30m";
45
46
  }
46
47
 
47
- function recordStale(thresholdSeconds: 600 | 1800, cutoff: string): number {
48
+ function recordStale(thresholdSeconds: 600 | 1800, cutoff: string, referenceIso: string): number {
48
49
  const busyType = `task.warning.queued_behind_busy_${thresholdSeconds}s`;
49
50
  const staleType = `task.warning.delivered_stale_${thresholdSeconds}s`;
50
51
  const label = thresholdLabel(thresholdSeconds);
@@ -68,11 +69,24 @@ function recordStale(thresholdSeconds: 600 | 1800, cutoff: string): number {
68
69
  THEN ?6 ELSE ?7 END,
69
70
  network_id
70
71
  FROM tasks
72
+ -- 这个子查询把列名改成 b_* 前缀:SELECT 列表里的 network_id 必须继续是 tasks 的
73
+ -- (test683 的 L9 变异门用 network_id + FROM tasks 相邻两行当锚点,裸列名一改锚点就失效)。
74
+ LEFT JOIN (SELECT alias AS b_alias, network_id AS b_network, p99_seconds AS b_p99
75
+ FROM alias_latency_baseline) b
76
+ ON b.b_alias = tasks.to_name AND b.b_network = tasks.network_id
71
77
  WHERE status = 'delivered'
72
78
  AND delivered_at IS NOT NULL
73
79
  AND delivered_at <= ?3
80
+ -- #2571 第三轮:目标自己的 p99 基线比全局下限更宽时,用它(全队变慢不炸屏,
81
+ -- 单节点卡住仍会显形)。没有基线行 ⇒ 与 .163 行为完全一致。
82
+ AND (
83
+ b.b_p99 IS NULL
84
+ OR delivered_at <= datetime(?8, '-' || CAST(b.b_p99 * 1.5 AS INTEGER) || ' seconds')
85
+ -- 上限:再慢的节点,等满 30 分钟也必须报(否则 p99=50min 的节点永远不响)。
86
+ OR delivered_at <= datetime(?8, '-' || CAST(?9 AS INTEGER) || ' seconds')
87
+ )
74
88
  ON CONFLICT(task_id, event_key) DO NOTHING`,
75
- [staleType, staleDetail, cutoff, busyType, staleType, busyDetail, staleDetail],
89
+ [staleType, staleDetail, cutoff, busyType, staleType, busyDetail, staleDetail, referenceIso, BASELINE_MAX_SECONDS],
76
90
  );
77
91
  return result.changes;
78
92
  }
@@ -80,12 +94,13 @@ function recordStale(thresholdSeconds: 600 | 1800, cutoff: string): number {
80
94
  export function recordDeliveredStaleEvents(now = new Date()): DeliveredStalePatrolResult {
81
95
  if (!Number.isFinite(now.getTime())) throw new Error("invalid lifecycle patrol time");
82
96
 
97
+ ensureAliasBaselineTable();
83
98
  const byThreshold = { 600: 0, 1800: 0 };
84
99
  let inserted = 0;
85
100
 
86
101
  for (const thresholdSeconds of DELIVERED_STALE_THRESHOLDS) {
87
102
  const cutoff = sqliteUtcTimestamp(new Date(now.getTime() - thresholdSeconds * 1000));
88
- const changes = recordStale(thresholdSeconds, cutoff);
103
+ const changes = recordStale(thresholdSeconds, cutoff, now.toISOString());
89
104
  byThreshold[thresholdSeconds] = changes;
90
105
  inserted += changes;
91
106
  }
@@ -0,0 +1,119 @@
1
+ // #2668 —— 投给已离线节点的任务,超过 TTL(默认 30 分钟)就过期,而不是躺满 24 小时的 TTL。
2
+ //
3
+ // 现场:任务 3b3c63e0 投给 TM基建鲸 后目标立刻 offline,`delivered` 状态挂了 **2 小时 56 分**,
4
+ // 10/30 分钟两档告警都报了但没有处置;`expires_at` 是创建后 24 小时。
5
+ //
6
+ // 这个文件钉住五件事:
7
+ // 1. 目标 offline ⇒ 超时后过期;
8
+ // 2. 目标没有 session 行(未知)⇒ 同样按 offline 处理;
9
+ // 3. 目标忙(working)⇒ **不处置**(那是 queued_behind_busy 的语义);
10
+ // 4. 节点已经取走(consumed_at 有值)⇒ 不处置(交给 #758 orphan 判据);
11
+ // 5. TTL 可配且可关(COMMHUB_OFFLINE_DELIVERY_TTL_MINUTES,0 = 关闭)。
12
+
13
+ import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
14
+ import { db } from "./db.js";
15
+ import {
16
+ OFFLINE_DELIVERY_TTL_ENV,
17
+ expireDeliveredToOfflineTargets,
18
+ offlineDeliveryTtlMinutes,
19
+ } from "./task-offline-delivery.js";
20
+
21
+ const stamp = `${Date.now()}`;
22
+ let NET = "";
23
+ const NOW = new Date("2026-10-27T03:30:00.000Z");
24
+ const savedEnv = process.env[OFFLINE_DELIVERY_TTL_ENV];
25
+
26
+ function deliveredAgo(minutes: number): string {
27
+ return new Date(NOW.getTime() - minutes * 60_000).toISOString().slice(0, 19).replace("T", " ");
28
+ }
29
+
30
+ function seedSession(alias: string, status: string) {
31
+ db.run(
32
+ `INSERT INTO sessions (resume_id, alias, node_id, status, network_id, registered_at)
33
+ VALUES (?1, ?2, ?3, ?4, ?5, datetime('now'))`,
34
+ [`res_${alias}_${stamp}`, alias, `n_${alias}_${stamp}`, status, NET],
35
+ );
36
+ }
37
+
38
+ function seedTask(id: string, toName: string, deliveredMinutesAgo: number, extra: { consumed?: string } = {}) {
39
+ db.run(
40
+ `INSERT INTO tasks (task_id, from_name, to_name, status, content, delivered_at, consumed_at, expires_at, network_id)
41
+ VALUES (?1, 'sender', ?2, 'delivered', 'offline fixture', ?3, ?4, datetime('now','+1 day'), ?5)`,
42
+ [id, toName, deliveredAgo(deliveredMinutesAgo), extra.consumed ?? null, NET],
43
+ );
44
+ }
45
+
46
+ const statusOf = (id: string) => db.get<{ status: string }>("SELECT status FROM tasks WHERE task_id = ?1", [id])!.status;
47
+
48
+ beforeAll(async () => {
49
+ const { register } = await import("./auth.js");
50
+ const auth = register(`offl_owner_${stamp}`, "Offl-Owner-Strong-1!", undefined, "tester");
51
+ expect(auth.ok).toBe(true);
52
+ NET = auth.network_id!;
53
+ process.env[OFFLINE_DELIVERY_TTL_ENV] = "30";
54
+
55
+ seedSession(`offl-off-${stamp}`, "offline");
56
+ seedSession(`offl-busy-${stamp}`, "working");
57
+
58
+ seedTask(`offl-1-${stamp}`, `offl-off-${stamp}`, 45); // 离线 + 45 分钟 ⇒ 过期
59
+ seedTask(`offl-2-${stamp}`, `offl-off-${stamp}`, 10); // 离线但只 10 分钟 ⇒ 不动
60
+ seedTask(`offl-3-${stamp}`, `offl-busy-${stamp}`, 45); // 目标忙 ⇒ 不动
61
+ seedTask(`offl-4-${stamp}`, `offl-missing-${stamp}`, 45); // 没有 session 行 ⇒ 视为离线 ⇒ 过期
62
+ seedTask(`offl-5-${stamp}`, `offl-off-${stamp}`, 45, { consumed: deliveredAgo(46) }); // 已被取走 ⇒ 不动
63
+ });
64
+
65
+ afterEach(() => { process.env[OFFLINE_DELIVERY_TTL_ENV] = "30"; });
66
+
67
+ afterAll(() => {
68
+ try { db.run(`DELETE FROM tasks WHERE task_id LIKE ?1`, [`offl-%-${stamp}`]); } catch {}
69
+ try { db.run(`DELETE FROM sessions WHERE alias LIKE ?1`, [`offl-%-${stamp}`]); } catch {}
70
+ if (savedEnv === undefined) delete process.env[OFFLINE_DELIVERY_TTL_ENV];
71
+ else process.env[OFFLINE_DELIVERY_TTL_ENV] = savedEnv;
72
+ });
73
+
74
+ describe("#2668 delivered-to-offline expiry", () => {
75
+ test("默认 TTL 30 分钟,可配、可关", () => {
76
+ delete process.env[OFFLINE_DELIVERY_TTL_ENV];
77
+ expect(offlineDeliveryTtlMinutes()).toBe(30);
78
+ process.env[OFFLINE_DELIVERY_TTL_ENV] = "10";
79
+ expect(offlineDeliveryTtlMinutes()).toBe(10);
80
+ process.env[OFFLINE_DELIVERY_TTL_ENV] = "0";
81
+ expect(offlineDeliveryTtlMinutes()).toBe(0);
82
+ });
83
+
84
+ test("目标离线且超过 TTL ⇒ 过期;未超时/目标忙/已取走 ⇒ 都不动", () => {
85
+ const dropped = expireDeliveredToOfflineTargets(NOW);
86
+ const ids = dropped.map(d => d.task_id);
87
+ expect(ids).toContain(`offl-1-${stamp}`);
88
+ expect(ids).toContain(`offl-4-${stamp}`);
89
+ expect(ids).not.toContain(`offl-2-${stamp}`);
90
+ expect(ids).not.toContain(`offl-3-${stamp}`);
91
+ expect(ids).not.toContain(`offl-5-${stamp}`);
92
+
93
+ expect(statusOf(`offl-1-${stamp}`)).toBe("expired");
94
+ expect(statusOf(`offl-4-${stamp}`)).toBe("expired");
95
+ expect(statusOf(`offl-2-${stamp}`)).toBe("delivered");
96
+ expect(statusOf(`offl-3-${stamp}`)).toBe("delivered");
97
+ expect(statusOf(`offl-5-${stamp}`)).toBe("delivered");
98
+ });
99
+
100
+ test("返回行带 waited_minutes(通知文案要用)", () => {
101
+ seedTask(`offl-6-${stamp}`, `offl-off-${stamp}`, 90);
102
+ const dropped = expireDeliveredToOfflineTargets(NOW);
103
+ const row = dropped.find(d => d.task_id === `offl-6-${stamp}`);
104
+ expect(row).toBeTruthy();
105
+ expect(row!.waited_minutes).toBeGreaterThanOrEqual(89);
106
+ });
107
+
108
+ test("第二次调用幂等(已过期的不再出现)", () => {
109
+ const again = expireDeliveredToOfflineTargets(NOW);
110
+ expect(again.map(d => d.task_id)).not.toContain(`offl-1-${stamp}`);
111
+ });
112
+
113
+ test("TTL=0 时整个阶段关闭", () => {
114
+ seedTask(`offl-7-${stamp}`, `offl-off-${stamp}`, 999);
115
+ process.env[OFFLINE_DELIVERY_TTL_ENV] = "0";
116
+ expect(expireDeliveredToOfflineTargets(NOW)).toEqual([]);
117
+ expect(statusOf(`offl-7-${stamp}`)).toBe("delivered");
118
+ });
119
+ });
@@ -0,0 +1,81 @@
1
+ // #2668 —— 投给**已离线**节点的任务不该躺满 24 小时。
2
+ //
3
+ // 现场(2026-10-27,生产):
4
+ // 任务 3b3c63e0 → TM基建鲸,投递后目标 session 立刻 offline,任务以 `delivered` 状态挂了 **2 小时 56 分**;
5
+ // 10 分钟 / 30 分钟两档告警都按时报了,然后**没有任何处置**;`expires_at` 是创建后 24 小时。
6
+ // 同期队列里还有几个同类任务。
7
+ //
8
+ // 为什么这是问题(三条,都实测过):
9
+ // 1. 队列里挂着**不可能被执行**的任务(目标不在),却计入“在途/积压”;与 #2363 的 in-flight 判读混在一起;
10
+ // 2. 发送方(常常是另一个 Agent)在自己 30 分钟超时后早已放弃,Hub 侧看起来还在“投递中”;
11
+ // 3. 目标重新上线后会消费这个陈旧任务 —— 可能做三小时前的工作。
12
+ //
13
+ // 本模块只做一件最小的事:**目标明确不在(session=offline 或没有 session 行)且投递已超过 TTL** ⇒ 过期。
14
+ // 「目标忙」不算(那走 queued_behind_busy,不做处置);`created` 状态不算(还没投递);
15
+ // 已经被节点取走(`consumed_at` 有值)的不算(那是节点在处理,交给 #758 的 orphan 判据)。
16
+
17
+ import { db } from "./db.js";
18
+
19
+ export const OFFLINE_DELIVERY_TTL_ENV = "COMMHUB_OFFLINE_DELIVERY_TTL_MINUTES";
20
+ export const OFFLINE_DELIVERY_TTL_DEFAULT_MINUTES = 30;
21
+
22
+ export function offlineDeliveryTtlMinutes(): number {
23
+ const raw = process.env[OFFLINE_DELIVERY_TTL_ENV];
24
+ if (raw === undefined || raw.trim() === "") return OFFLINE_DELIVERY_TTL_DEFAULT_MINUTES;
25
+ const n = Number(raw);
26
+ return Number.isFinite(n) && n >= 0 ? Math.floor(n) : OFFLINE_DELIVERY_TTL_DEFAULT_MINUTES;
27
+ }
28
+
29
+ export type OfflineDroppedTask = {
30
+ task_id: string;
31
+ network_id: string | null;
32
+ from_name: string | null;
33
+ from_node_id: string | null;
34
+ to_name: string | null;
35
+ content: string;
36
+ created_at: string;
37
+ expires_at: string | null;
38
+ parent_task_id: string | null;
39
+ meta_json: string | null;
40
+ waited_minutes: number;
41
+ };
42
+
43
+ /**
44
+ * 找出并过期「投给离线目标的陈年投递」。返回被过期的行(调用方负责发事件与通知,与 TTL 那条路径一致)。
45
+ * 同步执行:调用点在 5 分钟一次的 patrol 里,单次量级是几十行。
46
+ */
47
+ export function expireDeliveredToOfflineTargets(now = new Date()): OfflineDroppedTask[] {
48
+ const ttl = offlineDeliveryTtlMinutes();
49
+ if (ttl <= 0) return [];
50
+ const nowIso = now.toISOString();
51
+ const ageClause = `datetime(delivered_at) <= datetime(?1, '-${ttl} minutes')`;
52
+
53
+ const due = db.all<OfflineDroppedTask>(
54
+ `SELECT t.task_id, t.network_id, t.from_name, t.from_node_id, t.to_name, t.content,
55
+ t.created_at, t.expires_at, t.parent_task_id, t.meta_json,
56
+ CAST((julianday(?1) - julianday(t.delivered_at)) * 1440 AS INTEGER) AS waited_minutes
57
+ FROM tasks t
58
+ WHERE t.status = 'delivered'
59
+ AND t.delivered_at IS NOT NULL
60
+ AND t.consumed_at IS NULL
61
+ AND ${ageClause}
62
+ AND COALESCE((SELECT s.status FROM sessions s
63
+ WHERE s.alias = t.to_name AND s.network_id = t.network_id), 'offline') = 'offline'`,
64
+ [nowIso],
65
+ );
66
+ if (due.length === 0) return [];
67
+
68
+ const dropped: OfflineDroppedTask[] = [];
69
+ for (const task of due) {
70
+ const changed = db.run(
71
+ `UPDATE tasks SET status = 'expired', completed_at = datetime('now')
72
+ WHERE task_id = ?1 AND status = 'delivered' AND consumed_at IS NULL`,
73
+ [task.task_id],
74
+ );
75
+ if (changed.changes < 1) continue;
76
+ // 与 TTL 过期一致:把 inbox 行 ack 掉,否则 unread 徽标与 stop/delete 门会一直看到它。
77
+ db.run(`UPDATE inbox SET acked = 1 WHERE COALESCE(task_id, id) = ?1 AND acked = 0`, [task.task_id]);
78
+ dropped.push(task);
79
+ }
80
+ return dropped;
81
+ }