@sleep2agi/commhub-server 0.9.0-preview.158 → 0.9.0-preview.160

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sleep2agi/commhub-server",
-  "version": "0.9.0-preview.158",
+  "version": "0.9.0-preview.160",
   "description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
   "type": "module",
   "main": "src/index.ts",

package/src/mcp-inflight.test.ts

@@ -0,0 +1,92 @@
+// #2363 —— MCP 在飞计数的单测 + 源码断言。
+//
+// 计数值本身很小，但它是“RSS 尖峰能不能被解释”的唯一在线指标：`sse_connections`
+// 只反映长连接，看不到同秒打进来的一批短请求。这里钉住三件事：
+//   1. enter/leave 必须配平（含重复 leave 不把计数打成负数）；
+//   2. 峰值要能记住高水位，并能被显式重置；
+//   3. server.ts 必须在**任何 await 之前**进入计数，并且在“正常关闭 / 抛异常”
+//      两条路径上都退出计数——少一条就会让指标永远偏高，比没有指标更糟。
+
+import { describe, expect, test, beforeEach } from "bun:test";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import {
+  enterMcpInflight,
+  leaveMcpInflight,
+  getMcpInflightStats,
+  resetMcpInflightPeak,
+} from "./mcp-inflight.js";
+
+beforeEach(() => {
+  // 模块级单例：每个用例从 0 开始，避免用例顺序影响。
+  while (getMcpInflightStats().inflight > 0) leaveMcpInflight();
+  resetMcpInflightPeak();
+});
+
+describe("#2363 mcp inflight counter", () => {
+  test("enter/leave 配平，peak 记住高水位且不自己回落", () => {
+    expect(getMcpInflightStats()).toEqual({ inflight: 0, peak: 0 });
+    enterMcpInflight();
+    enterMcpInflight();
+    enterMcpInflight();
+    expect(getMcpInflightStats()).toEqual({ inflight: 3, peak: 3 });
+    leaveMcpInflight();
+    leaveMcpInflight();
+    // 当前只有 1 条在飞，但峰值仍是 3 —— 峰值是“撞过悬崖没有”的证据。
+    expect(getMcpInflightStats()).toEqual({ inflight: 1, peak: 3 });
+    leaveMcpInflight();
+    expect(getMcpInflightStats()).toEqual({ inflight: 0, peak: 3 });
+  });
+
+  test("多余的 leave 不会把计数打成负数", () => {
+    leaveMcpInflight();
+    leaveMcpInflight();
+    expect(getMcpInflightStats().inflight).toBe(0);
+  });
+
+  test("reset 把峰值对齐到当前在飞数（不会把峰值压到低于现况）", () => {
+    enterMcpInflight();
+    enterMcpInflight();
+    expect(getMcpInflightStats().peak).toBe(2);
+    leaveMcpInflight();
+    resetMcpInflightPeak();
+    expect(getMcpInflightStats()).toEqual({ inflight: 1, peak: 1 });
+  });
+});
+
+describe("#2363 server wiring (source assertions)", () => {
+  const src = readFileSync(join(import.meta.dir, "server.ts"), "utf8");
+
+  test("进入计数发生在创建 transport 之前（请求体解析阶段也要计入）", () => {
+    const enterAt = src.indexOf("enterMcpInflight();");
+    const transportAt = src.indexOf("new WebStandardStreamableHTTPServerTransport({", enterAt);
+    expect(enterAt).toBeGreaterThan(0);
+    expect(transportAt).toBeGreaterThan(enterAt);
+  });
+
+  test("两条退出路径都在：handleRequest 抛异常时、以及 MCP server 关闭后", () => {
+    // 抛异常路径
+    expect(src).toContain("settleMcpInflight();\n        throw e;");
+    // 正常路径（setImmediate 里 close 完成后结算，且重复调用安全）
+    expect(src).toContain("mcpServer.close().catch(() => {}).finally(settleMcpInflight)");
+    // 幂等闸门：只允许真正减一次
+    expect(src).toContain("if (mcpInflightSettled) return;");
+  });
+
+  test("/health 暴露的是聚合值（匿名也安全），且没有 new 的按会话明细", () => {
+    expect(src).toContain("mcp_inflight: getMcpInflightStats().inflight,");
+    expect(src).toContain("mcp_inflight_peak: getMcpInflightStats().peak,");
+    expect(src).not.toContain("mcp_inflight_sessions");
+  });
+
+  test("这一版只观测：/mcp 入口不得出现 429 / 拒绝分支", () => {
+    const mcpStart = src.indexOf('if (url.pathname === "/mcp") {');
+    const mcpEnd = src.indexOf("// ── #461 SSE network observer stream ──", mcpStart);
+    expect(mcpStart).toBeGreaterThan(0);
+    expect(mcpEnd).toBeGreaterThan(mcpStart);
+    const block = src.slice(mcpStart, mcpEnd);
+    expect(block).not.toContain("429");
+    expect(block).not.toContain("hub_busy");
+    expect(block).not.toContain("Retry-After");
+  });
+});

package/src/mcp-inflight.ts

@@ -0,0 +1,38 @@
+// #2363 —— MCP 在飞计数（先只观测，不拦请求）。
+//
+// 为什么需要它：隔离 Hub 上用**生产库副本**实测，RSS 与“同时在飞的 MCP 请求数”
+// 成正比（≈1.2–1.4 MB/条；`send_message` 与 `get_inbox` 同形），而与吞吐无关——
+// 顺序发 3,255 条/60s（≈54 条/秒）时 RSS 稳定在 151–156 MB，而 50/100/200 并发
+// 的峰值是 249/446/689 MB。`/health` 里的 `sse_connections` 只反映长连接，看不到
+// 这个量，所以生产上“为什么 RSS 突然抬到 450 MB”一直只能事后猜。
+//
+// 本模块只做计数与峰值，**不拒绝、不排队、不改任何响应**——限流（有界队列 + 429）
+// 是另一步，需要先有一天的真实分布再定上限。见 docs/runbooks/hub-memory-and-sse.md。
+
+let inflight = 0;
+let peak = 0;
+
+/** /mcp 进入时调用（在任何 await 之前，保证计入请求体解析阶段）。 */
+export function enterMcpInflight(): void {
+  inflight += 1;
+  if (inflight > peak) peak = inflight;
+}
+
+/** /mcp 结束时调用（响应对象产出且 MCP server 已关闭）；重复调用安全。 */
+export function leaveMcpInflight(): void {
+  if (inflight > 0) inflight -= 1;
+}
+
+/**
+ * 当前在飞数与**自上次 reset 以来的峰值**。
+ * 峰值是“滚动高水位”，不会自己回落——运维看它判断是否撞到过悬崖，
+ * 看 `inflight` 判断现在是否还在里面。
+ */
+export function getMcpInflightStats(): { inflight: number; peak: number } {
+  return { inflight, peak };
+}
+
+/** 把峰值重置为当前在飞数（运维定位完一次尖峰后调用，或测试用）。 */
+export function resetMcpInflightPeak(): void {
+  peak = inflight;
+}

package/src/push.test.ts

@@ -351,26 +351,36 @@ describe("real ReadableStream — byte-level backpressure (round-2 fix)", () =>
     await res.body!.cancel();
   });
 
-  test("non-reading consumer past MAX bytes triggers hard ceiling close", async () => {
+  test("non-reading consumer stops growing at COALESCE_AT and is reaped by the stuck timeout", async () => {
+    // #2363 —— 语义变更：落后超过 COALESCE_AT_BYTES 后**跳过非关键事件**，不再一路排队到
+    // 1 MB 硬顶。这样 150 条慢订阅者不会把内存放大到 连接数 × 1 MB（实测 2 GB 尖峰）。
+    // 兜底仍然存在：硬顶对这一帧的**关键帧**生效；长期不读的由 STUCK_CLOSE_MS 回收。
+    const coalesce = __SSE_INTERNALS_FOR_TEST.COALESCE_AT_BYTES;
     const ceiling = __SSE_INTERNALS_FOR_TEST.MAX_QUEUE_BACKPRESSURE_BYTES;
     const res = createSSEStream("byte-cap-probe", "net-bcp");
     const key = "net-bcp:byte-cap-probe";
     const c = __SSE_INTERNALS_FOR_TEST.clientsMap.get(key)![0];
 
-    // Push events whose total byte size exceeds the hard ceiling.
-    // With chunk-count strategy this would NOT close (each event is 1
-    // chunk; desiredSize would be at -N for N events, never -1M).
-    // With byte strategy this SHOULD close once accumulated bytes
-    // exceed ceiling.
-    const chunkSize = 100_000; // 100KB per push
-    const needed = Math.ceil(ceiling / chunkSize) + 2; // overshoot a bit
+    const chunkSize = 100_000; // 100KB per push, larger than COALESCE_AT
     const big = "x".repeat(chunkSize);
-    for (let i = 0; i < needed; i++) {
+    const before = __SSE_INTERNALS_FOR_TEST.degradedDropCount();
+    for (let i = 0; i < Math.ceil(ceiling / chunkSize) + 2; i++) {
       if (c.closed) break;
       pushEvent("byte-cap-probe", { type: "big", payload: big }, "net-bcp");
     }
+    // 关键断言 1：**没有被立刻关闭**（跳过的语义是“不排队”，不是“断开”）
+    expect(c.closed).toBe(false);
+    // 关键断言 2：队列没有一路涨到硬顶 —— 落后量被 COALESCE_AT 挡住
+    const desired = c.controller.desiredSize ?? 0;
+    expect(-desired).toBeLessThan(ceiling);
+    expect(-desired).toBeLessThanOrEqual(coalesce + chunkSize * 2);
+    // 关键断言 3：跳过被**计数**（不静默）
+    expect(__SSE_INTERNALS_FOR_TEST.degradedDropCount()).toBeGreaterThan(before);
+
+    // 关键断言 4：长期不读仍会被收回（沿用原有 stuck 回收）
+    c.stuckSince = Date.now() - (__SSE_INTERNALS_FOR_TEST.STUCK_CLOSE_MS + 1);
+    __SSE_INTERNALS_FOR_TEST.tryEnqueueBytes(c, new Uint8Array(1), { critical: true });
     expect(c.closed).toBe(true);
-    expect(__SSE_INTERNALS_FOR_TEST.clientsMap.has(key)).toBe(false);
 
     try { await res.body!.cancel(); } catch { /* already closed */ }
   });

package/src/push.ts

@@ -59,6 +59,24 @@ function envNum(k: string, fallback: number): number {
  *  half-open consumer that has stopped reading entirely. */
 const MAX_QUEUE_BACKPRESSURE_BYTES = envNum("ANET_SSE_MAX_QUEUE_BYTES", 1_000_000);
 
+/** #2363 —— 订阅者“已经落后多少字节”就不再给它非关键事件。
+ *
+ *  背景（2026-10-26 实测）：150 条慢订阅者 + 30 msg/s 广播，RSS 峰值可到 2 GB；
+ *  每条连接的上限只是 1 MB，但 300 条 × 1 MB 就是 GB 级。与其把上限一路压到底，
+ *  更合理的是：**明显落后的连接直接跳过非关键事件**（keepalive 与 connected 必须保留，
+ *  否则 liveness 探测和重连握手会失效），并计数 `sse_dropped_degraded`。
+ *
+ *  这不是“丢消息”的新语义：SSE 本来就是 best-effort，客户端重连后会拉一次快照；
+ *  真正的问题是把“慢消费者”变成“Hub 的内存账”。 */
+const COALESCE_AT_BYTES = envNum("ANET_SSE_COALESCE_AT_BYTES", 65_536);
+
+/** #2363 —— 所有广播共用同一个 TextEncoder：编码结果只读、不跨事件复用同一 Uint8Array。 */
+const SHARED_ENCODER = new TextEncoder();
+
+/** 被跳过的事件计数（进程内累计，供 /health 与测试读取）。 */
+let droppedDegraded = 0;
+export function sseDegradedDropCount(): number { return droppedDegraded; }
+
 /** How long `desiredSize` can stay negative (i.e. consumer behind HWM)
  *  before we declare the client dead. 60s by default — past 2× the
  *  keepalive interval, so a live-but-slow consumer wouldn't trip it. */
@@ -200,7 +218,7 @@ function closeClient(client: SSEClient, reason: string): void {
 /** Best-effort enqueue with backpressure guard. Returns:
  *    - "ok"    → bytes accepted, queue healthy
  *    - "dead"  → client closed; caller must remove from map */
-function tryEnqueueBytes(client: SSEClient, bytes: Uint8Array): "ok" | "dead" {
+function tryEnqueueBytes(client: SSEClient, bytes: Uint8Array, opts: { critical?: boolean } = {}): "ok" | "dead" {
   if (client.closed) return "dead";
 
   const ctrl = client.controller;
@@ -219,6 +237,14 @@ function tryEnqueueBytes(client: SSEClient, bytes: Uint8Array): "ok" | "dead" {
     return "dead";
   }
 
+  // #2363 —— 落后超过 COALESCE_AT_BYTES 且不是关键帧：跳过，不排队、也不关连接
+  // （上限之上的仍然按硬顶关闭，见上一段）。这样“慢消费者”不再把内存账单放大到
+  // 连接数 × 单连接上限，同时保留原有“彻底读不动就断开回收”的语义。
+  if (!opts.critical && desired < -COALESCE_AT_BYTES) {
+    droppedDegraded += 1;
+    return "ok";
+  }
+
   // Track stuck-since-when for the soft timeout below.
   if (desired < 0) {
     if (client.stuckSince === null) {
@@ -459,6 +485,7 @@ function createStreamForKey(
       tryEnqueueBytes(
         client,
         encoder.encode(`data: ${JSON.stringify(initialEvent)}\n\n`),
+        { critical: true },
       );
 
       // Periodic keepalive doubles as a half-open probe — if the
@@ -469,7 +496,7 @@ function createStreamForKey(
           if (client.keepaliveTimer) clearInterval(client.keepaliveTimer);
           return;
         }
-        const result = tryEnqueueBytes(client, encoder.encode(`: keepalive\n\n`));
+        const result = tryEnqueueBytes(client, encoder.encode(`: keepalive\n\n`), { critical: true });
         if (result === "ok" && isUserStreamClient(client)) touchUserSeen(client.subscriberUserId!);
         if (result === "dead") {
           // closeClient already cleared the timer; just make sure we
@@ -580,6 +607,9 @@ export function pushEvent(sessionName: string, event: Record<string, unknown>, n
   if (!arr || arr.length === 0) return;
 
   const data = `data: ${JSON.stringify(event)}\n\n`;
+  // #2363 —— 一次编码，所有订阅者共用同一份字节：150 条连接 × 每秒几十条事件时，
+  // 每连接重新 encode 一次会产生大量短命 Uint8Array（实测 150×30/s 时 RSS 峰值 2 GB）。
+  const bytes = SHARED_ENCODER.encode(data);
   let needPrune = false;
 
   for (const c of arr) {
@@ -587,7 +617,7 @@ export function pushEvent(sessionName: string, event: Record<string, unknown>, n
       needPrune = true;
       continue;
     }
-    const result = tryEnqueueBytes(c, c.encoder.encode(data));
+    const result = tryEnqueueBytes(c, bytes);
     if (result === "dead") needPrune = true;
   }
 
@@ -614,6 +644,7 @@ export function pushNetworkObserverEvent(
   if (!arr || arr.length === 0) return;
 
   const data = `data: ${JSON.stringify({ ...event, network_id: networkId, scope: "network" })}\n\n`;
+  const bytes = SHARED_ENCODER.encode(data); // #2363：一次编码，所有 observer 共用
   let needPrune = false;
 
   for (const c of arr) {
@@ -626,7 +657,7 @@ export function pushNetworkObserverEvent(
       try { pass = c.eventFilter(event); } catch { pass = false; }
       if (!pass) continue;
     }
-    const result = tryEnqueueBytes(c, c.encoder.encode(data));
+    const result = tryEnqueueBytes(c, bytes);
     if (result === "dead") needPrune = true;
   }
 
@@ -661,6 +692,7 @@ export function pushUserEvent(
   if (!arr || arr.length === 0) return;
 
   const data = `data: ${JSON.stringify({ ...event, network_id: networkId, user_id: userId, scope: "user" })}\n\n`;
+  const bytes = SHARED_ENCODER.encode(data); // #2363 同 pushEvent：一次编码共用
   let needPrune = false;
 
   for (const c of arr) {
@@ -668,7 +700,7 @@ export function pushUserEvent(
       needPrune = true;
       continue;
     }
-    const result = tryEnqueueBytes(c, c.encoder.encode(data));
+    const result = tryEnqueueBytes(c, bytes);
     if (result === "dead") needPrune = true;
   }
 
@@ -797,6 +829,8 @@ export const __SSE_INTERNALS_FOR_TEST = {
   sweepLiveness,
   closeClient,
   MAX_QUEUE_BACKPRESSURE_BYTES,
+  COALESCE_AT_BYTES,
+  degradedDropCount: sseDegradedDropCount,
   STUCK_CLOSE_MS,
   KEEPALIVE_MS,
   LIVENESS_SWEEP_MS,

package/src/server.ts

@@ -28,6 +28,7 @@ import { ensureNetworkRulesOnce, readNetworkRules, writeNetworkRules } from "./n
 import { deriveTeamRulesView } from "./team-rules-status.js";
 import { recentNodeIdentityConflicts } from "./node-identity-conflict.js";
 import { createSSEStream, createNetworkObserverStream, createUserEventStream, pushEvent, pushNetworkObserverEvent, getSSEStats, PRINTABLE_OBSERVER_KEY_PREFIX, closeUserStreamsInNetwork, getUserPresence, onUserPresenceChange, pushUserEventToNetwork } from "./push.js";
+import { enterMcpInflight, leaveMcpInflight, getMcpInflightStats } from "./mcp-inflight.js";
 import { assertNodeActive } from "./lifecycle-guard.js";
 import { pendingInboxCount } from "./inbox-count.js";
 import { notifyExpiredTasks, type ExpiredTaskRow } from "./task-expiry-notice.js";
@@ -1092,6 +1093,16 @@ return Bun.serve({
           callerIdentityRejected = true;
         }
       }
+      // #2363 —— 在飞计数：从进入 /mcp 到 MCP server 关闭。Hub 的内存与“同时在飞的
+      // 请求数”成正比（隔离实测 ≈1.2–1.4 MB/条；顺序 54 条/秒时 RSS 完全平稳），
+      // 所以这个数比 req/s 更能解释 RSS 尖峰。只观测，不拦请求。
+      enterMcpInflight();
+      let mcpInflightSettled = false;
+      const settleMcpInflight = (): void => {
+        if (mcpInflightSettled) return;
+        mcpInflightSettled = true;
+        leaveMcpInflight();
+      };
       const transport = new WebStandardStreamableHTTPServerTransport({
         sessionIdGenerator: undefined,
       });
@@ -1110,10 +1121,16 @@ return Bun.serve({
       }
       // #478:tools/list 按调用者过滤;X-Anet-Tools: all / ?tools=all 时节点令牌也列出协议工具。
       const mcpServer = createServer(clientIP, enforceNetId, authCtx?.userId || null, callerAlias, !!token?.startsWith("ntok_"), authCtx?.tokenId || null, wantsAllTools(req, url), callerIdentityRejected);
-      await mcpServer.connect(transport);
-      const response = await transport.handleRequest(req, mcpParsedBody === undefined ? undefined : { parsedBody: mcpParsedBody });
+      let response: Response;
+      try {
+        await mcpServer.connect(transport);
+        response = await transport.handleRequest(req, mcpParsedBody === undefined ? undefined : { parsedBody: mcpParsedBody });
+      } catch (e) {
+        settleMcpInflight();
+        throw e;
+      }
       // Disconnect after response to prevent McpServer leak
-      setImmediate(() => mcpServer.close().catch(() => {}));
+      setImmediate(() => { mcpServer.close().catch(() => {}).finally(settleMcpInflight); });
       // #426: the MCP SDK's WebStandardStreamableHTTPServerTransport builds
       // its own Response with `Content-Type: text/event-stream` (or
       // `application/json` for JSON-mode) but WITHOUT charset. Legacy
@@ -2765,6 +2782,10 @@ return Bun.serve({
         transport: "streamable-http",
         sessions_count: count?.cnt ?? 0,
         sse_connections: sse.total,
+        // #2363 —— MCP 在飞数（只观测）。RSS 与它成正比（≈1.2–1.4 MB/条），
+        // 而 `sse_connections` 只反映长连接，看不到“同秒打进来的一批请求”。
+        mcp_inflight: getMcpInflightStats().inflight,
+        mcp_inflight_peak: getMcpInflightStats().peak,
         auth: DEV_OPEN ? "dev-open" : "user-token",
         security: DEV_OPEN ? "dev-open" : "secured",
         tmux: TMUX_ENABLED ? "enabled" : "disabled",