@sleep2agi/commhub-server 0.8.8-preview.0 → 0.9.0-preview.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -3
- package/src/auth.ts +30 -2
- package/src/cross-tenant-injection.test.ts +225 -0
- package/src/db-adapter.ts +31 -0
- package/src/db.ts +55 -1
- package/src/push.test.ts +383 -2
- package/src/push.ts +293 -43
- package/src/send_dedup.test.ts +18 -0
- package/src/send_dedup.ts +22 -7
- package/src/tools.ts +43 -6
- package/src/uploads.test.ts +20 -0
- package/src/uploads.ts +29 -0
package/src/push.ts
CHANGED
|
@@ -1,12 +1,89 @@
|
|
|
1
1
|
// ── SSE Push: 实时推送事件给 Agent ──────────────────
|
|
2
2
|
// Agent 连 GET /events/:session → 保持 SSE 长连接
|
|
3
3
|
// send_task 写 inbox 后 → pushEvent() → 秒达
|
|
4
|
+
//
|
|
5
|
+
// Backpressure & half-open contract (Round-2/4 fix per 通信龙):
|
|
6
|
+
//
|
|
7
|
+
// 0. **CRITICAL**: `desiredSize` units are determined by the stream's
|
|
8
|
+
// queuing strategy. The web-streams default strategy is COUNT-based
|
|
9
|
+
// (each chunk = 1 unit), so a 1 MB chunk only nudges desiredSize by
|
|
10
|
+
// -1. That defeats the byte-cap entirely — pushing a million 1-byte
|
|
11
|
+
// chunks is what trips the threshold, not 1 MB of real bytes.
|
|
12
|
+
// Bun probe verified by 通信牛: default stream after 1 MiB chunk →
|
|
13
|
+
// desiredSize 0, after 2 MiB → -1; byte strategy after 1 MiB → 0,
|
|
14
|
+
// after 2 MiB → -1048576. We MUST construct the stream with an
|
|
15
|
+
// explicit byte-counting strategy:
|
|
16
|
+
// new ReadableStream(src, { highWaterMark: 0, size: c => c.byteLength })
|
|
17
|
+
// With highWaterMark=0, every enqueued byte counts against the cap
|
|
18
|
+
// directly. The HARD CEILING below then closes the stream at
|
|
19
|
+
// -MAX_QUEUE_BACKPRESSURE_BYTES bytes of queued data, which is what
|
|
20
|
+
// we actually want.
|
|
21
|
+
//
|
|
22
|
+
// 1. `ReadableStreamDefaultController.desiredSize` is the SSE queue
|
|
23
|
+
// headroom in BYTES (given the strategy above). Positive = room,
|
|
24
|
+
// 0 = at high-water-mark, negative = over by that many bytes.
|
|
25
|
+
// Without a bound, half-open TCP consumers (network drop, client
|
|
26
|
+
// crash without FIN) silently accumulate bytes in the queue forever
|
|
27
|
+
// — a public-hub OOM vector.
|
|
28
|
+
//
|
|
29
|
+
// 2. We bound it two ways:
|
|
30
|
+
//
|
|
31
|
+
// a. HARD CEILING — `desiredSize < -MAX_QUEUE_BACKPRESSURE_BYTES`
|
|
32
|
+
// → close the stream immediately. Single huge event can't blow
|
|
33
|
+
// up a stuck client.
|
|
34
|
+
//
|
|
35
|
+
// b. STUCK TIMEOUT — `desiredSize < 0` continuously for
|
|
36
|
+
// `STUCK_CLOSE_MS` → close. Catches slow-trickle leaks where
|
|
37
|
+
// the consumer accepts a byte every few seconds but can't keep
|
|
38
|
+
// up.
|
|
39
|
+
//
|
|
40
|
+
// 3. Liveness: every `LIVENESS_SWEEP_MS` we re-check every registered
|
|
41
|
+
// client even when nobody calls `pushEvent`, so a session that goes
|
|
42
|
+
// half-open during a quiet period still gets reaped — not waiting
|
|
43
|
+
// for the next inbox push.
|
|
4
44
|
|
|
5
45
|
import { eventBus, type RenameCommittedEvent } from "./event_bus";
|
|
6
46
|
|
|
47
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
48
|
+
// Tunables (env-overridable for tests and ops)
|
|
49
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
50
|
+
function envNum(k: string, fallback: number): number {
|
|
51
|
+
const v = process.env[k];
|
|
52
|
+
if (!v) return fallback;
|
|
53
|
+
const n = Number(v);
|
|
54
|
+
return Number.isFinite(n) && n > 0 ? n : fallback;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Hard ceiling on per-client SSE queue backpressure before we drop the
|
|
58
|
+
* client. 1 MB by default — generous for normal bursts, lethal for the
|
|
59
|
+
* half-open consumer that has stopped reading entirely. */
|
|
60
|
+
const MAX_QUEUE_BACKPRESSURE_BYTES = envNum("ANET_SSE_MAX_QUEUE_BYTES", 1_000_000);
|
|
61
|
+
|
|
62
|
+
/** How long `desiredSize` can stay negative (i.e. consumer behind HWM)
|
|
63
|
+
* before we declare the client dead. 60s by default — past 2× the
|
|
64
|
+
* keepalive interval, so a live-but-slow consumer wouldn't trip it. */
|
|
65
|
+
const STUCK_CLOSE_MS = envNum("ANET_SSE_STUCK_CLOSE_MS", 60_000);
|
|
66
|
+
|
|
67
|
+
/** Keepalive cadence. Doubles as half-open liveness probe — if enqueue
|
|
68
|
+
* silently piles up bytes here, the stuck-timeout kicks in. */
|
|
69
|
+
const KEEPALIVE_MS = envNum("ANET_SSE_KEEPALIVE_MS", 30_000);
|
|
70
|
+
|
|
71
|
+
/** Liveness sweep cadence — proactive scan even when no events flow. */
|
|
72
|
+
const LIVENESS_SWEEP_MS = envNum("ANET_SSE_LIVENESS_SWEEP_MS", 15_000);
|
|
73
|
+
|
|
7
74
|
type SSEClient = {
|
|
8
75
|
controller: ReadableStreamDefaultController;
|
|
9
76
|
encoder: TextEncoder;
|
|
77
|
+
/** When desiredSize first went negative; cleared on successful drain. */
|
|
78
|
+
stuckSince: number | null;
|
|
79
|
+
/** Once true, the stream has been closed and the client should be
|
|
80
|
+
* evicted from the map on the next sweep / push. Guards against
|
|
81
|
+
* double-close. */
|
|
82
|
+
closed: boolean;
|
|
83
|
+
/** keepalive timer handle so it can be cleared on close. */
|
|
84
|
+
keepaliveTimer?: ReturnType<typeof setInterval>;
|
|
85
|
+
/** Stable key for log/diagnostic context. */
|
|
86
|
+
key: string;
|
|
10
87
|
};
|
|
11
88
|
|
|
12
89
|
// 一个 session 可能有多个 SSE 连接(重连时短暂并存)
|
|
@@ -16,49 +93,189 @@ function ts(): string {
|
|
|
16
93
|
return new Date().toTimeString().slice(0, 8);
|
|
17
94
|
}
|
|
18
95
|
|
|
96
|
+
function clientKey(sessionName: string, networkId?: string | null): string {
|
|
97
|
+
return `${networkId || "global"}:${sessionName}`;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
101
|
+
// Close + dispatch helpers
|
|
102
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
103
|
+
|
|
104
|
+
/** Close a client's stream (idempotent) and clear its keepalive timer.
|
|
105
|
+
* Does NOT remove from the map — leave that to the caller so it can
|
|
106
|
+
* prune the right index without re-finding it. */
|
|
107
|
+
function closeClient(client: SSEClient, reason: string): void {
|
|
108
|
+
if (client.closed) return;
|
|
109
|
+
client.closed = true;
|
|
110
|
+
if (client.keepaliveTimer) clearInterval(client.keepaliveTimer);
|
|
111
|
+
try {
|
|
112
|
+
client.controller.close();
|
|
113
|
+
} catch {
|
|
114
|
+
// Already closed by the runtime, or controller errored. Either way
|
|
115
|
+
// we're done with it.
|
|
116
|
+
}
|
|
117
|
+
console.log(`[${ts()}] SSE ✕ ${client.key} closed (reason=${reason})`);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Best-effort enqueue with backpressure guard. Returns:
|
|
121
|
+
* - "ok" → bytes accepted, queue healthy
|
|
122
|
+
* - "dead" → client closed; caller must remove from map */
|
|
123
|
+
function tryEnqueueBytes(client: SSEClient, bytes: Uint8Array): "ok" | "dead" {
|
|
124
|
+
if (client.closed) return "dead";
|
|
125
|
+
|
|
126
|
+
const ctrl = client.controller;
|
|
127
|
+
// desiredSize may be null on a closed/errored stream — defensive
|
|
128
|
+
// null-check, then treat null as "no headroom" since we can't enqueue
|
|
129
|
+
// safely anyway.
|
|
130
|
+
const desired = ctrl.desiredSize;
|
|
131
|
+
if (desired === null) {
|
|
132
|
+
closeClient(client, "controller-null-desired");
|
|
133
|
+
return "dead";
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Hard ceiling: too far behind HWM, single push could OOM us. Close.
|
|
137
|
+
if (desired < -MAX_QUEUE_BACKPRESSURE_BYTES) {
|
|
138
|
+
closeClient(client, `queue-overflow-${Math.abs(desired)}b`);
|
|
139
|
+
return "dead";
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Track stuck-since-when for the soft timeout below.
|
|
143
|
+
if (desired < 0) {
|
|
144
|
+
if (client.stuckSince === null) {
|
|
145
|
+
client.stuckSince = Date.now();
|
|
146
|
+
} else if (Date.now() - client.stuckSince > STUCK_CLOSE_MS) {
|
|
147
|
+
closeClient(client, `stuck-${Date.now() - client.stuckSince}ms`);
|
|
148
|
+
return "dead";
|
|
149
|
+
}
|
|
150
|
+
} else {
|
|
151
|
+
client.stuckSince = null;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
try {
|
|
155
|
+
ctrl.enqueue(bytes);
|
|
156
|
+
return "ok";
|
|
157
|
+
} catch (e: any) {
|
|
158
|
+
closeClient(client, `enqueue-throw:${e?.message || e}`);
|
|
159
|
+
return "dead";
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Walk every client and close any that are stuck/half-open. Run on a
|
|
164
|
+
* timer so half-open detection doesn't depend on event traffic. */
|
|
165
|
+
function sweepLiveness(): void {
|
|
166
|
+
const now = Date.now();
|
|
167
|
+
for (const [key, arr] of clients) {
|
|
168
|
+
const alive: SSEClient[] = [];
|
|
169
|
+
for (const c of arr) {
|
|
170
|
+
if (c.closed) continue;
|
|
171
|
+
const desired = c.controller.desiredSize;
|
|
172
|
+
if (desired === null) {
|
|
173
|
+
closeClient(c, "sweep-null-desired");
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
if (desired < -MAX_QUEUE_BACKPRESSURE_BYTES) {
|
|
177
|
+
closeClient(c, `sweep-overflow-${Math.abs(desired)}b`);
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
if (desired < 0) {
|
|
181
|
+
if (c.stuckSince === null) c.stuckSince = now;
|
|
182
|
+
else if (now - c.stuckSince > STUCK_CLOSE_MS) {
|
|
183
|
+
closeClient(c, `sweep-stuck-${now - c.stuckSince}ms`);
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
} else {
|
|
187
|
+
c.stuckSince = null;
|
|
188
|
+
}
|
|
189
|
+
alive.push(c);
|
|
190
|
+
}
|
|
191
|
+
if (alive.length === 0) clients.delete(key);
|
|
192
|
+
else if (alive.length !== arr.length) clients.set(key, alive);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// Liveness sweep timer. Started lazily on first stream creation; never
|
|
197
|
+
// stopped in production (one timer for the lifetime of the process).
|
|
198
|
+
let livenessSweepTimer: ReturnType<typeof setInterval> | null = null;
|
|
199
|
+
function ensureLivenessSweep(): void {
|
|
200
|
+
if (livenessSweepTimer) return;
|
|
201
|
+
livenessSweepTimer = setInterval(() => {
|
|
202
|
+
try {
|
|
203
|
+
sweepLiveness();
|
|
204
|
+
} catch (e: any) {
|
|
205
|
+
console.log(`[${ts()}] SSE sweep failed: ${e?.message || e}`);
|
|
206
|
+
}
|
|
207
|
+
}, LIVENESS_SWEEP_MS);
|
|
208
|
+
// Don't keep the event loop alive just for the sweep.
|
|
209
|
+
(livenessSweepTimer as any)?.unref?.();
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
213
|
+
// Public API
|
|
214
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
215
|
+
|
|
19
216
|
/** 创建 SSE Response 并注册到 clients map */
|
|
20
217
|
export function createSSEStream(sessionName: string, networkId?: string | null): Response {
|
|
21
218
|
const encoder = new TextEncoder();
|
|
22
|
-
let ctrl: ReadableStreamDefaultController;
|
|
23
219
|
const key = clientKey(sessionName, networkId);
|
|
220
|
+
let client: SSEClient;
|
|
24
221
|
|
|
25
|
-
const stream = new ReadableStream({
|
|
222
|
+
const stream = new ReadableStream<Uint8Array>({
|
|
26
223
|
start(controller) {
|
|
27
|
-
|
|
28
|
-
|
|
224
|
+
client = {
|
|
225
|
+
controller,
|
|
226
|
+
encoder,
|
|
227
|
+
stuckSince: null,
|
|
228
|
+
closed: false,
|
|
229
|
+
key,
|
|
230
|
+
};
|
|
29
231
|
|
|
30
|
-
if (!clients.has(key))
|
|
31
|
-
clients.set(key, []);
|
|
32
|
-
}
|
|
232
|
+
if (!clients.has(key)) clients.set(key, []);
|
|
33
233
|
clients.get(key)!.push(client);
|
|
34
234
|
console.log(`[${ts()}] SSE ← ${key} connected (${clients.get(key)!.length} clients)`);
|
|
35
235
|
|
|
36
|
-
//
|
|
37
|
-
|
|
236
|
+
// Send initial connected frame through the backpressure guard so
|
|
237
|
+
// even the first byte respects the contract.
|
|
238
|
+
tryEnqueueBytes(
|
|
239
|
+
client,
|
|
240
|
+
encoder.encode(`data: ${JSON.stringify({ type: "connected", session: sessionName, network_id: networkId ?? null })}\n\n`),
|
|
241
|
+
);
|
|
38
242
|
|
|
39
|
-
// Periodic keepalive
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
clearInterval(
|
|
243
|
+
// Periodic keepalive doubles as a half-open probe — if the
|
|
244
|
+
// consumer is gone, this enqueue piles bytes in the queue until
|
|
245
|
+
// tryEnqueueBytes catches the ceiling / stuck timeout.
|
|
246
|
+
client.keepaliveTimer = setInterval(() => {
|
|
247
|
+
if (client.closed) {
|
|
248
|
+
if (client.keepaliveTimer) clearInterval(client.keepaliveTimer);
|
|
249
|
+
return;
|
|
45
250
|
}
|
|
46
|
-
|
|
47
|
-
|
|
251
|
+
const result = tryEnqueueBytes(client, encoder.encode(`: keepalive\n\n`));
|
|
252
|
+
if (result === "dead") {
|
|
253
|
+
// closeClient already cleared the timer; just make sure we
|
|
254
|
+
// also prune from the map.
|
|
255
|
+
pruneClosed(key);
|
|
256
|
+
}
|
|
257
|
+
}, KEEPALIVE_MS);
|
|
258
|
+
(client.keepaliveTimer as any)?.unref?.();
|
|
259
|
+
|
|
260
|
+
ensureLivenessSweep();
|
|
48
261
|
},
|
|
49
262
|
cancel() {
|
|
50
|
-
//
|
|
51
|
-
|
|
52
|
-
if (
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
clearInterval((arr[idx] as any)._keepalive);
|
|
56
|
-
arr.splice(idx, 1);
|
|
57
|
-
}
|
|
58
|
-
if (arr.length === 0) clients.delete(key);
|
|
59
|
-
console.log(`[${ts()}] SSE ✕ ${key} disconnected (${arr.length} remaining)`);
|
|
263
|
+
// Consumer side hangup — straightforward path, mark closed and
|
|
264
|
+
// prune. The half-open path goes through closeClient instead.
|
|
265
|
+
if (client) {
|
|
266
|
+
closeClient(client, "cancel");
|
|
267
|
+
pruneClosed(key);
|
|
60
268
|
}
|
|
61
269
|
},
|
|
270
|
+
}, {
|
|
271
|
+
// BYTE-COUNTING strategy. Without this, desiredSize is a chunk
|
|
272
|
+
// counter and the byte-cap below is meaningless (see file header
|
|
273
|
+
// comment #0). highWaterMark=0 means: after enqueuing N bytes, the
|
|
274
|
+
// queue is N bytes over the mark, so `desiredSize === -N`. Pair
|
|
275
|
+
// this with the hard ceiling in tryEnqueueBytes to actually close
|
|
276
|
+
// a half-open consumer at MAX_QUEUE_BACKPRESSURE_BYTES.
|
|
277
|
+
highWaterMark: 0,
|
|
278
|
+
size: (chunk: Uint8Array) => chunk.byteLength,
|
|
62
279
|
});
|
|
63
280
|
|
|
64
281
|
return new Response(stream, {
|
|
@@ -70,8 +287,17 @@ export function createSSEStream(sessionName: string, networkId?: string | null):
|
|
|
70
287
|
});
|
|
71
288
|
}
|
|
72
289
|
|
|
73
|
-
|
|
74
|
-
|
|
290
|
+
/** Remove closed entries for one key, drop the map slot if empty. */
|
|
291
|
+
function pruneClosed(key: string): void {
|
|
292
|
+
const arr = clients.get(key);
|
|
293
|
+
if (!arr) return;
|
|
294
|
+
const alive = arr.filter((c) => !c.closed);
|
|
295
|
+
if (alive.length === 0) {
|
|
296
|
+
clients.delete(key);
|
|
297
|
+
console.log(`[${ts()}] SSE ✕ ${key} disconnected (0 remaining)`);
|
|
298
|
+
} else if (alive.length !== arr.length) {
|
|
299
|
+
clients.set(key, alive);
|
|
300
|
+
}
|
|
75
301
|
}
|
|
76
302
|
|
|
77
303
|
function rekeyClient(oldSessionName: string, newSessionName: string, networkId?: string | null): number {
|
|
@@ -81,6 +307,10 @@ function rekeyClient(oldSessionName: string, newSessionName: string, networkId?:
|
|
|
81
307
|
const oldClients = clients.get(oldKey);
|
|
82
308
|
if (!oldClients || oldClients.length === 0) return 0;
|
|
83
309
|
|
|
310
|
+
// Update the cached key on each moved client so later closeClient
|
|
311
|
+
// logs report the right session.
|
|
312
|
+
for (const c of oldClients) c.key = newKey;
|
|
313
|
+
|
|
84
314
|
const existing = clients.get(newKey) || [];
|
|
85
315
|
clients.set(newKey, existing.concat(oldClients));
|
|
86
316
|
clients.delete(oldKey);
|
|
@@ -98,29 +328,34 @@ eventBus.on("rename-committed", (event: RenameCommittedEvent) => {
|
|
|
98
328
|
|
|
99
329
|
/** 推送事件给指定 session 的所有 SSE 连接 */
|
|
100
330
|
export function pushEvent(sessionName: string, event: Record<string, unknown>, networkId?: string | null): void {
|
|
101
|
-
const
|
|
331
|
+
const key = clientKey(sessionName, networkId ?? (event.network_id as string | null | undefined));
|
|
332
|
+
const arr = clients.get(key);
|
|
102
333
|
if (!arr || arr.length === 0) return;
|
|
103
334
|
|
|
104
335
|
const data = `data: ${JSON.stringify(event)}\n\n`;
|
|
105
|
-
|
|
336
|
+
let needPrune = false;
|
|
106
337
|
|
|
107
|
-
for (
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
dead.push(i);
|
|
338
|
+
for (const c of arr) {
|
|
339
|
+
if (c.closed) {
|
|
340
|
+
needPrune = true;
|
|
341
|
+
continue;
|
|
112
342
|
}
|
|
343
|
+
const result = tryEnqueueBytes(c, c.encoder.encode(data));
|
|
344
|
+
if (result === "dead") needPrune = true;
|
|
113
345
|
}
|
|
114
346
|
|
|
115
|
-
|
|
116
|
-
for (let i = dead.length - 1; i >= 0; i--) {
|
|
117
|
-
arr.splice(dead[i], 1);
|
|
118
|
-
}
|
|
119
|
-
if (arr.length === 0) clients.delete(clientKey(sessionName, networkId ?? (event.network_id as string | null | undefined)));
|
|
347
|
+
if (needPrune) pruneClosed(key);
|
|
120
348
|
}
|
|
121
349
|
|
|
122
350
|
export function __resetSSEClientsForTest(): void {
|
|
351
|
+
for (const arr of clients.values()) {
|
|
352
|
+
for (const c of arr) closeClient(c, "reset-for-test");
|
|
353
|
+
}
|
|
123
354
|
clients.clear();
|
|
355
|
+
if (livenessSweepTimer) {
|
|
356
|
+
clearInterval(livenessSweepTimer);
|
|
357
|
+
livenessSweepTimer = null;
|
|
358
|
+
}
|
|
124
359
|
}
|
|
125
360
|
|
|
126
361
|
/** 获取当前 SSE 连接统计 */
|
|
@@ -128,8 +363,23 @@ export function getSSEStats(): { total: number; sessions: Record<string, number>
|
|
|
128
363
|
let total = 0;
|
|
129
364
|
const sessions: Record<string, number> = {};
|
|
130
365
|
for (const [name, arr] of clients) {
|
|
131
|
-
|
|
132
|
-
|
|
366
|
+
const alive = arr.filter((c) => !c.closed);
|
|
367
|
+
if (alive.length === 0) continue;
|
|
368
|
+
sessions[name] = alive.length;
|
|
369
|
+
total += alive.length;
|
|
133
370
|
}
|
|
134
371
|
return { total, sessions };
|
|
135
372
|
}
|
|
373
|
+
|
|
374
|
+
// Test hooks — exported so the test file can simulate half-open / stuck
|
|
375
|
+
// consumers without spinning up a real TCP socket.
|
|
376
|
+
export const __SSE_INTERNALS_FOR_TEST = {
|
|
377
|
+
tryEnqueueBytes,
|
|
378
|
+
sweepLiveness,
|
|
379
|
+
closeClient,
|
|
380
|
+
MAX_QUEUE_BACKPRESSURE_BYTES,
|
|
381
|
+
STUCK_CLOSE_MS,
|
|
382
|
+
KEEPALIVE_MS,
|
|
383
|
+
LIVENESS_SWEEP_MS,
|
|
384
|
+
clientsMap: clients,
|
|
385
|
+
};
|
package/src/send_dedup.test.ts
CHANGED
|
@@ -117,6 +117,24 @@ describe("SendDedup", () => {
|
|
|
117
117
|
expect(d.size).toBeGreaterThan(0);
|
|
118
118
|
});
|
|
119
119
|
|
|
120
|
+
it("evictOldest drops to exactly floor(maxKeys * 0.9) on the first trigger", () => {
|
|
121
|
+
// Regression guard for the size-during-loop bug fixed in the
|
|
122
|
+
// robustness pass. Insert exactly maxKeys+1 records to fire exactly
|
|
123
|
+
// ONE evictOldest call, then snapshot size before any further inserts
|
|
124
|
+
// can re-grow it. With the bug, the loop bound
|
|
125
|
+
// `i < this.last.size - target` shrunk each iteration and only
|
|
126
|
+
// ~half the intended deletions ran (e.g. 14 of 27 for maxKeys=256,
|
|
127
|
+
// landing size at ~243 instead of 230). After the fix size lands at
|
|
128
|
+
// the documented target = floor(maxKeys * 0.9).
|
|
129
|
+
const maxKeys = 256;
|
|
130
|
+
const target = Math.floor(maxKeys * 0.9); // 230
|
|
131
|
+
const d = new SendDedup({ windowMs: 600_000, maxKeys });
|
|
132
|
+
for (let i = 0; i <= maxKeys; i++) {
|
|
133
|
+
d.record(`from-${i}`, "bob", `content-${i}`, 1_000 + i);
|
|
134
|
+
}
|
|
135
|
+
expect(d.size).toBe(target);
|
|
136
|
+
});
|
|
137
|
+
|
|
120
138
|
it("hashes content rather than holding the raw bytes", () => {
|
|
121
139
|
// The key function is the public surface that proves we don't keep
|
|
122
140
|
// raw content in memory between sends — the map keys themselves are
|
package/src/send_dedup.ts
CHANGED
|
@@ -44,13 +44,20 @@ export type DedupConfig = {
|
|
|
44
44
|
};
|
|
45
45
|
|
|
46
46
|
export function readDedupConfig(env: NodeJS.ProcessEnv = process.env): DedupConfig {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
47
|
+
// Parse-then-validate-then-clamp: `Number("abc")` is NaN and `Math.max(0, NaN)`
|
|
48
|
+
// is ALSO NaN per spec, so the old inline `Math.max(0, Number(rawWindow))`
|
|
49
|
+
// relied on the later `Number.isFinite` fallback to repair the result.
|
|
50
|
+
// Hoisting the validity check first makes the intent explicit and removes
|
|
51
|
+
// the risk that a future refactor drops the `isFinite` rescue.
|
|
52
|
+
const parseClamped = (raw: string | undefined, min: number, fallback: number): number => {
|
|
53
|
+
if (raw === undefined) return fallback;
|
|
54
|
+
const n = Number(raw);
|
|
55
|
+
if (!Number.isFinite(n)) return fallback;
|
|
56
|
+
return Math.max(min, n);
|
|
57
|
+
};
|
|
51
58
|
return {
|
|
52
|
-
windowMs:
|
|
53
|
-
maxKeys:
|
|
59
|
+
windowMs: parseClamped(env.COMMHUB_SEND_DEDUP_WINDOW_MS, 0, 300_000),
|
|
60
|
+
maxKeys: parseClamped(env.COMMHUB_SEND_DEDUP_MAX_KEYS, 64, 4096),
|
|
54
61
|
};
|
|
55
62
|
}
|
|
56
63
|
|
|
@@ -133,8 +140,16 @@ export class SendDedup {
|
|
|
133
140
|
private evictOldest(): void {
|
|
134
141
|
const target = Math.max(64, Math.floor(this.cfg.maxKeys * 0.9));
|
|
135
142
|
if (this.last.size <= target) return;
|
|
143
|
+
// CACHE toDelete BEFORE the deletion loop. Previously the loop bound
|
|
144
|
+
// was `i < this.last.size - target`, which is re-evaluated every
|
|
145
|
+
// iteration — and `this.last.size` shrinks by 1 per delete. The result
|
|
146
|
+
// was that we only evicted ~half of the intended count (size landed
|
|
147
|
+
// around 95 % of maxKeys instead of the documented ~90 %). The map
|
|
148
|
+
// stayed bounded so this was efficiency-only, not a leak, but the
|
|
149
|
+
// documented contract was being violated.
|
|
150
|
+
const toDelete = this.last.size - target;
|
|
136
151
|
const entries = Array.from(this.last.entries()).sort((a, b) => a[1] - b[1]);
|
|
137
|
-
for (let i = 0; i <
|
|
152
|
+
for (let i = 0; i < toDelete; i++) this.last.delete(entries[i][0]);
|
|
138
153
|
}
|
|
139
154
|
}
|
|
140
155
|
|
package/src/tools.ts
CHANGED
|
@@ -498,9 +498,12 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
|
|
|
498
498
|
if (updatedTaskId) logTaskEvent(updatedTaskId, null, "replied", alias, "report_completion");
|
|
499
499
|
|
|
500
500
|
// Auto-chain to parent lineage (mirror of send_reply path).
|
|
501
|
+
// round5 F2: pass caller's effectiveNetId so the chain refuses
|
|
502
|
+
// to write across tenants if some upstream parent links to a
|
|
503
|
+
// foreign network.
|
|
501
504
|
if (updatedTaskId) {
|
|
502
505
|
try {
|
|
503
|
-
chainReplyToParent(updatedTaskId, result, "replied");
|
|
506
|
+
chainReplyToParent(updatedTaskId, result, "replied", 5, effectiveNetId);
|
|
504
507
|
const parentChain = db.get<{ parent_task_id: string | null }>(
|
|
505
508
|
"SELECT parent_task_id FROM tasks WHERE task_id = ?1",
|
|
506
509
|
[updatedTaskId]
|
|
@@ -698,17 +701,48 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
|
|
|
698
701
|
// Resolve parent_task_id: explicit > inferred (caller's most recent
|
|
699
702
|
// delivered/started inbox task that's still open). Inference is the
|
|
700
703
|
// safety net for when the LLM forgets to pass parent_task_id.
|
|
704
|
+
//
|
|
705
|
+
// round5 F1 fix: the inference SELECT MUST be network-scoped. Without
|
|
706
|
+
// it, a caller in network B can pick up the parent_task_id of a
|
|
707
|
+
// network A dispatch and chain-reply into the wrong tenant. The
|
|
708
|
+
// explicit-parent path is verified below in F2.
|
|
701
709
|
let parentTaskId: string | null = parentIn ?? null;
|
|
702
710
|
if (!parentTaskId && from_session && from_session !== "hub" && from_session !== "api") {
|
|
703
711
|
try {
|
|
704
|
-
const
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
712
|
+
const recentParams: any[] = [from_session];
|
|
713
|
+
let recentSql = "SELECT task_id FROM tasks WHERE to_name = ?1 AND status IN ('delivered','started')";
|
|
714
|
+
recentSql = addScope(recentSql, recentParams, effectiveNetId);
|
|
715
|
+
recentSql += " ORDER BY created_at DESC LIMIT 1";
|
|
716
|
+
const recent = db.get<{ task_id: string }>(recentSql, ...recentParams);
|
|
708
717
|
if (recent?.task_id) parentTaskId = recent.task_id;
|
|
709
718
|
} catch {}
|
|
710
719
|
}
|
|
711
720
|
|
|
721
|
+
// round5 F2 fix: an explicit parent_task_id must belong to the
|
|
722
|
+
// caller's network. Otherwise a malicious caller in network B can
|
|
723
|
+
// hand us a parent id from network A and have chainReplyToParent
|
|
724
|
+
// (db.ts) write back into A's task result + inbox — cross-tenant
|
|
725
|
+
// write. Verify ownership, reject on mismatch.
|
|
726
|
+
if (parentIn) {
|
|
727
|
+
const parentRow = db.get<{ network_id: string | null }>(
|
|
728
|
+
"SELECT network_id FROM tasks WHERE task_id = ?1",
|
|
729
|
+
[parentIn]
|
|
730
|
+
);
|
|
731
|
+
if (!parentRow) {
|
|
732
|
+
// Parent doesn't exist (LLM hallucination, race with retention
|
|
733
|
+
// sweep, etc.). Drop the link silently so the dispatch can
|
|
734
|
+
// still proceed; the LLM may have meant to dispatch fresh.
|
|
735
|
+
console.log(`[${ts()}] ⚠ send_task: parent_task_id=${parentIn.slice(0, 8)} not found, dropping parent link`);
|
|
736
|
+
parentTaskId = null;
|
|
737
|
+
} else if ((parentRow.network_id ?? null) !== (effectiveNetId ?? null)) {
|
|
738
|
+
console.log(`[${ts()}] 🚫 send_task: cross-network parent rejected, parent=${parentIn.slice(0, 8)} parent-net=${parentRow.network_id ?? "null"} caller-net=${effectiveNetId ?? "null"}`);
|
|
739
|
+
return { content: [{ type: "text" as const, text: JSON.stringify({
|
|
740
|
+
ok: false, error: "cross_network_parent",
|
|
741
|
+
message: "parent_task_id belongs to a different network",
|
|
742
|
+
}) }] };
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
|
|
712
746
|
// Role check: viewer cannot send tasks
|
|
713
747
|
if (!canWrite(effectiveNetId)) {
|
|
714
748
|
return writeDeniedReply(effectiveNetId, "send_task");
|
|
@@ -980,9 +1014,12 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
|
|
|
980
1014
|
|
|
981
1015
|
// Auto-chain reply up to parent task lineage so admin sees the final
|
|
982
1016
|
// answer even if the intermediate session has died.
|
|
1017
|
+
// round5 F2: pass caller's effectiveNetId so the chain refuses
|
|
1018
|
+
// to write across tenants if some upstream parent links to a
|
|
1019
|
+
// foreign network.
|
|
983
1020
|
if (replyLogged && in_reply_to) {
|
|
984
1021
|
try {
|
|
985
|
-
chainReplyToParent(in_reply_to, text, replyStatus);
|
|
1022
|
+
chainReplyToParent(in_reply_to, text, replyStatus, 5, effectiveNetId);
|
|
986
1023
|
// Push SSE event for parent originator if there is a chain.
|
|
987
1024
|
const parentChain = db.get<{ parent_task_id: string | null; from_name: string }>(
|
|
988
1025
|
"SELECT parent_task_id, from_name FROM tasks WHERE task_id = ?1",
|
package/src/uploads.test.ts
CHANGED
|
@@ -187,6 +187,26 @@ describe("UploadRateLimiter", () => {
|
|
|
187
187
|
}
|
|
188
188
|
expect(lim.check("token-y", 1000).allowed).toBe(false);
|
|
189
189
|
});
|
|
190
|
+
|
|
191
|
+
test("caps in-memory state by evicting oldest entries past maxEntries", () => {
|
|
192
|
+
// A small maxEntries lets us drive the eviction path deterministically.
|
|
193
|
+
// Without this cap the limiter would grow unbounded under an IP-flood
|
|
194
|
+
// (public-facing hub, IPv6 rotation makes each request a "new" key).
|
|
195
|
+
const lim = new UploadRateLimiter(60_000, 5, 4);
|
|
196
|
+
const t0 = 1_700_000_000_000;
|
|
197
|
+
// Fill to cap with non-expired entries — eviction should fire on insert 5.
|
|
198
|
+
for (let i = 0; i < 4; i++) {
|
|
199
|
+
lim.check(`key-${i}`, t0);
|
|
200
|
+
}
|
|
201
|
+
expect(lim.size()).toBe(4);
|
|
202
|
+
lim.check("key-new", t0 + 100);
|
|
203
|
+
expect(lim.size()).toBeLessThanOrEqual(4);
|
|
204
|
+
// Hitting an EXISTING key while at cap must NOT trigger an eviction
|
|
205
|
+
// (an attacker hitting the same key shouldn't push out other callers).
|
|
206
|
+
const beforeSize = lim.size();
|
|
207
|
+
lim.check("key-new", t0 + 200);
|
|
208
|
+
expect(lim.size()).toBe(beforeSize);
|
|
209
|
+
});
|
|
190
210
|
});
|
|
191
211
|
|
|
192
212
|
describe("validateAttachments", () => {
|
package/src/uploads.ts
CHANGED
|
@@ -37,6 +37,12 @@ export const MAX_REQUEST_CONTENT_LENGTH = MAX_UPLOAD_BYTES + 1024 * 1024;
|
|
|
37
37
|
// the source of truth; resets on restart).
|
|
38
38
|
export const UPLOAD_RATE_WINDOW_MS = 60 * 60 * 1000;
|
|
39
39
|
export const UPLOAD_RATE_MAX_PER_WINDOW = 60;
|
|
40
|
+
// Cap the in-memory rate-limit map so a public-facing hub can't be
|
|
41
|
+
// pushed to OOM by a flood of unique IPs / token ids (the call site
|
|
42
|
+
// uses IP-as-key for legacy auth, which is rotatable cheaply over IPv6).
|
|
43
|
+
// Matches LOGIN_GUARD_MAX_ENTRIES in auth_login_guard.ts for symmetric
|
|
44
|
+
// behaviour across the two in-memory limiters this hub maintains.
|
|
45
|
+
export const UPLOAD_RATE_MAX_ENTRIES = 50_000;
|
|
40
46
|
|
|
41
47
|
// Strict file_id regex. Generated server-side via crypto.randomUUID()
|
|
42
48
|
// with hyphens stripped, so the natural output is 32 hex chars. The
|
|
@@ -159,6 +165,7 @@ export class UploadRateLimiter {
|
|
|
159
165
|
constructor(
|
|
160
166
|
private readonly windowMs: number = UPLOAD_RATE_WINDOW_MS,
|
|
161
167
|
private readonly maxPerWindow: number = UPLOAD_RATE_MAX_PER_WINDOW,
|
|
168
|
+
private readonly maxEntries: number = UPLOAD_RATE_MAX_ENTRIES,
|
|
162
169
|
) {}
|
|
163
170
|
|
|
164
171
|
/**
|
|
@@ -169,6 +176,14 @@ export class UploadRateLimiter {
|
|
|
169
176
|
* Window is rolling on first touch (not aligned to wall-clock hours).
|
|
170
177
|
*/
|
|
171
178
|
check(key: string, nowMs: number = Date.now()): { allowed: boolean; remaining: number; resetAt: number; retryAfterMs?: number } {
|
|
179
|
+
// Before inserting a brand-new key into a full map, sweep expired
|
|
180
|
+
// entries and (if still over cap) evict the oldest by insertion order.
|
|
181
|
+
// Existing-key updates skip the eviction path so an attacker who
|
|
182
|
+
// keeps hitting the same key can't push out other callers.
|
|
183
|
+
if (this.state.size >= this.maxEntries && !this.state.has(key)) {
|
|
184
|
+
this.pruneExpired(nowMs);
|
|
185
|
+
this.evictOldestUntilBelowCap();
|
|
186
|
+
}
|
|
172
187
|
const entry = this.state.get(key);
|
|
173
188
|
if (!entry || nowMs >= entry.resetAt) {
|
|
174
189
|
const resetAt = nowMs + this.windowMs;
|
|
@@ -191,6 +206,20 @@ export class UploadRateLimiter {
|
|
|
191
206
|
size(): number {
|
|
192
207
|
return this.state.size;
|
|
193
208
|
}
|
|
209
|
+
|
|
210
|
+
private pruneExpired(nowMs: number): void {
|
|
211
|
+
for (const [key, entry] of this.state) {
|
|
212
|
+
if (nowMs >= entry.resetAt) this.state.delete(key);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
private evictOldestUntilBelowCap(): void {
|
|
217
|
+
while (this.state.size >= this.maxEntries) {
|
|
218
|
+
const oldest = this.state.keys().next().value as string | undefined;
|
|
219
|
+
if (!oldest) return;
|
|
220
|
+
this.state.delete(oldest);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
194
223
|
}
|
|
195
224
|
|
|
196
225
|
/** Module-level singleton shared by the HTTP /api/upload handler. */
|