@sleep2agi/commhub-server 0.8.8 → 0.9.0-preview.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/auth.ts +30 -2
- package/src/cross-tenant-injection.test.ts +225 -0
- package/src/db-adapter.ts +2 -2
- package/src/db.ts +55 -1
- package/src/push.test.ts +383 -2
- package/src/push.ts +291 -55
- package/src/tools.ts +43 -6
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sleep2agi/commhub-server",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "CommHub Server
|
|
3
|
+
"version": "0.9.0-preview.0",
|
|
4
|
+
"description": "CommHub Server \u2014 AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and 17 MCP tools.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
7
7
|
"bin": {
|
package/src/auth.ts
CHANGED
|
@@ -19,6 +19,14 @@ export interface AuthResult {
|
|
|
19
19
|
token?: string; // user token (utok_)
|
|
20
20
|
network_token?: string; // network token (ntok_) for default network
|
|
21
21
|
network_id?: string;
|
|
22
|
+
// #261 P0-2 (2026-06-28): true when the logged-in user still has the
|
|
23
|
+
// bootstrap-default password. The CLI client uses this to print a
|
|
24
|
+
// prominent "must change password" warning after login succeeds. NOT
|
|
25
|
+
// a login-blocker — we don't lock out anyone whose deployment ran on
|
|
26
|
+
// the old `admin/anethub` default (back-compat per 通信龙 spec). The
|
|
27
|
+
// field is intentionally absent (rather than `false`) on the normal
|
|
28
|
+
// case so old clients don't even see it.
|
|
29
|
+
must_change_password?: boolean;
|
|
22
30
|
}
|
|
23
31
|
|
|
24
32
|
function validatePasswordStrength(password: string, label = "password"): string | null {
|
|
@@ -93,7 +101,7 @@ export function register(username: string, password: string, email?: string, dis
|
|
|
93
101
|
|
|
94
102
|
export function login(username: string, password: string): AuthResult {
|
|
95
103
|
const user = db.get<any>(
|
|
96
|
-
"SELECT user_id, username, password_hash, display_name, email, role FROM users WHERE username = ?1",
|
|
104
|
+
"SELECT user_id, username, password_hash, display_name, email, role, must_change_password FROM users WHERE username = ?1",
|
|
97
105
|
username);
|
|
98
106
|
|
|
99
107
|
if (!user) return { ok: false, error: "invalid username or password" };
|
|
@@ -123,6 +131,9 @@ export function login(username: string, password: string): AuthResult {
|
|
|
123
131
|
user: { user_id: user.user_id, username: user.username, display_name: user.display_name, email: user.email, role: user.role },
|
|
124
132
|
token,
|
|
125
133
|
network_id: networkId,
|
|
134
|
+
// #261 P0-2 — only include field when truthy (back-compat; old clients
|
|
135
|
+
// don't see this field at all unless their account is flagged).
|
|
136
|
+
...(user.must_change_password === 1 ? { must_change_password: true } : {}),
|
|
126
137
|
};
|
|
127
138
|
}
|
|
128
139
|
|
|
@@ -277,11 +288,28 @@ export function changePassword(userId: string, oldPassword: string, newPassword:
|
|
|
277
288
|
const user = db.get<any>("SELECT password_hash FROM users WHERE user_id = ?1", userId);
|
|
278
289
|
if (!user) return { ok: false, error: "user not found" };
|
|
279
290
|
if (user.password_hash !== hashPassword(oldPassword)) return { ok: false, error: "incorrect current password" };
|
|
280
|
-
|
|
291
|
+
// #261 P0-2 — clearing must_change_password as a side-effect of a real
|
|
292
|
+
// password change. If the user changes via `anet passwd`, the bootstrap
|
|
293
|
+
// nudge goes away on next login. SET to 0 explicitly (rather than skip)
|
|
294
|
+
// so a future flag flip can't drift the state.
|
|
295
|
+
db.run("UPDATE users SET password_hash = ?1, must_change_password = 0, updated_at = datetime('now') WHERE user_id = ?2", [hashPassword(newPassword), userId]);
|
|
281
296
|
const revoked = revokeOtherUserTokens(userId, currentTokenId);
|
|
282
297
|
return { ok: true, revoked };
|
|
283
298
|
}
|
|
284
299
|
|
|
300
|
+
/**
|
|
301
|
+
* #261 P0-2 — mark a user as needing to change their password on first
|
|
302
|
+
* login. Called by the hub-bootstrap path (`anet hub start`) AFTER it
|
|
303
|
+
* auto-registers the admin with a random bootstrap password. Idempotent.
|
|
304
|
+
* Returns true if a row was affected (i.e. the user exists), false
|
|
305
|
+
* otherwise. NOT exposed via any HTTP endpoint — internal-only, called
|
|
306
|
+
* via direct module import or a local-process SQLite handle.
|
|
307
|
+
*/
|
|
308
|
+
export function markMustChangePassword(userId: string): boolean {
|
|
309
|
+
const r = db.run("UPDATE users SET must_change_password = 1 WHERE user_id = ?1", [userId]);
|
|
310
|
+
return r.changes > 0;
|
|
311
|
+
}
|
|
312
|
+
|
|
285
313
|
export function resetUserPassword(targetUsername: string, callerIsHubAdmin: boolean): { ok: boolean; error?: string; username?: string; user_id?: string; password?: string; token?: string; token_id?: string; revoked?: number } {
|
|
286
314
|
if (!callerIsHubAdmin) return { ok: false, error: "hub admin required" };
|
|
287
315
|
const user = db.get<any>("SELECT user_id, username FROM users WHERE username = ?1", targetUsername);
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
// Round-5 F1+F2: cross-tenant write/inject guards.
|
|
2
|
+
//
|
|
3
|
+
// All three guards are exercised against a real in-process SQLite
|
|
4
|
+
// (COMMHUB_DB=/tmp/...) so we're not just unit-testing the SQL string,
|
|
5
|
+
// we're proving the rejection actually happens against persisted rows.
|
|
6
|
+
//
|
|
7
|
+
// Setup: two networks (A, B). Create a task in network A with a known
|
|
8
|
+
// task_id. Then drive the three attack vectors below from a session
|
|
9
|
+
// belonging to network B and assert each is blocked.
|
|
10
|
+
//
|
|
11
|
+
// Run with:
|
|
12
|
+
// COMMHUB_DB=/tmp/xtenant-test.db bun test src/cross-tenant-injection.test.ts
|
|
13
|
+
|
|
14
|
+
import { describe, expect, test, beforeEach } from "bun:test";
|
|
15
|
+
import { db, uuidv4, chainReplyToParent } from "./db.js";
|
|
16
|
+
|
|
17
|
+
function insertTask(opts: {
|
|
18
|
+
task_id?: string;
|
|
19
|
+
from_name: string;
|
|
20
|
+
to_name: string;
|
|
21
|
+
network_id: string | null;
|
|
22
|
+
status?: string;
|
|
23
|
+
parent_task_id?: string | null;
|
|
24
|
+
result?: string | null;
|
|
25
|
+
}): string {
|
|
26
|
+
const id = opts.task_id ?? uuidv4();
|
|
27
|
+
db.run(
|
|
28
|
+
`INSERT INTO tasks (task_id, from_name, to_name, priority, status, content, requires_response, created_at, network_id, parent_task_id, result)
|
|
29
|
+
VALUES (?1, ?2, ?3, 'normal', ?4, 'test content', 'reply', datetime('now'), ?5, ?6, ?7)`,
|
|
30
|
+
[id, opts.from_name, opts.to_name, opts.status ?? "delivered", opts.network_id, opts.parent_task_id ?? null, opts.result ?? null]
|
|
31
|
+
);
|
|
32
|
+
return id;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function getTask(taskId: string) {
|
|
36
|
+
return db.get<{ task_id: string; status: string; result: string | null; network_id: string | null }>(
|
|
37
|
+
"SELECT task_id, status, result, network_id FROM tasks WHERE task_id = ?1",
|
|
38
|
+
taskId
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function countInbox(toSession: string): number {
|
|
43
|
+
const row = db.get<{ cnt: number }>(
|
|
44
|
+
"SELECT COUNT(*) as cnt FROM inbox WHERE session_name = ?1",
|
|
45
|
+
toSession
|
|
46
|
+
);
|
|
47
|
+
return row?.cnt ?? 0;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
beforeEach(() => {
|
|
51
|
+
// Clean slate per test so prior fixtures don't leak across runs.
|
|
52
|
+
db.run("DELETE FROM tasks");
|
|
53
|
+
db.run("DELETE FROM inbox");
|
|
54
|
+
db.run("DELETE FROM task_events");
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
58
|
+
// F2 #3 — chainReplyToParent cross-network rejection
|
|
59
|
+
//
|
|
60
|
+
// The unit-level test of the most direct vector: caller in network B
|
|
61
|
+
// holds a child task whose `parent_task_id` points at a parent in
|
|
62
|
+
// network A. Without the fix, chainReplyToParent walks the link and
|
|
63
|
+
// writes `result` + an inbox notification into network A. With the
|
|
64
|
+
// fix, the function refuses to traverse and writes nothing.
|
|
65
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
66
|
+
|
|
67
|
+
describe("chainReplyToParent — cross-network rejection (round5 F2 #3)", () => {
|
|
68
|
+
test("blocks chain when parent.network_id != callerNetId", () => {
|
|
69
|
+
// Network A: a task that should NEVER receive a foreign reply.
|
|
70
|
+
const parentA = insertTask({
|
|
71
|
+
from_name: "admin-a",
|
|
72
|
+
to_name: "worker-a",
|
|
73
|
+
network_id: "net-A",
|
|
74
|
+
status: "delivered",
|
|
75
|
+
});
|
|
76
|
+
// Network B: a child task that maliciously names parentA as its
|
|
77
|
+
// parent (the attack — possible because send_task wrote a row,
|
|
78
|
+
// and B controls the parent_task_id field of its own dispatch
|
|
79
|
+
// when the F2 #2 guard isn't in place either).
|
|
80
|
+
const childB = insertTask({
|
|
81
|
+
from_name: "attacker-b",
|
|
82
|
+
to_name: "victim-b",
|
|
83
|
+
network_id: "net-B",
|
|
84
|
+
status: "replied",
|
|
85
|
+
parent_task_id: parentA,
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
// Caller is in network B (effectiveNetId = "net-B"). The chain
|
|
89
|
+
// walks child → parentA, finds parent.network_id="net-A" !=
|
|
90
|
+
// caller="net-B", refuses to write.
|
|
91
|
+
chainReplyToParent(childB, "INJECTED PAYLOAD", "replied", 5, "net-B");
|
|
92
|
+
|
|
93
|
+
const after = getTask(parentA);
|
|
94
|
+
expect(after?.result).toBeNull(); // result NOT polluted
|
|
95
|
+
expect(after?.status).toBe("delivered"); // status NOT bumped
|
|
96
|
+
expect(countInbox("admin-a")).toBe(0); // no rogue inbox row
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test("legitimate same-network chain still works", () => {
|
|
100
|
+
const parentB = insertTask({
|
|
101
|
+
from_name: "admin-b",
|
|
102
|
+
to_name: "worker-b",
|
|
103
|
+
network_id: "net-B",
|
|
104
|
+
status: "delivered",
|
|
105
|
+
});
|
|
106
|
+
const childB = insertTask({
|
|
107
|
+
from_name: "child-b",
|
|
108
|
+
to_name: "leaf-b",
|
|
109
|
+
network_id: "net-B",
|
|
110
|
+
status: "replied",
|
|
111
|
+
parent_task_id: parentB,
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
chainReplyToParent(childB, "legitimate result", "replied", 5, "net-B");
|
|
115
|
+
|
|
116
|
+
const after = getTask(parentB);
|
|
117
|
+
expect(after?.result).toContain("legitimate result");
|
|
118
|
+
expect(after?.status).toBe("replied");
|
|
119
|
+
expect(countInbox("admin-b")).toBe(1);
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test("default-network (null) caller cannot chain into named network", () => {
|
|
123
|
+
const parentNamed = insertTask({
|
|
124
|
+
from_name: "admin-x",
|
|
125
|
+
to_name: "worker-x",
|
|
126
|
+
network_id: "net-X",
|
|
127
|
+
status: "delivered",
|
|
128
|
+
});
|
|
129
|
+
const childDefault = insertTask({
|
|
130
|
+
from_name: "stray",
|
|
131
|
+
to_name: "leaf",
|
|
132
|
+
network_id: null,
|
|
133
|
+
status: "replied",
|
|
134
|
+
parent_task_id: parentNamed,
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
chainReplyToParent(childDefault, "stray payload", "replied", 5, null);
|
|
138
|
+
|
|
139
|
+
expect(getTask(parentNamed)?.result).toBeNull();
|
|
140
|
+
expect(countInbox("admin-x")).toBe(0);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
test("named-network caller cannot chain into default-network parent", () => {
|
|
144
|
+
const parentDefault = insertTask({
|
|
145
|
+
from_name: "admin-default",
|
|
146
|
+
to_name: "worker-default",
|
|
147
|
+
network_id: null,
|
|
148
|
+
status: "delivered",
|
|
149
|
+
});
|
|
150
|
+
const childNamed = insertTask({
|
|
151
|
+
from_name: "named",
|
|
152
|
+
to_name: "leaf",
|
|
153
|
+
network_id: "net-Y",
|
|
154
|
+
status: "replied",
|
|
155
|
+
parent_task_id: parentDefault,
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
chainReplyToParent(childNamed, "named payload", "replied", 5, "net-Y");
|
|
159
|
+
|
|
160
|
+
expect(getTask(parentDefault)?.result).toBeNull();
|
|
161
|
+
expect(countInbox("admin-default")).toBe(0);
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
test("recursive chain stops at the first cross-network boundary", () => {
|
|
165
|
+
// 3-level chain: grandparent (net-A) ← parent (net-B) ← child (net-B)
|
|
166
|
+
// Caller B replies. F2 #3 should write into parent (same net) but
|
|
167
|
+
// refuse to recurse into grandparent (foreign net).
|
|
168
|
+
const grandparent = insertTask({
|
|
169
|
+
from_name: "gp-admin",
|
|
170
|
+
to_name: "gp-worker",
|
|
171
|
+
network_id: "net-A",
|
|
172
|
+
status: "delivered",
|
|
173
|
+
});
|
|
174
|
+
const parent = insertTask({
|
|
175
|
+
from_name: "p-admin",
|
|
176
|
+
to_name: "p-worker",
|
|
177
|
+
network_id: "net-B",
|
|
178
|
+
status: "delivered",
|
|
179
|
+
parent_task_id: grandparent,
|
|
180
|
+
});
|
|
181
|
+
const child = insertTask({
|
|
182
|
+
from_name: "c",
|
|
183
|
+
to_name: "leaf",
|
|
184
|
+
network_id: "net-B",
|
|
185
|
+
status: "replied",
|
|
186
|
+
parent_task_id: parent,
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
chainReplyToParent(child, "result", "replied", 5, "net-B");
|
|
190
|
+
|
|
191
|
+
// Parent (same network) — chain wrote.
|
|
192
|
+
expect(getTask(parent)?.result).toContain("result");
|
|
193
|
+
expect(getTask(parent)?.status).toBe("replied");
|
|
194
|
+
// Grandparent (foreign network) — chain refused.
|
|
195
|
+
expect(getTask(grandparent)?.result).toBeNull();
|
|
196
|
+
expect(getTask(grandparent)?.status).toBe("delivered");
|
|
197
|
+
expect(countInbox("gp-admin")).toBe(0);
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
test("undefined callerNetId preserves legacy network-blind behaviour", () => {
|
|
201
|
+
// Internal/server-only callers that have no tenant context can
|
|
202
|
+
// still call chainReplyToParent without the enforcement (they
|
|
203
|
+
// pass undefined). This is the back-compat escape hatch.
|
|
204
|
+
const parent = insertTask({
|
|
205
|
+
from_name: "internal-admin",
|
|
206
|
+
to_name: "internal-worker",
|
|
207
|
+
network_id: "net-A",
|
|
208
|
+
status: "delivered",
|
|
209
|
+
});
|
|
210
|
+
const child = insertTask({
|
|
211
|
+
from_name: "child",
|
|
212
|
+
to_name: "leaf",
|
|
213
|
+
network_id: "net-B",
|
|
214
|
+
status: "replied",
|
|
215
|
+
parent_task_id: parent,
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
chainReplyToParent(child, "legacy result", "replied"); // no callerNetId
|
|
219
|
+
|
|
220
|
+
// Legacy behaviour: writes through regardless of network mismatch.
|
|
221
|
+
// Documents the back-compat surface so a future tightening is a
|
|
222
|
+
// deliberate decision.
|
|
223
|
+
expect(getTask(parent)?.result).toContain("legacy result");
|
|
224
|
+
});
|
|
225
|
+
});
|
package/src/db-adapter.ts
CHANGED
|
@@ -233,8 +233,8 @@ export class PgAdapter implements DbAdapter {
|
|
|
233
233
|
*
|
|
234
234
|
* Operators who run `bun -e` snippets that touch the DB are expected to
|
|
235
235
|
* `COMMHUB_DB=/tmp/...` themselves — `bun -e` doesn't set NODE_ENV, so the
|
|
236
|
-
* guard can't catch that case. The rule
|
|
237
|
-
*
|
|
236
|
+
* guard can't catch that case. The rule (never read or write the
|
|
237
|
+
* production hub database) is documented in CLAUDE.md.
|
|
238
238
|
*/
|
|
239
239
|
export function createAdapter(): DbAdapter {
|
|
240
240
|
const dbUrl = process.env.DATABASE_URL;
|
package/src/db.ts
CHANGED
|
@@ -231,6 +231,21 @@ db.exec(`
|
|
|
231
231
|
CREATE INDEX IF NOT EXISTS idx_users_username ON users(username);
|
|
232
232
|
`);
|
|
233
233
|
|
|
234
|
+
// #261 P0-2 (2026-06-28): must_change_password flag. Added via ALTER
|
|
235
|
+
// (not in the CREATE block above) so it's a no-op on fresh DBs that
|
|
236
|
+
// already get the column via the CREATE path's full definition AND a
|
|
237
|
+
// pure additive migration on existing prod DBs — column defaults 0
|
|
238
|
+
// for every existing row, so old `admin/anethub` deployments are NOT
|
|
239
|
+
// locked or nudged on upgrade (back-compat per 通信龙 spec). Wrapped
|
|
240
|
+
// in try/catch because SQLite throws "duplicate column" if the column
|
|
241
|
+
// already exists on subsequent restarts (no IF NOT EXISTS syntax for
|
|
242
|
+
// ALTER ADD COLUMN until SQLite 3.35+, can't assume that everywhere).
|
|
243
|
+
try {
|
|
244
|
+
db.exec(`ALTER TABLE users ADD COLUMN must_change_password INTEGER DEFAULT 0`);
|
|
245
|
+
} catch (e: any) {
|
|
246
|
+
if (!/duplicate column|already exists/i.test(e?.message || "")) throw e;
|
|
247
|
+
}
|
|
248
|
+
|
|
234
249
|
// ── V3: networks table ──
|
|
235
250
|
db.exec(`
|
|
236
251
|
CREATE TABLE IF NOT EXISTS networks (
|
|
@@ -734,10 +749,37 @@ export function logAudit(userId: string | null, username: string | null, action:
|
|
|
734
749
|
// 'replied' if it's still open, and post an inbox notification to the parent's
|
|
735
750
|
// originator (so an upstream agent or the dashboard sees the chained answer).
|
|
736
751
|
// Recurse up the chain (in case of N hops).
|
|
737
|
-
|
|
752
|
+
/**
|
|
753
|
+
* Auto-chain a sub-task reply up to the parent task lineage.
|
|
754
|
+
*
|
|
755
|
+
* `callerNetId` (round5 F2 fix): the network the *caller* (send_reply
|
|
756
|
+
* / report_completion) is operating in. When supplied, the chain
|
|
757
|
+
* refuses to traverse into a parent whose `network_id` doesn't match —
|
|
758
|
+
* blocks cross-tenant write where network B replies to a task they've
|
|
759
|
+
* named as parent but which actually lives in network A.
|
|
760
|
+
*
|
|
761
|
+
* Callers that legitimately need the legacy network-blind behavior
|
|
762
|
+
* (e.g. server-internal bookkeeping that doesn't have a tenant
|
|
763
|
+
* context) can omit `callerNetId`, but every MCP tool path MUST
|
|
764
|
+
* supply it.
|
|
765
|
+
*/
|
|
766
|
+
export function chainReplyToParent(
|
|
767
|
+
childTaskId: string,
|
|
768
|
+
replyText: string,
|
|
769
|
+
replyStatus: "replied" | "failed" | "cancelled" = "replied",
|
|
770
|
+
maxDepth = 5,
|
|
771
|
+
callerNetId?: string | null,
|
|
772
|
+
): void {
|
|
738
773
|
let currentChildId: string | null = childTaskId;
|
|
739
774
|
let currentReply = replyText;
|
|
740
775
|
let depth = 0;
|
|
776
|
+
// Normalize so undefined ("don't enforce") stays distinct from
|
|
777
|
+
// null ("explicit default network"). Caller passes undefined to
|
|
778
|
+
// opt out of the cross-tenant check; null to require the parent
|
|
779
|
+
// also live in the default (null) network.
|
|
780
|
+
const enforce = callerNetId !== undefined;
|
|
781
|
+
const callerNorm = callerNetId ?? null;
|
|
782
|
+
|
|
741
783
|
while (currentChildId && depth < maxDepth) {
|
|
742
784
|
depth++;
|
|
743
785
|
type ChildRow = { parent_task_id: string | null; to_name: string; from_name: string; content: string };
|
|
@@ -753,6 +795,18 @@ export function chainReplyToParent(childTaskId: string, replyText: string, reply
|
|
|
753
795
|
);
|
|
754
796
|
if (!parent) return;
|
|
755
797
|
|
|
798
|
+
// round5 F2 fix: refuse to write into a parent that lives in a
|
|
799
|
+
// different network than the caller. Stops chain-reply being a
|
|
800
|
+
// cross-tenant write primitive. Logs once per attempt so ops can
|
|
801
|
+
// see the rejection in the access log.
|
|
802
|
+
if (enforce) {
|
|
803
|
+
const parentNet = parent.network_id ?? null;
|
|
804
|
+
if (parentNet !== callerNorm) {
|
|
805
|
+
console.log(`[commhub] 🚫 chainReplyToParent cross-network blocked: parent=${parent.task_id.slice(0, 8)} parent-net=${parentNet ?? "null"} caller-net=${callerNorm ?? "null"}`);
|
|
806
|
+
return;
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
|
|
756
810
|
const childAlias = child.to_name;
|
|
757
811
|
const marker = `\n\n[via ${childAlias} 子任务结果]\n${currentReply}`;
|
|
758
812
|
const newResult = parent.result ? parent.result + marker : `[via ${childAlias} 子任务结果]\n${currentReply}`;
|
package/src/push.test.ts
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
|
-
import { afterEach, expect, test } from "bun:test";
|
|
1
|
+
import { afterEach, expect, test, describe } from "bun:test";
|
|
2
2
|
import { eventBus } from "./event_bus";
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
__resetSSEClientsForTest,
|
|
5
|
+
createSSEStream,
|
|
6
|
+
getSSEStats,
|
|
7
|
+
pushEvent,
|
|
8
|
+
__SSE_INTERNALS_FOR_TEST,
|
|
9
|
+
} from "./push";
|
|
4
10
|
|
|
5
11
|
afterEach(() => {
|
|
6
12
|
__resetSSEClientsForTest();
|
|
@@ -45,3 +51,378 @@ test("rename-committed merges old and new SSE client buckets without dropping cl
|
|
|
45
51
|
await oldReader.cancel();
|
|
46
52
|
await newReader.cancel();
|
|
47
53
|
});
|
|
54
|
+
|
|
55
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
56
|
+
// Round-2/4 fix: SSE backpressure + half-open detection
|
|
57
|
+
//
|
|
58
|
+
// These tests exercise tryEnqueueBytes / sweepLiveness directly with a
|
|
59
|
+
// hand-rolled fake ReadableStreamDefaultController so we can model the
|
|
60
|
+
// failure modes that matter — a controller whose `desiredSize` goes
|
|
61
|
+
// negative because the consumer stopped reading (half-open TCP), and
|
|
62
|
+
// one whose enqueue throws because the runtime already errored it.
|
|
63
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
64
|
+
|
|
65
|
+
const enc = new TextEncoder();
|
|
66
|
+
|
|
67
|
+
type FakeCtrlOpts = {
|
|
68
|
+
desiredSize?: number | null;
|
|
69
|
+
enqueueThrows?: Error;
|
|
70
|
+
onClose?: () => void;
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
/** Hand-rolled controller stand-in. `desiredSize` is mutable so a test
|
|
74
|
+
* can flip a "client" from healthy → backpressured by tweaking one
|
|
75
|
+
* number; `enqueue` records the bytes (for assertion) and optionally
|
|
76
|
+
* throws. */
|
|
77
|
+
function fakeController(opts: FakeCtrlOpts = {}) {
|
|
78
|
+
const enqueued: Uint8Array[] = [];
|
|
79
|
+
let closed = false;
|
|
80
|
+
const hasDesired = "desiredSize" in opts;
|
|
81
|
+
const ctrl = {
|
|
82
|
+
get desiredSize(): number | null {
|
|
83
|
+
// Explicit null must NOT coalesce to the default — the whole point
|
|
84
|
+
// of these tests is to exercise the half-open path where
|
|
85
|
+
// desiredSize comes back null.
|
|
86
|
+
return hasDesired ? (opts.desiredSize as number | null) : 1024;
|
|
87
|
+
},
|
|
88
|
+
enqueue(bytes: Uint8Array) {
|
|
89
|
+
if (opts.enqueueThrows) throw opts.enqueueThrows;
|
|
90
|
+
enqueued.push(bytes);
|
|
91
|
+
},
|
|
92
|
+
close() {
|
|
93
|
+
if (!closed) {
|
|
94
|
+
closed = true;
|
|
95
|
+
opts.onClose?.();
|
|
96
|
+
}
|
|
97
|
+
},
|
|
98
|
+
_enqueued: enqueued,
|
|
99
|
+
_closed: () => closed,
|
|
100
|
+
_setDesired(n: number | null) {
|
|
101
|
+
opts.desiredSize = n;
|
|
102
|
+
},
|
|
103
|
+
};
|
|
104
|
+
return ctrl;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function makeClient(ctrl: ReturnType<typeof fakeController>) {
|
|
108
|
+
return {
|
|
109
|
+
controller: ctrl as unknown as ReadableStreamDefaultController,
|
|
110
|
+
encoder: enc,
|
|
111
|
+
stuckSince: null as number | null,
|
|
112
|
+
closed: false,
|
|
113
|
+
key: "test:fake",
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
describe("tryEnqueueBytes — backpressure ceiling", () => {
|
|
118
|
+
test("desiredSize healthy → enqueue OK, no close", () => {
|
|
119
|
+
const ctrl = fakeController({ desiredSize: 500_000 });
|
|
120
|
+
const c = makeClient(ctrl);
|
|
121
|
+
const r = __SSE_INTERNALS_FOR_TEST.tryEnqueueBytes(c, enc.encode("hello"));
|
|
122
|
+
expect(r).toBe("ok");
|
|
123
|
+
expect(c.closed).toBe(false);
|
|
124
|
+
expect(ctrl._enqueued).toHaveLength(1);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test("desiredSize < -MAX_BACKPRESSURE → close, return dead, no enqueue", () => {
|
|
128
|
+
const ceiling = __SSE_INTERNALS_FOR_TEST.MAX_QUEUE_BACKPRESSURE_BYTES;
|
|
129
|
+
const ctrl = fakeController({ desiredSize: -(ceiling + 1) });
|
|
130
|
+
const c = makeClient(ctrl);
|
|
131
|
+
const r = __SSE_INTERNALS_FOR_TEST.tryEnqueueBytes(c, enc.encode("x"));
|
|
132
|
+
expect(r).toBe("dead");
|
|
133
|
+
expect(c.closed).toBe(true);
|
|
134
|
+
expect(ctrl._closed()).toBe(true);
|
|
135
|
+
expect(ctrl._enqueued).toHaveLength(0);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
test("desiredSize negative but within ceiling → enqueue still OK, stuckSince set", () => {
|
|
139
|
+
const ctrl = fakeController({ desiredSize: -100 });
|
|
140
|
+
const c = makeClient(ctrl);
|
|
141
|
+
expect(c.stuckSince).toBeNull();
|
|
142
|
+
const r = __SSE_INTERNALS_FOR_TEST.tryEnqueueBytes(c, enc.encode("x"));
|
|
143
|
+
expect(r).toBe("ok");
|
|
144
|
+
expect(c.closed).toBe(false);
|
|
145
|
+
expect(c.stuckSince).not.toBeNull();
|
|
146
|
+
expect(ctrl._enqueued).toHaveLength(1);
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test("desiredSize recovers to positive → stuckSince cleared", () => {
|
|
150
|
+
const ctrl = fakeController({ desiredSize: -100 });
|
|
151
|
+
const c = makeClient(ctrl);
|
|
152
|
+
__SSE_INTERNALS_FOR_TEST.tryEnqueueBytes(c, enc.encode("a"));
|
|
153
|
+
expect(c.stuckSince).not.toBeNull();
|
|
154
|
+
ctrl._setDesired(500);
|
|
155
|
+
__SSE_INTERNALS_FOR_TEST.tryEnqueueBytes(c, enc.encode("b"));
|
|
156
|
+
expect(c.stuckSince).toBeNull();
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
test("desiredSize null (closed/errored controller) → close, return dead", () => {
|
|
160
|
+
const ctrl = fakeController({ desiredSize: null });
|
|
161
|
+
const c = makeClient(ctrl);
|
|
162
|
+
const r = __SSE_INTERNALS_FOR_TEST.tryEnqueueBytes(c, enc.encode("x"));
|
|
163
|
+
expect(r).toBe("dead");
|
|
164
|
+
expect(c.closed).toBe(true);
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
test("enqueue throws → close, return dead", () => {
|
|
168
|
+
const ctrl = fakeController({
|
|
169
|
+
desiredSize: 100,
|
|
170
|
+
enqueueThrows: new Error("controller errored"),
|
|
171
|
+
});
|
|
172
|
+
const c = makeClient(ctrl);
|
|
173
|
+
const r = __SSE_INTERNALS_FOR_TEST.tryEnqueueBytes(c, enc.encode("x"));
|
|
174
|
+
expect(r).toBe("dead");
|
|
175
|
+
expect(c.closed).toBe(true);
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
test("already-closed client → return dead immediately, no enqueue/close attempt", () => {
|
|
179
|
+
const ctrl = fakeController({ desiredSize: 100 });
|
|
180
|
+
const c = makeClient(ctrl);
|
|
181
|
+
c.closed = true;
|
|
182
|
+
const r = __SSE_INTERNALS_FOR_TEST.tryEnqueueBytes(c, enc.encode("x"));
|
|
183
|
+
expect(r).toBe("dead");
|
|
184
|
+
expect(ctrl._enqueued).toHaveLength(0);
|
|
185
|
+
expect(ctrl._closed()).toBe(false); // never re-closed
|
|
186
|
+
});
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
describe("tryEnqueueBytes — stuck timeout", () => {
|
|
190
|
+
test("stuck longer than STUCK_CLOSE_MS → close, return dead", () => {
|
|
191
|
+
const ctrl = fakeController({ desiredSize: -500 });
|
|
192
|
+
const c = makeClient(ctrl);
|
|
193
|
+
// Backdate stuckSince so the threshold has elapsed.
|
|
194
|
+
c.stuckSince = Date.now() - (__SSE_INTERNALS_FOR_TEST.STUCK_CLOSE_MS + 1000);
|
|
195
|
+
const r = __SSE_INTERNALS_FOR_TEST.tryEnqueueBytes(c, enc.encode("x"));
|
|
196
|
+
expect(r).toBe("dead");
|
|
197
|
+
expect(c.closed).toBe(true);
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
test("stuck but below STUCK_CLOSE_MS → enqueue still OK", () => {
|
|
201
|
+
const ctrl = fakeController({ desiredSize: -500 });
|
|
202
|
+
const c = makeClient(ctrl);
|
|
203
|
+
c.stuckSince = Date.now() - 1000; // far less than 60s default
|
|
204
|
+
const r = __SSE_INTERNALS_FOR_TEST.tryEnqueueBytes(c, enc.encode("x"));
|
|
205
|
+
expect(r).toBe("ok");
|
|
206
|
+
expect(c.closed).toBe(false);
|
|
207
|
+
});
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
describe("sweepLiveness — proactive half-open detection", () => {
|
|
211
|
+
test("sweep closes a client whose desiredSize is null", () => {
|
|
212
|
+
const ctrl = fakeController({ desiredSize: null });
|
|
213
|
+
const c = makeClient(ctrl);
|
|
214
|
+
__SSE_INTERNALS_FOR_TEST.clientsMap.set("test:sweep-null", [c as any]);
|
|
215
|
+
__SSE_INTERNALS_FOR_TEST.sweepLiveness();
|
|
216
|
+
expect(c.closed).toBe(true);
|
|
217
|
+
expect(__SSE_INTERNALS_FOR_TEST.clientsMap.has("test:sweep-null")).toBe(false);
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
test("sweep closes a client over the backpressure ceiling without any push", () => {
|
|
221
|
+
const ceiling = __SSE_INTERNALS_FOR_TEST.MAX_QUEUE_BACKPRESSURE_BYTES;
|
|
222
|
+
const ctrl = fakeController({ desiredSize: -(ceiling + 5_000_000) });
|
|
223
|
+
const c = makeClient(ctrl);
|
|
224
|
+
__SSE_INTERNALS_FOR_TEST.clientsMap.set("test:sweep-over", [c as any]);
|
|
225
|
+
__SSE_INTERNALS_FOR_TEST.sweepLiveness();
|
|
226
|
+
expect(c.closed).toBe(true);
|
|
227
|
+
expect(__SSE_INTERNALS_FOR_TEST.clientsMap.has("test:sweep-over")).toBe(false);
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
test("sweep closes a client stuck longer than STUCK_CLOSE_MS", () => {
|
|
231
|
+
const ctrl = fakeController({ desiredSize: -500 });
|
|
232
|
+
const c = makeClient(ctrl);
|
|
233
|
+
c.stuckSince = Date.now() - (__SSE_INTERNALS_FOR_TEST.STUCK_CLOSE_MS + 5_000);
|
|
234
|
+
__SSE_INTERNALS_FOR_TEST.clientsMap.set("test:sweep-stuck", [c as any]);
|
|
235
|
+
__SSE_INTERNALS_FOR_TEST.sweepLiveness();
|
|
236
|
+
expect(c.closed).toBe(true);
|
|
237
|
+
expect(__SSE_INTERNALS_FOR_TEST.clientsMap.has("test:sweep-stuck")).toBe(false);
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
test("sweep first sees stuck → records stuckSince, does not close yet", () => {
|
|
241
|
+
const ctrl = fakeController({ desiredSize: -500 });
|
|
242
|
+
const c = makeClient(ctrl);
|
|
243
|
+
expect(c.stuckSince).toBeNull();
|
|
244
|
+
__SSE_INTERNALS_FOR_TEST.clientsMap.set("test:sweep-firststuck", [c as any]);
|
|
245
|
+
__SSE_INTERNALS_FOR_TEST.sweepLiveness();
|
|
246
|
+
expect(c.closed).toBe(false);
|
|
247
|
+
expect(c.stuckSince).not.toBeNull();
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
test("sweep leaves healthy clients alone", () => {
|
|
251
|
+
const ctrl = fakeController({ desiredSize: 500 });
|
|
252
|
+
const c = makeClient(ctrl);
|
|
253
|
+
__SSE_INTERNALS_FOR_TEST.clientsMap.set("test:sweep-healthy", [c as any]);
|
|
254
|
+
__SSE_INTERNALS_FOR_TEST.sweepLiveness();
|
|
255
|
+
expect(c.closed).toBe(false);
|
|
256
|
+
expect(__SSE_INTERNALS_FOR_TEST.clientsMap.get("test:sweep-healthy")).toHaveLength(1);
|
|
257
|
+
});
|
|
258
|
+
|
|
259
|
+
test("sweep prunes only dead entries when multiple clients share a key", () => {
|
|
260
|
+
const dead = fakeController({ desiredSize: null });
|
|
261
|
+
const alive = fakeController({ desiredSize: 500 });
|
|
262
|
+
const cDead = makeClient(dead);
|
|
263
|
+
const cAlive = makeClient(alive);
|
|
264
|
+
__SSE_INTERNALS_FOR_TEST.clientsMap.set("test:sweep-mixed", [cDead, cAlive] as any);
|
|
265
|
+
__SSE_INTERNALS_FOR_TEST.sweepLiveness();
|
|
266
|
+
const remaining = __SSE_INTERNALS_FOR_TEST.clientsMap.get("test:sweep-mixed");
|
|
267
|
+
expect(remaining).toHaveLength(1);
|
|
268
|
+
expect(cDead.closed).toBe(true);
|
|
269
|
+
expect(cAlive.closed).toBe(false);
|
|
270
|
+
});
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
describe("pushEvent integration with backpressure", () => {
|
|
274
|
+
test("push to half-open client → client evicted from map", () => {
|
|
275
|
+
const ctrl = fakeController({ desiredSize: null });
|
|
276
|
+
const c = makeClient(ctrl);
|
|
277
|
+
c.key = "global:half-open";
|
|
278
|
+
__SSE_INTERNALS_FOR_TEST.clientsMap.set("global:half-open", [c as any]);
|
|
279
|
+
pushEvent("half-open", { type: "test", payload: 1 });
|
|
280
|
+
expect(c.closed).toBe(true);
|
|
281
|
+
expect(__SSE_INTERNALS_FOR_TEST.clientsMap.has("global:half-open")).toBe(false);
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
test("push to mixed (one alive, one dead) → alive receives, dead pruned", () => {
|
|
285
|
+
const dead = fakeController({ desiredSize: null });
|
|
286
|
+
const alive = fakeController({ desiredSize: 500 });
|
|
287
|
+
const cDead = makeClient(dead);
|
|
288
|
+
cDead.key = "global:mixed";
|
|
289
|
+
const cAlive = makeClient(alive);
|
|
290
|
+
cAlive.key = "global:mixed";
|
|
291
|
+
__SSE_INTERNALS_FOR_TEST.clientsMap.set("global:mixed", [cDead, cAlive] as any);
|
|
292
|
+
pushEvent("mixed", { type: "msg" });
|
|
293
|
+
expect(alive._enqueued).toHaveLength(1);
|
|
294
|
+
expect(__SSE_INTERNALS_FOR_TEST.clientsMap.get("global:mixed")).toHaveLength(1);
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
test("push to over-ceiling client → client closed, no enqueue, map evicted", () => {
|
|
298
|
+
const ceiling = __SSE_INTERNALS_FOR_TEST.MAX_QUEUE_BACKPRESSURE_BYTES;
|
|
299
|
+
const ctrl = fakeController({ desiredSize: -(ceiling + 1) });
|
|
300
|
+
const c = makeClient(ctrl);
|
|
301
|
+
c.key = "global:over";
|
|
302
|
+
__SSE_INTERNALS_FOR_TEST.clientsMap.set("global:over", [c as any]);
|
|
303
|
+
pushEvent("over", { type: "msg" });
|
|
304
|
+
expect(ctrl._enqueued).toHaveLength(0);
|
|
305
|
+
expect(c.closed).toBe(true);
|
|
306
|
+
expect(__SSE_INTERNALS_FOR_TEST.clientsMap.has("global:over")).toBe(false);
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
test("push to unknown session is a no-op (no throw)", () => {
|
|
310
|
+
expect(() => pushEvent("never-connected", { type: "x" })).not.toThrow();
|
|
311
|
+
});
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
315
|
+
// Round-2 review fix: byte-level backpressure on a REAL ReadableStream
|
|
316
|
+
//
|
|
317
|
+
// The fake-controller tests above can't catch the unit-mismatch bug
|
|
318
|
+
// (treating desiredSize as bytes when the runtime treats it as chunk
|
|
319
|
+
// count). These tests exercise the actual `createSSEStream` Response
|
|
320
|
+
// against a non-reading consumer and verify that bytes — not chunks —
|
|
321
|
+
// trip the backpressure cap.
|
|
322
|
+
//
|
|
323
|
+
// Probe baseline (per 通信牛, also verified locally in this test):
|
|
324
|
+
// - Default ReadableStream (no strategy) → desiredSize is the
|
|
325
|
+
// CHUNK count. 1MB chunk → desiredSize goes 1 → 0; second chunk
|
|
326
|
+
// → -1. Million 1-byte chunks needed to hit our -1MB ceiling.
|
|
327
|
+
// - Byte-counting strategy `{highWaterMark: 0, size: c.byteLength}`
|
|
328
|
+
// → desiredSize is the BYTE count. 1MB chunk → -1_000_000.
|
|
329
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
330
|
+
|
|
331
|
+
describe("real ReadableStream — byte-level backpressure (round-2 fix)", () => {
|
|
332
|
+
test("createSSEStream uses byte-counting strategy: desiredSize tracks bytes not chunks", async () => {
|
|
333
|
+
const res = createSSEStream("byte-strategy-probe", "net-bsp");
|
|
334
|
+
// Don't read from res.body. The stream sits with whatever was
|
|
335
|
+
// enqueued in `start()` (initial 'connected' frame) until we push.
|
|
336
|
+
const key = "net-bsp:byte-strategy-probe";
|
|
337
|
+
const arr = __SSE_INTERNALS_FOR_TEST.clientsMap.get(key);
|
|
338
|
+
expect(arr).toBeDefined();
|
|
339
|
+
expect(arr).toHaveLength(1);
|
|
340
|
+
const ctrl = arr![0].controller;
|
|
341
|
+
|
|
342
|
+
// The unit check: enqueue a 1 KB chunk and verify desiredSize
|
|
343
|
+
// drops by ~1024, not by 1. With chunk-count strategy desiredSize
|
|
344
|
+
// would drop by exactly 1; with byte strategy by 1024.
|
|
345
|
+
const before = ctrl.desiredSize!;
|
|
346
|
+
ctrl.enqueue(new Uint8Array(1024));
|
|
347
|
+
const after = ctrl.desiredSize!;
|
|
348
|
+
expect(before - after).toBeGreaterThanOrEqual(1024);
|
|
349
|
+
|
|
350
|
+
// Cleanup so the afterEach reset doesn't leak.
|
|
351
|
+
await res.body!.cancel();
|
|
352
|
+
});
|
|
353
|
+
|
|
354
|
+
test("non-reading consumer past MAX bytes triggers hard ceiling close", async () => {
|
|
355
|
+
const ceiling = __SSE_INTERNALS_FOR_TEST.MAX_QUEUE_BACKPRESSURE_BYTES;
|
|
356
|
+
const res = createSSEStream("byte-cap-probe", "net-bcp");
|
|
357
|
+
const key = "net-bcp:byte-cap-probe";
|
|
358
|
+
const c = __SSE_INTERNALS_FOR_TEST.clientsMap.get(key)![0];
|
|
359
|
+
|
|
360
|
+
// Push events whose total byte size exceeds the hard ceiling.
|
|
361
|
+
// With chunk-count strategy this would NOT close (each event is 1
|
|
362
|
+
// chunk; desiredSize would be at -N for N events, never -1M).
|
|
363
|
+
// With byte strategy this SHOULD close once accumulated bytes
|
|
364
|
+
// exceed ceiling.
|
|
365
|
+
const chunkSize = 100_000; // 100KB per push
|
|
366
|
+
const needed = Math.ceil(ceiling / chunkSize) + 2; // overshoot a bit
|
|
367
|
+
const big = "x".repeat(chunkSize);
|
|
368
|
+
for (let i = 0; i < needed; i++) {
|
|
369
|
+
if (c.closed) break;
|
|
370
|
+
pushEvent("byte-cap-probe", { type: "big", payload: big }, "net-bcp");
|
|
371
|
+
}
|
|
372
|
+
expect(c.closed).toBe(true);
|
|
373
|
+
expect(__SSE_INTERNALS_FOR_TEST.clientsMap.has(key)).toBe(false);
|
|
374
|
+
|
|
375
|
+
try { await res.body!.cancel(); } catch { /* already closed */ }
|
|
376
|
+
});
|
|
377
|
+
|
|
378
|
+
test("reading consumer drains queue → no spurious close even after many pushes", async () => {
|
|
379
|
+
const res = createSSEStream("drain-probe", "net-drain");
|
|
380
|
+
const reader = res.body!.getReader();
|
|
381
|
+
const key = "net-drain:drain-probe";
|
|
382
|
+
const c = __SSE_INTERNALS_FOR_TEST.clientsMap.get(key)![0];
|
|
383
|
+
|
|
384
|
+
// Drain in the background while pushes happen.
|
|
385
|
+
let drained = 0;
|
|
386
|
+
const drainPromise = (async () => {
|
|
387
|
+
while (drained < 50) {
|
|
388
|
+
const { done, value } = await reader.read();
|
|
389
|
+
if (done) break;
|
|
390
|
+
if (value) drained += value.byteLength;
|
|
391
|
+
}
|
|
392
|
+
})();
|
|
393
|
+
|
|
394
|
+
for (let i = 0; i < 20; i++) {
|
|
395
|
+
pushEvent("drain-probe", { type: "tick", n: i }, "net-drain");
|
|
396
|
+
}
|
|
397
|
+
await drainPromise;
|
|
398
|
+
expect(c.closed).toBe(false);
|
|
399
|
+
expect(__SSE_INTERNALS_FOR_TEST.clientsMap.has(key)).toBe(true);
|
|
400
|
+
|
|
401
|
+
await reader.cancel();
|
|
402
|
+
});
|
|
403
|
+
});
|
|
404
|
+
|
|
405
|
+
describe("getSSEStats — closed clients excluded", () => {
|
|
406
|
+
test("manually-closed client is not counted in stats", () => {
|
|
407
|
+
const ctrl = fakeController({ desiredSize: 500 });
|
|
408
|
+
const c = makeClient(ctrl);
|
|
409
|
+
c.key = "global:stats-test";
|
|
410
|
+
c.closed = true;
|
|
411
|
+
__SSE_INTERNALS_FOR_TEST.clientsMap.set("global:stats-test", [c as any]);
|
|
412
|
+
const stats = getSSEStats();
|
|
413
|
+
expect(stats.sessions["global:stats-test"]).toBeUndefined();
|
|
414
|
+
expect(stats.total).toBe(0);
|
|
415
|
+
});
|
|
416
|
+
|
|
417
|
+
test("mixed alive + closed clients → only alive count", () => {
|
|
418
|
+
const dead = fakeController({ desiredSize: 500 });
|
|
419
|
+
const alive = fakeController({ desiredSize: 500 });
|
|
420
|
+
const cDead = makeClient(dead);
|
|
421
|
+
cDead.closed = true;
|
|
422
|
+
const cAlive = makeClient(alive);
|
|
423
|
+
__SSE_INTERNALS_FOR_TEST.clientsMap.set("global:stats-mixed", [cDead, cAlive] as any);
|
|
424
|
+
const stats = getSSEStats();
|
|
425
|
+
expect(stats.sessions["global:stats-mixed"]).toBe(1);
|
|
426
|
+
expect(stats.total).toBe(1);
|
|
427
|
+
});
|
|
428
|
+
});
|
package/src/push.ts
CHANGED
|
@@ -1,16 +1,89 @@
|
|
|
1
1
|
// ── SSE Push: 实时推送事件给 Agent ──────────────────
|
|
2
2
|
// Agent 连 GET /events/:session → 保持 SSE 长连接
|
|
3
3
|
// send_task 写 inbox 后 → pushEvent() → 秒达
|
|
4
|
+
//
|
|
5
|
+
// Backpressure & half-open contract (Round-2/4 fix per 通信龙):
|
|
6
|
+
//
|
|
7
|
+
// 0. **CRITICAL**: `desiredSize` units are determined by the stream's
|
|
8
|
+
// queuing strategy. The web-streams default strategy is COUNT-based
|
|
9
|
+
// (each chunk = 1 unit), so a 1 MB chunk only nudges desiredSize by
|
|
10
|
+
// -1. That defeats the byte-cap entirely — pushing a million 1-byte
|
|
11
|
+
// chunks is what trips the threshold, not 1 MB of real bytes.
|
|
12
|
+
// Bun probe verified by 通信牛: default stream after 1 MiB chunk →
|
|
13
|
+
// desiredSize 0, after 2 MiB → -1; byte strategy after 1 MiB → 0,
|
|
14
|
+
// after 2 MiB → -1048576. We MUST construct the stream with an
|
|
15
|
+
// explicit byte-counting strategy:
|
|
16
|
+
// new ReadableStream(src, { highWaterMark: 0, size: c => c.byteLength })
|
|
17
|
+
// With highWaterMark=0, every enqueued byte counts against the cap
|
|
18
|
+
// directly. The HARD CEILING below then closes the stream at
|
|
19
|
+
// -MAX_QUEUE_BACKPRESSURE_BYTES bytes of queued data, which is what
|
|
20
|
+
// we actually want.
|
|
21
|
+
//
|
|
22
|
+
// 1. `ReadableStreamDefaultController.desiredSize` is the SSE queue
|
|
23
|
+
// headroom in BYTES (given the strategy above). Positive = room,
|
|
24
|
+
// 0 = at high-water-mark, negative = over by that many bytes.
|
|
25
|
+
// Without a bound, half-open TCP consumers (network drop, client
|
|
26
|
+
// crash without FIN) silently accumulate bytes in the queue forever
|
|
27
|
+
// — a public-hub OOM vector.
|
|
28
|
+
//
|
|
29
|
+
// 2. We bound it two ways:
|
|
30
|
+
//
|
|
31
|
+
// a. HARD CEILING — `desiredSize < -MAX_QUEUE_BACKPRESSURE_BYTES`
|
|
32
|
+
// → close the stream immediately. Single huge event can't blow
|
|
33
|
+
// up a stuck client.
|
|
34
|
+
//
|
|
35
|
+
// b. STUCK TIMEOUT — `desiredSize < 0` continuously for
|
|
36
|
+
// `STUCK_CLOSE_MS` → close. Catches slow-trickle leaks where
|
|
37
|
+
// the consumer accepts a byte every few seconds but can't keep
|
|
38
|
+
// up.
|
|
39
|
+
//
|
|
40
|
+
// 3. Liveness: every `LIVENESS_SWEEP_MS` we re-check every registered
|
|
41
|
+
// client even when nobody calls `pushEvent`, so a session that goes
|
|
42
|
+
// half-open during a quiet period still gets reaped — not waiting
|
|
43
|
+
// for the next inbox push.
|
|
4
44
|
|
|
5
45
|
import { eventBus, type RenameCommittedEvent } from "./event_bus";
|
|
6
46
|
|
|
47
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
48
|
+
// Tunables (env-overridable for tests and ops)
|
|
49
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
50
|
+
function envNum(k: string, fallback: number): number {
|
|
51
|
+
const v = process.env[k];
|
|
52
|
+
if (!v) return fallback;
|
|
53
|
+
const n = Number(v);
|
|
54
|
+
return Number.isFinite(n) && n > 0 ? n : fallback;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Hard ceiling on per-client SSE queue backpressure before we drop the
|
|
58
|
+
* client. 1 MB by default — generous for normal bursts, lethal for the
|
|
59
|
+
* half-open consumer that has stopped reading entirely. */
|
|
60
|
+
const MAX_QUEUE_BACKPRESSURE_BYTES = envNum("ANET_SSE_MAX_QUEUE_BYTES", 1_000_000);
|
|
61
|
+
|
|
62
|
+
/** How long `desiredSize` can stay negative (i.e. consumer behind HWM)
|
|
63
|
+
* before we declare the client dead. 60s by default — past 2× the
|
|
64
|
+
* keepalive interval, so a live-but-slow consumer wouldn't trip it. */
|
|
65
|
+
const STUCK_CLOSE_MS = envNum("ANET_SSE_STUCK_CLOSE_MS", 60_000);
|
|
66
|
+
|
|
67
|
+
/** Keepalive cadence. Doubles as half-open liveness probe — if enqueue
|
|
68
|
+
* silently piles up bytes here, the stuck-timeout kicks in. */
|
|
69
|
+
const KEEPALIVE_MS = envNum("ANET_SSE_KEEPALIVE_MS", 30_000);
|
|
70
|
+
|
|
71
|
+
/** Liveness sweep cadence — proactive scan even when no events flow. */
|
|
72
|
+
const LIVENESS_SWEEP_MS = envNum("ANET_SSE_LIVENESS_SWEEP_MS", 15_000);
|
|
73
|
+
|
|
7
74
|
type SSEClient = {
|
|
8
75
|
controller: ReadableStreamDefaultController;
|
|
9
76
|
encoder: TextEncoder;
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
77
|
+
/** When desiredSize first went negative; cleared on successful drain. */
|
|
78
|
+
stuckSince: number | null;
|
|
79
|
+
/** Once true, the stream has been closed and the client should be
|
|
80
|
+
* evicted from the map on the next sweep / push. Guards against
|
|
81
|
+
* double-close. */
|
|
82
|
+
closed: boolean;
|
|
83
|
+
/** keepalive timer handle so it can be cleared on close. */
|
|
84
|
+
keepaliveTimer?: ReturnType<typeof setInterval>;
|
|
85
|
+
/** Stable key for log/diagnostic context. */
|
|
86
|
+
key: string;
|
|
14
87
|
};
|
|
15
88
|
|
|
16
89
|
// 一个 session 可能有多个 SSE 连接(重连时短暂并存)
|
|
@@ -20,49 +93,189 @@ function ts(): string {
|
|
|
20
93
|
return new Date().toTimeString().slice(0, 8);
|
|
21
94
|
}
|
|
22
95
|
|
|
96
|
+
function clientKey(sessionName: string, networkId?: string | null): string {
|
|
97
|
+
return `${networkId || "global"}:${sessionName}`;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
101
|
+
// Close + dispatch helpers
|
|
102
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
103
|
+
|
|
104
|
+
/** Close a client's stream (idempotent) and clear its keepalive timer.
|
|
105
|
+
* Does NOT remove from the map — leave that to the caller so it can
|
|
106
|
+
* prune the right index without re-finding it. */
|
|
107
|
+
function closeClient(client: SSEClient, reason: string): void {
|
|
108
|
+
if (client.closed) return;
|
|
109
|
+
client.closed = true;
|
|
110
|
+
if (client.keepaliveTimer) clearInterval(client.keepaliveTimer);
|
|
111
|
+
try {
|
|
112
|
+
client.controller.close();
|
|
113
|
+
} catch {
|
|
114
|
+
// Already closed by the runtime, or controller errored. Either way
|
|
115
|
+
// we're done with it.
|
|
116
|
+
}
|
|
117
|
+
console.log(`[${ts()}] SSE ✕ ${client.key} closed (reason=${reason})`);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Best-effort enqueue with backpressure guard. Returns:
|
|
121
|
+
* - "ok" → bytes accepted, queue healthy
|
|
122
|
+
* - "dead" → client closed; caller must remove from map */
|
|
123
|
+
function tryEnqueueBytes(client: SSEClient, bytes: Uint8Array): "ok" | "dead" {
|
|
124
|
+
if (client.closed) return "dead";
|
|
125
|
+
|
|
126
|
+
const ctrl = client.controller;
|
|
127
|
+
// desiredSize may be null on a closed/errored stream — defensive
|
|
128
|
+
// null-check, then treat null as "no headroom" since we can't enqueue
|
|
129
|
+
// safely anyway.
|
|
130
|
+
const desired = ctrl.desiredSize;
|
|
131
|
+
if (desired === null) {
|
|
132
|
+
closeClient(client, "controller-null-desired");
|
|
133
|
+
return "dead";
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Hard ceiling: too far behind HWM, single push could OOM us. Close.
|
|
137
|
+
if (desired < -MAX_QUEUE_BACKPRESSURE_BYTES) {
|
|
138
|
+
closeClient(client, `queue-overflow-${Math.abs(desired)}b`);
|
|
139
|
+
return "dead";
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Track stuck-since-when for the soft timeout below.
|
|
143
|
+
if (desired < 0) {
|
|
144
|
+
if (client.stuckSince === null) {
|
|
145
|
+
client.stuckSince = Date.now();
|
|
146
|
+
} else if (Date.now() - client.stuckSince > STUCK_CLOSE_MS) {
|
|
147
|
+
closeClient(client, `stuck-${Date.now() - client.stuckSince}ms`);
|
|
148
|
+
return "dead";
|
|
149
|
+
}
|
|
150
|
+
} else {
|
|
151
|
+
client.stuckSince = null;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
try {
|
|
155
|
+
ctrl.enqueue(bytes);
|
|
156
|
+
return "ok";
|
|
157
|
+
} catch (e: any) {
|
|
158
|
+
closeClient(client, `enqueue-throw:${e?.message || e}`);
|
|
159
|
+
return "dead";
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Walk every client and close any that are stuck/half-open. Run on a
|
|
164
|
+
* timer so half-open detection doesn't depend on event traffic. */
|
|
165
|
+
function sweepLiveness(): void {
|
|
166
|
+
const now = Date.now();
|
|
167
|
+
for (const [key, arr] of clients) {
|
|
168
|
+
const alive: SSEClient[] = [];
|
|
169
|
+
for (const c of arr) {
|
|
170
|
+
if (c.closed) continue;
|
|
171
|
+
const desired = c.controller.desiredSize;
|
|
172
|
+
if (desired === null) {
|
|
173
|
+
closeClient(c, "sweep-null-desired");
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
if (desired < -MAX_QUEUE_BACKPRESSURE_BYTES) {
|
|
177
|
+
closeClient(c, `sweep-overflow-${Math.abs(desired)}b`);
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
if (desired < 0) {
|
|
181
|
+
if (c.stuckSince === null) c.stuckSince = now;
|
|
182
|
+
else if (now - c.stuckSince > STUCK_CLOSE_MS) {
|
|
183
|
+
closeClient(c, `sweep-stuck-${now - c.stuckSince}ms`);
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
} else {
|
|
187
|
+
c.stuckSince = null;
|
|
188
|
+
}
|
|
189
|
+
alive.push(c);
|
|
190
|
+
}
|
|
191
|
+
if (alive.length === 0) clients.delete(key);
|
|
192
|
+
else if (alive.length !== arr.length) clients.set(key, alive);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// Liveness sweep timer. Started lazily on first stream creation; never
|
|
197
|
+
// stopped in production (one timer for the lifetime of the process).
|
|
198
|
+
let livenessSweepTimer: ReturnType<typeof setInterval> | null = null;
|
|
199
|
+
function ensureLivenessSweep(): void {
|
|
200
|
+
if (livenessSweepTimer) return;
|
|
201
|
+
livenessSweepTimer = setInterval(() => {
|
|
202
|
+
try {
|
|
203
|
+
sweepLiveness();
|
|
204
|
+
} catch (e: any) {
|
|
205
|
+
console.log(`[${ts()}] SSE sweep failed: ${e?.message || e}`);
|
|
206
|
+
}
|
|
207
|
+
}, LIVENESS_SWEEP_MS);
|
|
208
|
+
// Don't keep the event loop alive just for the sweep.
|
|
209
|
+
(livenessSweepTimer as any)?.unref?.();
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
213
|
+
// Public API
|
|
214
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
215
|
+
|
|
23
216
|
/** 创建 SSE Response 并注册到 clients map */
|
|
24
217
|
export function createSSEStream(sessionName: string, networkId?: string | null): Response {
|
|
25
218
|
const encoder = new TextEncoder();
|
|
26
|
-
let ctrl: ReadableStreamDefaultController;
|
|
27
219
|
const key = clientKey(sessionName, networkId);
|
|
220
|
+
let client: SSEClient;
|
|
28
221
|
|
|
29
|
-
const stream = new ReadableStream({
|
|
222
|
+
const stream = new ReadableStream<Uint8Array>({
|
|
30
223
|
start(controller) {
|
|
31
|
-
|
|
32
|
-
|
|
224
|
+
client = {
|
|
225
|
+
controller,
|
|
226
|
+
encoder,
|
|
227
|
+
stuckSince: null,
|
|
228
|
+
closed: false,
|
|
229
|
+
key,
|
|
230
|
+
};
|
|
33
231
|
|
|
34
|
-
if (!clients.has(key))
|
|
35
|
-
clients.set(key, []);
|
|
36
|
-
}
|
|
232
|
+
if (!clients.has(key)) clients.set(key, []);
|
|
37
233
|
clients.get(key)!.push(client);
|
|
38
234
|
console.log(`[${ts()}] SSE ← ${key} connected (${clients.get(key)!.length} clients)`);
|
|
39
235
|
|
|
40
|
-
//
|
|
41
|
-
|
|
236
|
+
// Send initial connected frame through the backpressure guard so
|
|
237
|
+
// even the first byte respects the contract.
|
|
238
|
+
tryEnqueueBytes(
|
|
239
|
+
client,
|
|
240
|
+
encoder.encode(`data: ${JSON.stringify({ type: "connected", session: sessionName, network_id: networkId ?? null })}\n\n`),
|
|
241
|
+
);
|
|
42
242
|
|
|
43
|
-
// Periodic keepalive
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
clearInterval(
|
|
243
|
+
// Periodic keepalive doubles as a half-open probe — if the
|
|
244
|
+
// consumer is gone, this enqueue piles bytes in the queue until
|
|
245
|
+
// tryEnqueueBytes catches the ceiling / stuck timeout.
|
|
246
|
+
client.keepaliveTimer = setInterval(() => {
|
|
247
|
+
if (client.closed) {
|
|
248
|
+
if (client.keepaliveTimer) clearInterval(client.keepaliveTimer);
|
|
249
|
+
return;
|
|
49
250
|
}
|
|
50
|
-
|
|
51
|
-
|
|
251
|
+
const result = tryEnqueueBytes(client, encoder.encode(`: keepalive\n\n`));
|
|
252
|
+
if (result === "dead") {
|
|
253
|
+
// closeClient already cleared the timer; just make sure we
|
|
254
|
+
// also prune from the map.
|
|
255
|
+
pruneClosed(key);
|
|
256
|
+
}
|
|
257
|
+
}, KEEPALIVE_MS);
|
|
258
|
+
(client.keepaliveTimer as any)?.unref?.();
|
|
259
|
+
|
|
260
|
+
ensureLivenessSweep();
|
|
52
261
|
},
|
|
53
262
|
cancel() {
|
|
54
|
-
//
|
|
55
|
-
|
|
56
|
-
if (
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
if (arr[idx].keepalive) clearInterval(arr[idx].keepalive);
|
|
60
|
-
arr.splice(idx, 1);
|
|
61
|
-
}
|
|
62
|
-
if (arr.length === 0) clients.delete(key);
|
|
63
|
-
console.log(`[${ts()}] SSE ✕ ${key} disconnected (${arr.length} remaining)`);
|
|
263
|
+
// Consumer side hangup — straightforward path, mark closed and
|
|
264
|
+
// prune. The half-open path goes through closeClient instead.
|
|
265
|
+
if (client) {
|
|
266
|
+
closeClient(client, "cancel");
|
|
267
|
+
pruneClosed(key);
|
|
64
268
|
}
|
|
65
269
|
},
|
|
270
|
+
}, {
|
|
271
|
+
// BYTE-COUNTING strategy. Without this, desiredSize is a chunk
|
|
272
|
+
// counter and the byte-cap below is meaningless (see file header
|
|
273
|
+
// comment #0). highWaterMark=0 means: after enqueuing N bytes, the
|
|
274
|
+
// queue is N bytes over the mark, so `desiredSize === -N`. Pair
|
|
275
|
+
// this with the hard ceiling in tryEnqueueBytes to actually close
|
|
276
|
+
// a half-open consumer at MAX_QUEUE_BACKPRESSURE_BYTES.
|
|
277
|
+
highWaterMark: 0,
|
|
278
|
+
size: (chunk: Uint8Array) => chunk.byteLength,
|
|
66
279
|
});
|
|
67
280
|
|
|
68
281
|
return new Response(stream, {
|
|
@@ -74,8 +287,17 @@ export function createSSEStream(sessionName: string, networkId?: string | null):
|
|
|
74
287
|
});
|
|
75
288
|
}
|
|
76
289
|
|
|
77
|
-
|
|
78
|
-
|
|
290
|
+
/** Remove closed entries for one key, drop the map slot if empty. */
|
|
291
|
+
function pruneClosed(key: string): void {
|
|
292
|
+
const arr = clients.get(key);
|
|
293
|
+
if (!arr) return;
|
|
294
|
+
const alive = arr.filter((c) => !c.closed);
|
|
295
|
+
if (alive.length === 0) {
|
|
296
|
+
clients.delete(key);
|
|
297
|
+
console.log(`[${ts()}] SSE ✕ ${key} disconnected (0 remaining)`);
|
|
298
|
+
} else if (alive.length !== arr.length) {
|
|
299
|
+
clients.set(key, alive);
|
|
300
|
+
}
|
|
79
301
|
}
|
|
80
302
|
|
|
81
303
|
function rekeyClient(oldSessionName: string, newSessionName: string, networkId?: string | null): number {
|
|
@@ -85,6 +307,10 @@ function rekeyClient(oldSessionName: string, newSessionName: string, networkId?:
|
|
|
85
307
|
const oldClients = clients.get(oldKey);
|
|
86
308
|
if (!oldClients || oldClients.length === 0) return 0;
|
|
87
309
|
|
|
310
|
+
// Update the cached key on each moved client so later closeClient
|
|
311
|
+
// logs report the right session.
|
|
312
|
+
for (const c of oldClients) c.key = newKey;
|
|
313
|
+
|
|
88
314
|
const existing = clients.get(newKey) || [];
|
|
89
315
|
clients.set(newKey, existing.concat(oldClients));
|
|
90
316
|
clients.delete(oldKey);
|
|
@@ -102,39 +328,34 @@ eventBus.on("rename-committed", (event: RenameCommittedEvent) => {
|
|
|
102
328
|
|
|
103
329
|
/** 推送事件给指定 session 的所有 SSE 连接 */
|
|
104
330
|
export function pushEvent(sessionName: string, event: Record<string, unknown>, networkId?: string | null): void {
|
|
105
|
-
// Resolve the key once — both lookup and post-cleanup `clients.delete` MUST
|
|
106
|
-
// use the same key, otherwise an emptied bucket would leak (the explicit
|
|
107
|
-
// arg wins over event.network_id, falling back so legacy callers that pass
|
|
108
|
-
// network_id inside `event` still hit the right bucket).
|
|
109
331
|
const key = clientKey(sessionName, networkId ?? (event.network_id as string | null | undefined));
|
|
110
332
|
const arr = clients.get(key);
|
|
111
333
|
if (!arr || arr.length === 0) return;
|
|
112
334
|
|
|
113
335
|
const data = `data: ${JSON.stringify(event)}\n\n`;
|
|
114
|
-
|
|
336
|
+
let needPrune = false;
|
|
115
337
|
|
|
116
|
-
for (
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
dead.push(i);
|
|
338
|
+
for (const c of arr) {
|
|
339
|
+
if (c.closed) {
|
|
340
|
+
needPrune = true;
|
|
341
|
+
continue;
|
|
121
342
|
}
|
|
343
|
+
const result = tryEnqueueBytes(c, c.encoder.encode(data));
|
|
344
|
+
if (result === "dead") needPrune = true;
|
|
122
345
|
}
|
|
123
346
|
|
|
124
|
-
|
|
125
|
-
// keep an interval handle alive past its controller (small leak previously
|
|
126
|
-
// bounded by the next failed enqueue catching itself, but cleanup-on-detect
|
|
127
|
-
// is cheaper than waiting 30s).
|
|
128
|
-
for (let i = dead.length - 1; i >= 0; i--) {
|
|
129
|
-
const c = arr[dead[i]];
|
|
130
|
-
if (c.keepalive) clearInterval(c.keepalive);
|
|
131
|
-
arr.splice(dead[i], 1);
|
|
132
|
-
}
|
|
133
|
-
if (arr.length === 0) clients.delete(key);
|
|
347
|
+
if (needPrune) pruneClosed(key);
|
|
134
348
|
}
|
|
135
349
|
|
|
136
350
|
export function __resetSSEClientsForTest(): void {
|
|
351
|
+
for (const arr of clients.values()) {
|
|
352
|
+
for (const c of arr) closeClient(c, "reset-for-test");
|
|
353
|
+
}
|
|
137
354
|
clients.clear();
|
|
355
|
+
if (livenessSweepTimer) {
|
|
356
|
+
clearInterval(livenessSweepTimer);
|
|
357
|
+
livenessSweepTimer = null;
|
|
358
|
+
}
|
|
138
359
|
}
|
|
139
360
|
|
|
140
361
|
/** 获取当前 SSE 连接统计 */
|
|
@@ -142,8 +363,23 @@ export function getSSEStats(): { total: number; sessions: Record<string, number>
|
|
|
142
363
|
let total = 0;
|
|
143
364
|
const sessions: Record<string, number> = {};
|
|
144
365
|
for (const [name, arr] of clients) {
|
|
145
|
-
|
|
146
|
-
|
|
366
|
+
const alive = arr.filter((c) => !c.closed);
|
|
367
|
+
if (alive.length === 0) continue;
|
|
368
|
+
sessions[name] = alive.length;
|
|
369
|
+
total += alive.length;
|
|
147
370
|
}
|
|
148
371
|
return { total, sessions };
|
|
149
372
|
}
|
|
373
|
+
|
|
374
|
+
// Test hooks — exported so the test file can simulate half-open / stuck
|
|
375
|
+
// consumers without spinning up a real TCP socket.
|
|
376
|
+
export const __SSE_INTERNALS_FOR_TEST = {
|
|
377
|
+
tryEnqueueBytes,
|
|
378
|
+
sweepLiveness,
|
|
379
|
+
closeClient,
|
|
380
|
+
MAX_QUEUE_BACKPRESSURE_BYTES,
|
|
381
|
+
STUCK_CLOSE_MS,
|
|
382
|
+
KEEPALIVE_MS,
|
|
383
|
+
LIVENESS_SWEEP_MS,
|
|
384
|
+
clientsMap: clients,
|
|
385
|
+
};
|
package/src/tools.ts
CHANGED
|
@@ -498,9 +498,12 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
|
|
|
498
498
|
if (updatedTaskId) logTaskEvent(updatedTaskId, null, "replied", alias, "report_completion");
|
|
499
499
|
|
|
500
500
|
// Auto-chain to parent lineage (mirror of send_reply path).
|
|
501
|
+
// round5 F2: pass caller's effectiveNetId so the chain refuses
|
|
502
|
+
// to write across tenants if some upstream parent links to a
|
|
503
|
+
// foreign network.
|
|
501
504
|
if (updatedTaskId) {
|
|
502
505
|
try {
|
|
503
|
-
chainReplyToParent(updatedTaskId, result, "replied");
|
|
506
|
+
chainReplyToParent(updatedTaskId, result, "replied", 5, effectiveNetId);
|
|
504
507
|
const parentChain = db.get<{ parent_task_id: string | null }>(
|
|
505
508
|
"SELECT parent_task_id FROM tasks WHERE task_id = ?1",
|
|
506
509
|
[updatedTaskId]
|
|
@@ -698,17 +701,48 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
|
|
|
698
701
|
// Resolve parent_task_id: explicit > inferred (caller's most recent
|
|
699
702
|
// delivered/started inbox task that's still open). Inference is the
|
|
700
703
|
// safety net for when the LLM forgets to pass parent_task_id.
|
|
704
|
+
//
|
|
705
|
+
// round5 F1 fix: the inference SELECT MUST be network-scoped. Without
|
|
706
|
+
// it, a caller in network B can pick up the parent_task_id of a
|
|
707
|
+
// network A dispatch and chain-reply into the wrong tenant. The
|
|
708
|
+
// explicit-parent path is verified below in F2.
|
|
701
709
|
let parentTaskId: string | null = parentIn ?? null;
|
|
702
710
|
if (!parentTaskId && from_session && from_session !== "hub" && from_session !== "api") {
|
|
703
711
|
try {
|
|
704
|
-
const
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
712
|
+
const recentParams: any[] = [from_session];
|
|
713
|
+
let recentSql = "SELECT task_id FROM tasks WHERE to_name = ?1 AND status IN ('delivered','started')";
|
|
714
|
+
recentSql = addScope(recentSql, recentParams, effectiveNetId);
|
|
715
|
+
recentSql += " ORDER BY created_at DESC LIMIT 1";
|
|
716
|
+
const recent = db.get<{ task_id: string }>(recentSql, ...recentParams);
|
|
708
717
|
if (recent?.task_id) parentTaskId = recent.task_id;
|
|
709
718
|
} catch {}
|
|
710
719
|
}
|
|
711
720
|
|
|
721
|
+
// round5 F2 fix: an explicit parent_task_id must belong to the
|
|
722
|
+
// caller's network. Otherwise a malicious caller in network B can
|
|
723
|
+
// hand us a parent id from network A and have chainReplyToParent
|
|
724
|
+
// (db.ts) write back into A's task result + inbox — cross-tenant
|
|
725
|
+
// write. Verify ownership, reject on mismatch.
|
|
726
|
+
if (parentIn) {
|
|
727
|
+
const parentRow = db.get<{ network_id: string | null }>(
|
|
728
|
+
"SELECT network_id FROM tasks WHERE task_id = ?1",
|
|
729
|
+
[parentIn]
|
|
730
|
+
);
|
|
731
|
+
if (!parentRow) {
|
|
732
|
+
// Parent doesn't exist (LLM hallucination, race with retention
|
|
733
|
+
// sweep, etc.). Drop the link silently so the dispatch can
|
|
734
|
+
// still proceed; the LLM may have meant to dispatch fresh.
|
|
735
|
+
console.log(`[${ts()}] ⚠ send_task: parent_task_id=${parentIn.slice(0, 8)} not found, dropping parent link`);
|
|
736
|
+
parentTaskId = null;
|
|
737
|
+
} else if ((parentRow.network_id ?? null) !== (effectiveNetId ?? null)) {
|
|
738
|
+
console.log(`[${ts()}] 🚫 send_task: cross-network parent rejected, parent=${parentIn.slice(0, 8)} parent-net=${parentRow.network_id ?? "null"} caller-net=${effectiveNetId ?? "null"}`);
|
|
739
|
+
return { content: [{ type: "text" as const, text: JSON.stringify({
|
|
740
|
+
ok: false, error: "cross_network_parent",
|
|
741
|
+
message: "parent_task_id belongs to a different network",
|
|
742
|
+
}) }] };
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
|
|
712
746
|
// Role check: viewer cannot send tasks
|
|
713
747
|
if (!canWrite(effectiveNetId)) {
|
|
714
748
|
return writeDeniedReply(effectiveNetId, "send_task");
|
|
@@ -980,9 +1014,12 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
|
|
|
980
1014
|
|
|
981
1015
|
// Auto-chain reply up to parent task lineage so admin sees the final
|
|
982
1016
|
// answer even if the intermediate session has died.
|
|
1017
|
+
// round5 F2: pass caller's effectiveNetId so the chain refuses
|
|
1018
|
+
// to write across tenants if some upstream parent links to a
|
|
1019
|
+
// foreign network.
|
|
983
1020
|
if (replyLogged && in_reply_to) {
|
|
984
1021
|
try {
|
|
985
|
-
chainReplyToParent(in_reply_to, text, replyStatus);
|
|
1022
|
+
chainReplyToParent(in_reply_to, text, replyStatus, 5, effectiveNetId);
|
|
986
1023
|
// Push SSE event for parent originator if there is a chain.
|
|
987
1024
|
const parentChain = db.get<{ parent_task_id: string | null; from_name: string }>(
|
|
988
1025
|
"SELECT parent_task_id, from_name FROM tasks WHERE task_id = ?1",
|