@preventive/triage 1.0.0-alpha.0 → 1.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/client-sync.js +14 -10
- package/out/graph.js +4 -4
- package/out/index.html +39 -23
- package/out/view.css +1 -1
- package/out/view.js +47 -46
- package/package.json +6 -1
- package/server/auth.ts +11 -0
- package/server/bus-receiver.ts +95 -0
- package/server/db-neon.ts +11 -1
- package/server/db-revision-sql.ts +9 -0
- package/server/db.ts +18 -1
- package/server/http.ts +41 -5
- package/server/hub.ts +22 -2
- package/server/index.ts +124 -15
- package/server/lifecycle.ts +23 -1
- package/server/npm-proxy.ts +348 -0
- package/server/objstore/handlers.ts +12 -0
- package/server/objstore/init.ts +7 -1
- package/server/objstore/rest.ts +18 -0
- package/server/pubsub.ts +404 -0
- package/server/sse-server.ts +352 -0
- package/server/sse-session.ts +202 -0
- package/server/sync-handlers.ts +17 -1
- package/server/ws-server.ts +199 -174
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@preventive/triage",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.1",
|
|
4
4
|
"description": "Client & relay server for triaging of automated reports",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": {
|
|
@@ -33,6 +33,7 @@
|
|
|
33
33
|
},
|
|
34
34
|
"files": [
|
|
35
35
|
"server/auth.ts",
|
|
36
|
+
"server/bus-receiver.ts",
|
|
36
37
|
"server/config.ts",
|
|
37
38
|
"server/db-neon.ts",
|
|
38
39
|
"server/db-revision-sql.ts",
|
|
@@ -43,6 +44,7 @@
|
|
|
43
44
|
"server/index.ts",
|
|
44
45
|
"server/lifecycle.ts",
|
|
45
46
|
"server/neon-driver.ts",
|
|
47
|
+
"server/npm-proxy.ts",
|
|
46
48
|
"server/objstore/blob-fs.ts",
|
|
47
49
|
"server/objstore/blob-vercel.ts",
|
|
48
50
|
"server/objstore/blob.ts",
|
|
@@ -57,7 +59,10 @@
|
|
|
57
59
|
"server/objstore/tokens.ts",
|
|
58
60
|
"server/origin.ts",
|
|
59
61
|
"server/peer.ts",
|
|
62
|
+
"server/pubsub.ts",
|
|
60
63
|
"server/sign.ts",
|
|
64
|
+
"server/sse-server.ts",
|
|
65
|
+
"server/sse-session.ts",
|
|
61
66
|
"server/static.ts",
|
|
62
67
|
"server/sync-handlers.ts",
|
|
63
68
|
"server/util.ts",
|
package/server/auth.ts
CHANGED
|
@@ -76,6 +76,17 @@ export function createAuth(deps: {
|
|
|
76
76
|
}
|
|
77
77
|
function handleAuthenticate(socket: WebSocket, msg: AuthenticateMsg): void {
|
|
78
78
|
if (typeof msg.password !== 'string' || msg.password.length === 0 || msg.password.length > MAX_AUTH_PASSWORD_LEN) return
|
|
79
|
+
// Already-authorized short-circuit. The SSE plane carries
|
|
80
|
+
// `body.password` on every POST (sse-server.ts:dispatchBody
|
|
81
|
+
// synthesises a fresh `authenticate` frame each time), so re-
|
|
82
|
+
// emitting `authenticated` per POST would chatter the wire AND
|
|
83
|
+
// re-fire triage-sync's `authenticated` consumer hook (its "kick
|
|
84
|
+
// deferred sends" path) on every flush. The wire signal is needed
|
|
85
|
+
// exactly once per `unauthorized → authorized` transition; after
|
|
86
|
+
// that, the client already knows. WS plane unaffected (sends
|
|
87
|
+
// authenticate at most once per socket per the runAuthFlow guard
|
|
88
|
+
// in client/sync/socket-transport.ts).
|
|
89
|
+
if (isAuthorized(socket)) return
|
|
79
90
|
// No-config short-circuit: when not gating, treat any authenticate
|
|
80
91
|
// as success so a client can cache + replay its password on
|
|
81
92
|
// reconnect even against an un-gated server (wire shape stays
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
// Cross-instance bus receiver: maps a `BusMessage` envelope (lookup
|
|
2
|
+
// hint + sender from the Postgres LISTEN/NOTIFY channel) into a local
|
|
3
|
+
// wire broadcast. Pulled out of `server/index.ts` so the rev / objput /
|
|
4
|
+
// objdel branches — including the keyframe boolean coercion and the
|
|
5
|
+
// `objectMetaWire` shape — are testable without bringing up the whole
|
|
6
|
+
// server.
|
|
7
|
+
//
|
|
8
|
+
// The receiver mirrors the publisher's commit→broadcast→publish
|
|
9
|
+
// ordering, just shifted by the bus round-trip:
|
|
10
|
+
// 1. NOTIFY lands → bus envelope arrives here.
|
|
11
|
+
// 2. For hint-only kinds (rev, objput) re-fetch the row from the
|
|
12
|
+
// shared DB; for self-contained kinds (objdel) skip the fetch.
|
|
13
|
+
// 3. Broadcast to LOCAL subscribers via `broadcastLocalRaw` with no
|
|
14
|
+
// `except` — the originator is on a DIFFERENT instance by
|
|
15
|
+
// construction, so it must NOT be filtered out (the symmetric
|
|
16
|
+
// same-instance broadcast already happened on the publisher).
|
|
17
|
+
|
|
18
|
+
import type { Handle as DbHandle } from './db.ts'
|
|
19
|
+
import { type Handle as ObjstoreHandle, getLive, objectMetaWire } from './objstore/store.ts'
|
|
20
|
+
import type { BusMessage } from './pubsub.ts'
|
|
21
|
+
|
|
22
|
+
export type BusReceiverDeps = {
|
|
23
|
+
// Read-side of the workspace_revision Handle (the bus uses only
|
|
24
|
+
// `revisionById`; the publisher path uses the full Handle). Typed
|
|
25
|
+
// as the broader Handle for ease of wiring at the call site.
|
|
26
|
+
handle: DbHandle
|
|
27
|
+
// Read-side of the objstore Handle (used for `getLive` on objput).
|
|
28
|
+
objstoreHandle: ObjstoreHandle
|
|
29
|
+
// Local-only fan-out into this instance's subscriber map. The
|
|
30
|
+
// payload is pre-serialised so the hub can stringify once for N
|
|
31
|
+
// subscribers (mirrors the existing `broadcast` fast path).
|
|
32
|
+
broadcastLocalRaw: (tag: string, payload: string) => void
|
|
33
|
+
debug: boolean
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function createBusReceiver(deps: BusReceiverDeps): (msg: BusMessage) => Promise<void> {
|
|
37
|
+
const { handle, objstoreHandle, broadcastLocalRaw, debug } = deps
|
|
38
|
+
return async (msg) => {
|
|
39
|
+
if (msg.kind === 'rev') {
|
|
40
|
+
const row = await handle.revisionById.get(msg.tag, msg.id)
|
|
41
|
+
if (!row) {
|
|
42
|
+
// Not found is expected when a peer instance published a
|
|
43
|
+
// revision id that hasn't replicated to this instance yet —
|
|
44
|
+
// shouldn't happen on a single-primary Neon endpoint, but a
|
|
45
|
+
// future read-replica deployment could see it transiently.
|
|
46
|
+
// Drop the broadcast; the client will reconcile via chain
|
|
47
|
+
// re-pull on its next subscribe.
|
|
48
|
+
if (debug) console.warn(`pubsub: revision ${msg.id.slice(0, 8)}… not found for ${msg.tag.slice(0, 12)}…`)
|
|
49
|
+
return
|
|
50
|
+
}
|
|
51
|
+
broadcastLocalRaw(msg.tag, JSON.stringify({
|
|
52
|
+
type: 'workspace-state',
|
|
53
|
+
workspaceTag: msg.tag,
|
|
54
|
+
// Mirror `chainForWire`'s strict-boolean coercion in
|
|
55
|
+
// ./sync-handlers.ts — the DB stores `keyframe` as an integer
|
|
56
|
+
// (0/1) but the wire contract uses strict `=== true`.
|
|
57
|
+
revisions: [{ ...row, keyframe: row.keyframe === 1 }],
|
|
58
|
+
}))
|
|
59
|
+
return
|
|
60
|
+
}
|
|
61
|
+
if (msg.kind === 'objput') {
|
|
62
|
+
const row = await getLive(objstoreHandle, msg.tag, msg.res)
|
|
63
|
+
if (!row) {
|
|
64
|
+
// Possible after a remote put → remote delete sequence where
|
|
65
|
+
// both NOTIFYs landed by the time we processed the put. The
|
|
66
|
+
// subsequent objdel NOTIFY will (or has) drive the right
|
|
67
|
+
// broadcast — skip silently. Also possible under a future
|
|
68
|
+
// read-replica deployment with replication lag.
|
|
69
|
+
if (debug) console.warn(`pubsub: objput ${msg.res.slice(0, 8)}… missing for ${msg.tag.slice(0, 12)}…`)
|
|
70
|
+
return
|
|
71
|
+
}
|
|
72
|
+
// Note: `getLive` returns the CURRENT live row, which may be a
|
|
73
|
+
// strictly NEWER version than the NOTIFY referred to (two
|
|
74
|
+
// closely-spaced puts on one resourceTag arrive at the receiver
|
|
75
|
+
// with the DB already showing v2 for both). Broadcasting v2 twice
|
|
76
|
+
// is sound — client `putHandlers` already absorb same-instance
|
|
77
|
+
// PUT echoes (rest.ts uses `except: null`), so handlers must be
|
|
78
|
+
// idempotent on (resourceTag, version) anyway.
|
|
79
|
+
broadcastLocalRaw(msg.tag, JSON.stringify({
|
|
80
|
+
type: 'objstore-put',
|
|
81
|
+
workspaceTag: msg.tag,
|
|
82
|
+
...objectMetaWire(row),
|
|
83
|
+
}))
|
|
84
|
+
return
|
|
85
|
+
}
|
|
86
|
+
// 'objdel' — the row is gone from workspace_object post-commit, so
|
|
87
|
+
// the bus payload itself IS the wire data; no DB lookup.
|
|
88
|
+
broadcastLocalRaw(msg.tag, JSON.stringify({
|
|
89
|
+
type: 'objstore-deleted',
|
|
90
|
+
workspaceTag: msg.tag,
|
|
91
|
+
resourceTag: msg.res,
|
|
92
|
+
version: msg.ver,
|
|
93
|
+
}))
|
|
94
|
+
}
|
|
95
|
+
}
|
package/server/db-neon.ts
CHANGED
|
@@ -46,7 +46,8 @@ import type { AllStmt, GetStmt, RunStmt } from './db-stmt.ts'
|
|
|
46
46
|
import { type CommitResult, type Handle, type RevisionInsert, type RevisionRow, isUniqueViolation } from './db.ts'
|
|
47
47
|
import {
|
|
48
48
|
CHAIN_AFTER_SQL, CHAIN_ALL_SQL, CHAIN_FROM_SQL, GATED_INSERT_SQL_PG, HEAD_FOR_SQL,
|
|
49
|
-
LAST_KEYFRAME_SEQ_SQL, REVISION_EXISTS_SQL, SEQ_OF_ID_SQL,
|
|
49
|
+
LAST_KEYFRAME_SEQ_SQL, REVISION_BY_ID_SQL, REVISION_EXISTS_SQL, SEQ_OF_ID_SQL,
|
|
50
|
+
mapRevisionRow, num, numOrNull,
|
|
50
51
|
} from './db-revision-sql.ts'
|
|
51
52
|
|
|
52
53
|
// `num` / `numOrNull` (safe-integer BIGINT coercion) live in the shared
|
|
@@ -218,6 +219,14 @@ function buildRevisionExists(sql: NeonSql): GetStmt<[string, string], unknown> {
|
|
|
218
219
|
return getRowStmt(sql, REVISION_EXISTS_SQL)
|
|
219
220
|
}
|
|
220
221
|
|
|
222
|
+
function buildRevisionById(sql: NeonSql): GetStmt<[string, string], RevisionRow> {
|
|
223
|
+
return { get: async (tag, id) => {
|
|
224
|
+
const rows = await sql(REVISION_BY_ID_SQL, [tag, id]) as Array<Record<string, unknown>>
|
|
225
|
+
const r = rows[0]
|
|
226
|
+
return r ? mapRevisionRow(r) : undefined
|
|
227
|
+
} }
|
|
228
|
+
}
|
|
229
|
+
|
|
221
230
|
// Atomic commit of a single revision (Neon backend). Wraps the
|
|
222
231
|
// dup-check, head-check and gated INSERT in one pipelined
|
|
223
232
|
// transaction — NO commit-time advisory lock. Cross-replica
|
|
@@ -359,6 +368,7 @@ export async function openNeonDb(connectionString: string): Promise<Handle> {
|
|
|
359
368
|
chainAfterSeq: buildChainSeq(sql, CHAIN_AFTER_SQL),
|
|
360
369
|
chainFromSeq: buildChainSeq(sql, CHAIN_FROM_SQL),
|
|
361
370
|
revisionExists: buildRevisionExists(sql),
|
|
371
|
+
revisionById: buildRevisionById(sql),
|
|
362
372
|
tryCommit: tryCommitNeon(sql),
|
|
363
373
|
// The serverless HTTP client is stateless — no socket to close.
|
|
364
374
|
// Async no-op so shutdown's `handle.close()` works uniformly
|
|
@@ -108,6 +108,15 @@ export const CHAIN_FROM_SQL =
|
|
|
108
108
|
FROM workspace_revision WHERE workspace_tag = $1 AND seq >= $2 ORDER BY seq ASC`
|
|
109
109
|
export const REVISION_EXISTS_SQL =
|
|
110
110
|
`SELECT 1 AS one FROM workspace_revision WHERE workspace_tag = $1 AND id = $2`
|
|
111
|
+
// Fetch a single revision row by content-addressed id. The cross-instance
|
|
112
|
+
// pubsub (server/pubsub.ts) NOTIFY payload carries only `(tag, revisionId)`
|
|
113
|
+
// because the full `workspace-state` broadcast envelope is bounded by
|
|
114
|
+
// `MAX_CIPHERTEXT_LEN` (2 MiB) and Postgres NOTIFY caps payloads at ~8 KB.
|
|
115
|
+
// The receiver re-fetches the row from this shared table to construct the
|
|
116
|
+
// wire broadcast for its local peers.
|
|
117
|
+
export const REVISION_BY_ID_SQL =
|
|
118
|
+
`SELECT base, id, keyframe, nonce, ciphertext, signature
|
|
119
|
+
FROM workspace_revision WHERE workspace_tag = $1 AND id = $2`
|
|
111
120
|
|
|
112
121
|
// The gated commit INSERT, in `$N` form. One statement folds the
|
|
113
122
|
// dup-check, the head-equals-base check, the server-assigned seq
|
package/server/db.ts
CHANGED
|
@@ -60,7 +60,8 @@ import { dirname } from 'node:path'
|
|
|
60
60
|
import { type AllStmt, type GetStmt, wrapAll, wrapGet } from './db-stmt.ts'
|
|
61
61
|
import {
|
|
62
62
|
CHAIN_AFTER_SQL, CHAIN_ALL_SQL, CHAIN_FROM_SQL, GATED_INSERT_SQL_SQLITE, HEAD_FOR_SQL,
|
|
63
|
-
LAST_KEYFRAME_SEQ_SQL, REVISION_EXISTS_SQL, SEQ_OF_ID_SQL,
|
|
63
|
+
LAST_KEYFRAME_SEQ_SQL, REVISION_BY_ID_SQL, REVISION_EXISTS_SQL, SEQ_OF_ID_SQL,
|
|
64
|
+
mapRevisionRow, toSqlitePlaceholders,
|
|
64
65
|
} from './db-revision-sql.ts'
|
|
65
66
|
|
|
66
67
|
// `CHECK (keyframe IN (0, 1))` is the value-domain guard on the
|
|
@@ -181,6 +182,10 @@ export type Handle = {
|
|
|
181
182
|
chainAfterSeq: AllStmt<[string, number], RevisionRow>
|
|
182
183
|
chainFromSeq: AllStmt<[string, number], RevisionRow>
|
|
183
184
|
revisionExists: GetStmt<[string, string], unknown>
|
|
185
|
+
// Single-revision fetch by content-addressed id. The cross-instance
|
|
186
|
+
// pubsub receiver uses this to assemble a `workspace-state` from a
|
|
187
|
+
// NOTIFY hint (see `server/pubsub.ts`).
|
|
188
|
+
revisionById: GetStmt<[string, string], RevisionRow>
|
|
184
189
|
gatedInsert?: GetStmt<[string, string, string | null, number, string, string, string, number], { seq: number }>
|
|
185
190
|
tryCommit: (input: RevisionInsert) => Promise<CommitResult>
|
|
186
191
|
close: () => Promise<void>
|
|
@@ -288,6 +293,17 @@ function openDbInner(db: DatabaseSync): SqliteHandle {
|
|
|
288
293
|
const raw = wrapAll<P, Record<string, unknown>>(db.prepare(toSqlitePlaceholders(query)))
|
|
289
294
|
return { all: async (...args: P) => (await raw.all(...args)).map(mapRevisionRow) }
|
|
290
295
|
}
|
|
296
|
+
// Reuses `wrapGet` for the prepared statement, then maps the row
|
|
297
|
+
// through the shared coercion so the returned shape matches `chainFrom`.
|
|
298
|
+
const revisionByIdStmt: GetStmt<[string, string], RevisionRow> = (() => {
|
|
299
|
+
const raw = wrapGet<[string, string], Record<string, unknown>>(
|
|
300
|
+
db.prepare(toSqlitePlaceholders(REVISION_BY_ID_SQL)),
|
|
301
|
+
)
|
|
302
|
+
return { get: async (tag, id) => {
|
|
303
|
+
const row = await raw.get(tag, id)
|
|
304
|
+
return row ? mapRevisionRow(row) : undefined
|
|
305
|
+
} }
|
|
306
|
+
})()
|
|
291
307
|
const handle: SqliteHandle = {
|
|
292
308
|
db,
|
|
293
309
|
headFor: wrapGet<[string], { id: string }>(db.prepare(toSqlitePlaceholders(HEAD_FOR_SQL))),
|
|
@@ -297,6 +313,7 @@ function openDbInner(db: DatabaseSync): SqliteHandle {
|
|
|
297
313
|
chainAfterSeq: chainStmt<[string, number]>(CHAIN_AFTER_SQL),
|
|
298
314
|
chainFromSeq: chainStmt<[string, number]>(CHAIN_FROM_SQL),
|
|
299
315
|
revisionExists: wrapGet<[string, string], unknown>(db.prepare(toSqlitePlaceholders(REVISION_EXISTS_SQL))),
|
|
316
|
+
revisionById: revisionByIdStmt,
|
|
300
317
|
// SQLite null-safe equality is `IS`; the numbered `?N` form (with
|
|
301
318
|
// reuse) maps `$1`/`$2`/`$3` to repeated positional binds. `RETURNING
|
|
302
319
|
// seq` works in node:sqlite (see objstore's `insertLiveIfAbsent`).
|
package/server/http.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// HTTP plane: the REST byte-transfer routing (`/api/objstore/...`), the
|
|
2
|
-
// static UI bundle,
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
2
|
+
// static UI bundle, the SSE+POST fallback transport for the sync
|
|
3
|
+
// protocol, and the WebSocket upgrade gate. Built once at boot with
|
|
4
|
+
// the WS server plus the lifecycle hooks it needs (`track` to drain
|
|
5
|
+
// in-flight requests on shutdown, `isShuttingDown` to gate new ones).
|
|
6
|
+
// The WS *connection* handler is wired on `wss` separately in
|
|
6
7
|
// index.ts; this module only owns the upgrade handshake.
|
|
7
8
|
|
|
8
9
|
import { type IncomingMessage as HttpRequest, type Server, type ServerResponse, createServer } from 'node:http'
|
|
@@ -10,6 +11,8 @@ import { Buffer } from 'node:buffer'
|
|
|
10
11
|
import { fileURLToPath } from 'node:url'
|
|
11
12
|
import type { WebSocketServer } from 'ws'
|
|
12
13
|
import { type ObjstoreRestDeps, handleRest, matchRoute } from './objstore/rest.ts'
|
|
14
|
+
import { SSE_OPEN_PATH, type SseServer } from './sse-server.ts'
|
|
15
|
+
import { dispatchNpmAdvisories } from './npm-proxy.ts'
|
|
13
16
|
import { loadStatic } from './static.ts'
|
|
14
17
|
import { errStack } from './util.ts'
|
|
15
18
|
|
|
@@ -31,6 +34,11 @@ type HasHeaders = { headers: HttpRequest['headers'] }
|
|
|
31
34
|
export type HttpServerDeps = {
|
|
32
35
|
wss: WebSocketServer
|
|
33
36
|
restDeps: ObjstoreRestDeps
|
|
37
|
+
// SSE+POST fallback. Owns its own session map + lifecycle; we just
|
|
38
|
+
// give it first crack at requests that match `SSE_OPEN_PATH`. Same
|
|
39
|
+
// same-origin / shutdown gates run BEFORE the dispatch so the SSE
|
|
40
|
+
// plane inherits them.
|
|
41
|
+
sseServer: SseServer
|
|
34
42
|
isOriginAllowed: (req: HasHeaders) => boolean
|
|
35
43
|
isShuttingDown: () => boolean
|
|
36
44
|
track: (promise: Promise<unknown>) => void
|
|
@@ -39,7 +47,7 @@ export type HttpServerDeps = {
|
|
|
39
47
|
}
|
|
40
48
|
|
|
41
49
|
export function createHttpServer(deps: HttpServerDeps): Server {
|
|
42
|
-
const { wss, restDeps, isOriginAllowed, isShuttingDown, track, restPutIdleTimeoutMs, debug } = deps
|
|
50
|
+
const { wss, restDeps, sseServer, isOriginAllowed, isShuttingDown, track, restPutIdleTimeoutMs, debug } = deps
|
|
43
51
|
// Static-file plane (see ./static.ts). The directory is the
|
|
44
52
|
// `build.js build` output sibling to this file; the loader handles
|
|
45
53
|
// enumeration, pre-compression, and ETag derivation. Plugged in after
|
|
@@ -47,6 +55,34 @@ export function createHttpServer(deps: HttpServerDeps): Server {
|
|
|
47
55
|
const handleStatic = loadStatic(fileURLToPath(new URL('../out', import.meta.url)))
|
|
48
56
|
|
|
49
57
|
const httpServer = createServer((req: HttpRequest, res: ServerResponse) => {
|
|
58
|
+
// SSE+POST fallback plane. Same-origin gate first (Origin header
|
|
59
|
+
// is set by the browser on cross-origin EventSource + fetch, so
|
|
60
|
+
// a hostile origin would surface here just like it does on the
|
|
61
|
+
// WS upgrade and REST plane). The sseServer.handle() function
|
|
62
|
+
// returns true iff it matched and consumed the request; on false
|
|
63
|
+
// we fall through to the REST / static / 404 ladder below.
|
|
64
|
+
if (typeof req.url === 'string' && req.url.split('?', 1)[0] === SSE_OPEN_PATH) {
|
|
65
|
+
if (!isOriginAllowed(req)) {
|
|
66
|
+
res.writeHead(403, { 'content-type': 'application/json' })
|
|
67
|
+
res.end(JSON.stringify({ error: 'origin-denied' }))
|
|
68
|
+
return
|
|
69
|
+
}
|
|
70
|
+
// sseServer.handle owns its own shutdown / cap / 503 ladder.
|
|
71
|
+
// Each POST's response stays open as the session's downstream
|
|
72
|
+
// channel until the next POST takes over — the response is
|
|
73
|
+
// long-lived even though the request is one-shot — and the
|
|
74
|
+
// shutdown gate inside sseServer prevents accepting new POSTs
|
|
75
|
+
// once shuttingDown latches, so outstanding sessions drain via
|
|
76
|
+
// the lifecycle's sseSessions() close loop.
|
|
77
|
+
if (sseServer.handle(req, res)) return
|
|
78
|
+
}
|
|
79
|
+
// npm advisories proxy — same-origin + shutdown gates live in the
|
|
80
|
+
// helper so this dispatcher stays compact. `dispatchNpmAdvisories`
|
|
81
|
+
// returns the in-flight promise (or null when the route didn't
|
|
82
|
+
// match); the lifecycle's `track` awaits it so SIGTERM drains
|
|
83
|
+
// outstanding upstream fetches.
|
|
84
|
+
const npmP = dispatchNpmAdvisories(req, res, { isOriginAllowed, isShuttingDown, debug })
|
|
85
|
+
if (npmP) { track(npmP); return }
|
|
50
86
|
if (matchRoute(req.url) != null) {
|
|
51
87
|
// Shutdown gate. The WS plane gates new messages on `shuttingDown`;
|
|
52
88
|
// REST handlers go through a separate path and must mirror it.
|
package/server/hub.ts
CHANGED
|
@@ -19,8 +19,18 @@ export type Hub = {
|
|
|
19
19
|
// `except: null` is the REST-originated path — byte transfer landed
|
|
20
20
|
// via HTTP, not a particular WS socket, so it hits every subscriber.
|
|
21
21
|
// WS-originated broadcasts pass the originator so it doesn't see its
|
|
22
|
-
// own message echoed back.
|
|
22
|
+
// own message echoed back. Local-only: callers that ALSO want a
|
|
23
|
+
// cross-instance fan-out are expected to publish to the pubsub bus
|
|
24
|
+
// alongside this call (server/pubsub.ts). The hub deliberately stays
|
|
25
|
+
// ignorant of the bus so its transport invariants (backpressure cap,
|
|
26
|
+
// stringify-once, terminate-on-overflow) are unchanged.
|
|
23
27
|
broadcast(tag: string, msg: object, except: WebSocket | null): void
|
|
28
|
+
// Broadcasts an ALREADY-SERIALISED payload to every local subscriber
|
|
29
|
+
// for `tag`. Used by the pubsub bus receiver
|
|
30
|
+
// (server/bus-receiver.ts, wired up from server/index.ts) to relay
|
|
31
|
+
// a remote-instance event into this instance's fan-out. No `except`:
|
|
32
|
+
// the originator is on a different instance by construction.
|
|
33
|
+
broadcastLocalRaw(tag: string, payload: string): void
|
|
24
34
|
}
|
|
25
35
|
|
|
26
36
|
export function createHub(deps: { peers: PeerRegistry; maxBufferedBytes: number; debug: boolean }): Hub {
|
|
@@ -82,6 +92,16 @@ export function createHub(deps: { peers: PeerRegistry; maxBufferedBytes: number;
|
|
|
82
92
|
// catch-up with a multi-MB ciphertext × N subscribers, per-recipient
|
|
83
93
|
// JSON.stringify would dominate CPU; this is the cheap win.
|
|
84
94
|
const payload = JSON.stringify(msg)
|
|
95
|
+
fanOut(set, payload, except)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function broadcastLocalRaw(tag: string, payload: string): void {
|
|
99
|
+
const set = subscribers.get(tag)
|
|
100
|
+
if (!set) return
|
|
101
|
+
fanOut(set, payload, null)
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function fanOut(set: Set<WebSocket>, payload: string, except: WebSocket | null): void {
|
|
85
105
|
// Snapshot before iterating — `send`'s try/catch swallows
|
|
86
106
|
// socket.send errors, but a socket transitioning to CLOSED
|
|
87
107
|
// mid-broadcast triggers `unsubscribeAll` from the 'close' handler,
|
|
@@ -94,5 +114,5 @@ export function createHub(deps: { peers: PeerRegistry; maxBufferedBytes: number;
|
|
|
94
114
|
}
|
|
95
115
|
}
|
|
96
116
|
|
|
97
|
-
return { subscribe, unsubscribeAll, send, sendRaw, broadcast }
|
|
117
|
+
return { subscribe, unsubscribeAll, send, sendRaw, broadcast, broadcastLocalRaw }
|
|
98
118
|
}
|
package/server/index.ts
CHANGED
|
@@ -73,7 +73,7 @@
|
|
|
73
73
|
// and would not reject the attaching socket.
|
|
74
74
|
|
|
75
75
|
import { type WebSocket, WebSocketServer } from 'ws'
|
|
76
|
-
import { errMsg } from './util.ts'
|
|
76
|
+
import { errMsg, errStack } from './util.ts'
|
|
77
77
|
import type { PeerRegistry } from './peer.ts'
|
|
78
78
|
import { LOOPBACK_HOSTS, createOriginGate } from './origin.ts'
|
|
79
79
|
import { createHub } from './hub.ts'
|
|
@@ -81,6 +81,7 @@ import { createAuth } from './auth.ts'
|
|
|
81
81
|
import { createSyncHandlers } from './sync-handlers.ts'
|
|
82
82
|
import { WS_UPGRADE_PATH, createHttpServer } from './http.ts'
|
|
83
83
|
import { installWsServer } from './ws-server.ts'
|
|
84
|
+
import { SSE_OPEN_PATH, installSseServer } from './sse-server.ts'
|
|
84
85
|
import { createLifecycle } from './lifecycle.ts'
|
|
85
86
|
import { loadConfig } from './config.ts'
|
|
86
87
|
import { type Handle, openDb } from './db.ts'
|
|
@@ -89,6 +90,11 @@ import { initObjstore } from './objstore/init.ts'
|
|
|
89
90
|
import { type Handle as ObjstoreHandle, listLive, objectMetaWire, openObjstore } from './objstore/store.ts'
|
|
90
91
|
import { openNeonObjstore } from './objstore/store-neon.ts'
|
|
91
92
|
import { openVercelBlobBackend } from './objstore/blob-vercel.ts'
|
|
93
|
+
import {
|
|
94
|
+
type NeonClientCtor, type PubSub,
|
|
95
|
+
createNeonPubSub, createNoopPubSub,
|
|
96
|
+
} from './pubsub.ts'
|
|
97
|
+
import { createBusReceiver } from './bus-receiver.ts'
|
|
92
98
|
|
|
93
99
|
// All external inputs (env vars + optional config.json) are parsed
|
|
94
100
|
// and validated in ./config.ts. Destructure into the existing
|
|
@@ -235,7 +241,39 @@ async function workspaceExists(tag: string): Promise<boolean> {
|
|
|
235
241
|
// broadcast (see ./hub.ts). Destructure into the existing names so the
|
|
236
242
|
// handlers / dispatcher / objstore wiring below read unchanged.
|
|
237
243
|
const hub = createHub({ peers, maxBufferedBytes: MAX_BUFFERED_BYTES, debug: DEBUG })
|
|
238
|
-
const { send, broadcast, subscribe, unsubscribeAll } = hub
|
|
244
|
+
const { send, broadcast, subscribe, unsubscribeAll, broadcastLocalRaw } = hub
|
|
245
|
+
|
|
246
|
+
// Cross-instance pub/sub for live broadcasts (see ./pubsub.ts). SQLite
|
|
247
|
+
// mode is single-process by construction so it gets a no-op; Neon mode
|
|
248
|
+
// uses Postgres LISTEN/NOTIFY on a dedicated long-lived Client
|
|
249
|
+
// connection (separate from the HTTP `neon()` callable the queries flow
|
|
250
|
+
// over — LISTEN needs a session-bound socket, the HTTP path is
|
|
251
|
+
// stateless). The bus carries lookup hints, not the full wire payload:
|
|
252
|
+
// the `workspace-state` broadcast's ciphertext alone can exceed the
|
|
253
|
+
// ~8 KB NOTIFY payload cap, so the receiver re-fetches from the shared
|
|
254
|
+
// DB to construct the wire frame. The bus is best-effort fan-out, not a
|
|
255
|
+
// durability layer — the DB itself is the source of truth, and a
|
|
256
|
+
// dropped publish only means peers on other instances miss the live
|
|
257
|
+
// push (they still catch up via the chain on their next subscribe).
|
|
258
|
+
let pubsub: PubSub = createNoopPubSub()
|
|
259
|
+
if (NEON_URL) {
|
|
260
|
+
// Dynamic import: the Client export lives in the same optional peer
|
|
261
|
+
// dep as the HTTP `neon()` callable (see ./neon-driver.ts), but the
|
|
262
|
+
// dep itself is only present on a Neon-mode deploy. The wrapper path
|
|
263
|
+
// also lets tests swap in a PGlite-backed shim (`tests/pubsub.test.js`).
|
|
264
|
+
const mod = (await import('./neon-driver.ts')) as unknown as { Client?: NeonClientCtor }
|
|
265
|
+
if (!mod.Client) {
|
|
266
|
+
console.error('DATABASE_URL is set but the @neondatabase/serverless Client export is not available.')
|
|
267
|
+
console.error('Cross-instance broadcasts require the WebSocket-based Client (the HTTP `neon()` callable cannot LISTEN).')
|
|
268
|
+
console.error('Reinstall the peer dep: pnpm add @neondatabase/serverless')
|
|
269
|
+
process.exit(1)
|
|
270
|
+
}
|
|
271
|
+
const NeonClientImpl = mod.Client
|
|
272
|
+
pubsub = createNeonPubSub({
|
|
273
|
+
newClient: () => new NeonClientImpl(NEON_URL),
|
|
274
|
+
debug: DEBUG,
|
|
275
|
+
})
|
|
276
|
+
}
|
|
239
277
|
|
|
240
278
|
// Password gate (see ./auth.ts) — HMAC derivation + the `authenticate`
|
|
241
279
|
// handshake. Destructure into the existing names for the wiring below.
|
|
@@ -247,8 +285,18 @@ const { requiresAuth, handleAuthenticate, sendUnauthorized } = auth
|
|
|
247
285
|
// wiring below; `sendSaveError` is reused by the dispatcher's `busy`
|
|
248
286
|
// inflight-cap NACK path.
|
|
249
287
|
const getNonce = (socket: WebSocket): string | undefined => peers.get(socket)?.challenge
|
|
288
|
+
const publishRevision = (tag: string, revisionId: string): void => {
|
|
289
|
+
pubsub.publish({ kind: 'rev', tag, id: revisionId })
|
|
290
|
+
}
|
|
291
|
+
const publishObjPut = (tag: string, resourceTag: string): void => {
|
|
292
|
+
pubsub.publish({ kind: 'objput', tag, res: resourceTag })
|
|
293
|
+
}
|
|
294
|
+
const publishObjDeleted = (tag: string, resourceTag: string, version: number): void => {
|
|
295
|
+
pubsub.publish({ kind: 'objdel', tag, res: resourceTag, ver: version })
|
|
296
|
+
}
|
|
297
|
+
|
|
250
298
|
const { handleSave, handleSubscribe, sendSaveError } = createSyncHandlers({
|
|
251
|
-
handle, send, broadcast, subscribe, getNonce,
|
|
299
|
+
handle, send, broadcast, publishRevision, subscribe, getNonce,
|
|
252
300
|
requiresAuth, sendUnauthorized, workspaceExists,
|
|
253
301
|
// Folds the objstore inventory into the `workspace-subscribed` ack.
|
|
254
302
|
// The objstore store keeps its own richer `Handle`, so we wire the
|
|
@@ -260,7 +308,8 @@ const { handleSave, handleSubscribe, sendSaveError } = createSyncHandlers({
|
|
|
260
308
|
|
|
261
309
|
const { handlers: objstore, restDeps: objstoreRestDeps, startupReap, stopReaper } = initObjstore({
|
|
262
310
|
handle: objstoreHandle, reapIntervalMs: OBJSTORE_REAP_INTERVAL_MS,
|
|
263
|
-
send, broadcast,
|
|
311
|
+
send, broadcast, publishObjPut, publishObjDeleted,
|
|
312
|
+
getNonce, debug: DEBUG,
|
|
264
313
|
// Auth gate for the FIRST objstore-put-begin against a workspace
|
|
265
314
|
// that doesn't yet exist on the server. Mirrors handleSave's gate
|
|
266
315
|
// below; handlers.ts calls this AFTER sig verify so the
|
|
@@ -289,12 +338,46 @@ const wss = new WebSocketServer({ noServer: true, maxPayload: 4 * 1024 * 1024 })
|
|
|
289
338
|
// every server object exists.
|
|
290
339
|
const { track, isShuttingDown, install: installLifecycle } = createLifecycle()
|
|
291
340
|
|
|
292
|
-
//
|
|
293
|
-
//
|
|
294
|
-
//
|
|
295
|
-
//
|
|
341
|
+
// Shared per-connection dispatch surface. Both the WS plane
|
|
342
|
+
// (installWsServer below) and the SSE+POST fallback (installSseServer
|
|
343
|
+
// below) drive `setupPeerConnection` with this; one Peer per accepted
|
|
344
|
+
// connection, one message-handler tree.
|
|
345
|
+
const peerConnectionDeps = {
|
|
346
|
+
peers, send, unsubscribeAll,
|
|
347
|
+
handleSave, handleSubscribe, handleAuthenticate, sendSaveError, objstore,
|
|
348
|
+
track, isShuttingDown,
|
|
349
|
+
maxInflightPerSocket: MAX_INFLIGHT_PER_SOCKET,
|
|
350
|
+
debug: DEBUG,
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// SSE+POST fallback transport (see ./sse-server.ts). The WS upgrade is
|
|
354
|
+
// the preferred path; clients fall back to SSE when `new WebSocket(…)`
|
|
355
|
+
// fails to open (corporate proxies that strip the Upgrade header,
|
|
356
|
+
// hostile network middleboxes, etc.). Same protocol on the wire — only
|
|
357
|
+
// the byte transport differs.
|
|
358
|
+
const sseServer = installSseServer({
|
|
359
|
+
peerDeps: peerConnectionDeps, isShuttingDown,
|
|
360
|
+
// Same cap shape as `wss` (no explicit per-process cap; OS FD limits
|
|
361
|
+
// are the upstream bound). 1024 leaves room above the typical web
|
|
362
|
+
// session count without being a soft DoS hammer.
|
|
363
|
+
maxSessions: 1024,
|
|
364
|
+
// Mirror the WS `maxPayload` so the SSE plane can't accept frames
|
|
365
|
+
// the WS plane would reject.
|
|
366
|
+
maxBodyBytes: 4 * 1024 * 1024,
|
|
367
|
+
// 90s idle ceiling. The client's JSON ping/pong is 15s; this is ~6
|
|
368
|
+
// missed pings before we close — well past any transient network
|
|
369
|
+
// hiccup, much shorter than the kernel's hours-long TCP keepalive.
|
|
370
|
+
sessionIdleMs: 90_000,
|
|
371
|
+
debug: DEBUG,
|
|
372
|
+
})
|
|
373
|
+
|
|
374
|
+
// HTTP plane: REST byte-transfer routing + SSE fallback + the WS
|
|
375
|
+
// upgrade gate (see ./http.ts). Built after the lifecycle state above
|
|
376
|
+
// because the REST shutdown gate reads `shuttingDown` and the request
|
|
377
|
+
// drain uses `track`. The WS connection handler is wired on `wss`
|
|
378
|
+
// below.
|
|
296
379
|
const httpServer = createHttpServer({
|
|
297
|
-
wss, restDeps: objstoreRestDeps, isOriginAllowed,
|
|
380
|
+
wss, restDeps: objstoreRestDeps, sseServer, isOriginAllowed,
|
|
298
381
|
isShuttingDown, track,
|
|
299
382
|
restPutIdleTimeoutMs: REST_PUT_IDLE_TIMEOUT_MS, debug: DEBUG,
|
|
300
383
|
})
|
|
@@ -303,11 +386,8 @@ const httpServer = createHttpServer({
|
|
|
303
386
|
// sweep (see ./ws-server.ts). Returns the heartbeat timer so shutdown
|
|
304
387
|
// can clear it.
|
|
305
388
|
const { heartbeatTimer } = installWsServer({
|
|
306
|
-
wss,
|
|
307
|
-
|
|
308
|
-
track, isShuttingDown,
|
|
309
|
-
maxInflightPerSocket: MAX_INFLIGHT_PER_SOCKET, heartbeatIntervalMs: HEARTBEAT_INTERVAL_MS,
|
|
310
|
-
debug: DEBUG,
|
|
389
|
+
wss, heartbeatIntervalMs: HEARTBEAT_INTERVAL_MS,
|
|
390
|
+
...peerConnectionDeps,
|
|
311
391
|
})
|
|
312
392
|
|
|
313
393
|
httpServer.on('listening', () => {
|
|
@@ -323,12 +403,40 @@ httpServer.on('listening', () => {
|
|
|
323
403
|
// doesn't claim a misleading DB_PATH under Neon, or a misleading
|
|
324
404
|
// OBJSTORE_DIR under Vercel Blob.
|
|
325
405
|
const dbBanner = NEON_URL ? 'db: neon-postgres' : `db: ${DB_PATH}`
|
|
326
|
-
console.log(`DeepView triage-sync server: ws://${HOST}:${boundPort}${WS_UPGRADE_PATH} http://${HOST}:${boundPort}/api/objstore/{workspaceTag}/{resourceTag} (${dbBanner}, ${objstoreBanner})`)
|
|
406
|
+
console.log(`DeepView triage-sync server: ws://${HOST}:${boundPort}${WS_UPGRADE_PATH} (sse fallback http://${HOST}:${boundPort}${SSE_OPEN_PATH}) http://${HOST}:${boundPort}/api/objstore/{workspaceTag}/{resourceTag} (${dbBanner}, ${objstoreBanner})`)
|
|
407
|
+
})
|
|
408
|
+
|
|
409
|
+
// Cross-instance bus receiver (see ./bus-receiver.ts): a remote NOTIFY
|
|
410
|
+
// lands here, we re-fetch any data the bus payload only hinted at,
|
|
411
|
+
// then broadcast to local peers via `broadcastLocalRaw`. Extracted so
|
|
412
|
+
// the rev / objput / objdel mapping (including the keyframe boolean
|
|
413
|
+
// coercion and the `objectMetaWire` shape) is testable without the
|
|
414
|
+
// full server bring-up.
|
|
415
|
+
const onBusMessage = createBusReceiver({
|
|
416
|
+
handle, objstoreHandle, broadcastLocalRaw, debug: DEBUG,
|
|
417
|
+
})
|
|
418
|
+
|
|
419
|
+
// Kick off the LISTEN loop in the background. `pubsub.start` resolves
|
|
420
|
+
// only after the FIRST successful connect, and the internal reconnect
|
|
421
|
+
// loop retries indefinitely on transport / LISTEN failure — awaiting
|
|
422
|
+
// it here would convert a transient Neon WS hiccup at boot into a
|
|
423
|
+
// server that never binds (and a `.catch` that never runs). Instead
|
|
424
|
+
// the server binds immediately and the bus comes up async; publishes
|
|
425
|
+
// during the down window drop silently (best-effort semantics
|
|
426
|
+
// documented in pubsub.ts). The SQLite no-op resolves immediately,
|
|
427
|
+
// so the behaviour is identical in that mode.
|
|
428
|
+
void pubsub.start(onBusMessage).catch((err) => {
|
|
429
|
+
console.warn('pubsub: startup error:', errStack(err))
|
|
327
430
|
})
|
|
328
431
|
|
|
329
432
|
// App-specific shutdown step (run after the in-flight drain), wired
|
|
330
433
|
// into the lifecycle teardown below.
|
|
331
434
|
const closeDb = async (): Promise<void> => {
|
|
435
|
+
// Stop the bus first so a publish from a still-draining handler
|
|
436
|
+
// can't fire into a half-closed Client. The in-flight drain runs
|
|
437
|
+
// before this (see lifecycle.ts), so by here all `broadcast` →
|
|
438
|
+
// `publish*` calls are settled.
|
|
439
|
+
try { await pubsub.stop() } catch (err) { console.warn('pubsub close error:', errMsg(err)) }
|
|
332
440
|
// objstoreHandle has no close(): SQLite shares this DatabaseSync and
|
|
333
441
|
// Neon has no persistent connection; `handle.close()` covers both.
|
|
334
442
|
try { await handle.close() } catch (err) { console.warn('DB close error:', errMsg(err)) }
|
|
@@ -338,6 +446,7 @@ const closeDb = async (): Promise<void> => {
|
|
|
338
446
|
// and the heartbeat timer all exist.
|
|
339
447
|
installLifecycle({
|
|
340
448
|
httpServer, wss, heartbeatTimer, stopReaper,
|
|
449
|
+
sseSessions: sseServer.sessions,
|
|
341
450
|
closeDb,
|
|
342
451
|
})
|
|
343
452
|
|
package/server/lifecycle.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import type { Server } from 'node:http'
|
|
9
9
|
import type { WebSocketServer } from 'ws'
|
|
10
|
+
import type { SseSession } from './sse-session.ts'
|
|
10
11
|
import { errStack } from './util.ts'
|
|
11
12
|
|
|
12
13
|
export type ShutdownDeps = {
|
|
@@ -15,6 +16,11 @@ export type ShutdownDeps = {
|
|
|
15
16
|
heartbeatTimer: ReturnType<typeof setInterval>
|
|
16
17
|
// Stops the periodic reaper AND awaits any in-flight sweep.
|
|
17
18
|
stopReaper: () => Promise<void>
|
|
19
|
+
// Live SSE+POST session iterator (mirrors `wss.clients` for the SSE
|
|
20
|
+
// fallback transport). Walked alongside `wss.clients` to send the
|
|
21
|
+
// 1001 close frame and apply the terminate-grace timer to both
|
|
22
|
+
// transports uniformly.
|
|
23
|
+
sseSessions: () => Iterable<SseSession>
|
|
18
24
|
// App-specific teardown, run AFTER the in-flight drain: close the DB.
|
|
19
25
|
// Swallows its own errors (a failed close shouldn't abort the exit).
|
|
20
26
|
closeDb: () => Promise<void>
|
|
@@ -51,7 +57,7 @@ export function createLifecycle(): Lifecycle {
|
|
|
51
57
|
let pendingExitCode = 0
|
|
52
58
|
|
|
53
59
|
function install(deps: ShutdownDeps): void {
|
|
54
|
-
const { httpServer, wss, heartbeatTimer, stopReaper, closeDb } = deps
|
|
60
|
+
const { httpServer, wss, heartbeatTimer, stopReaper, sseSessions, closeDb } = deps
|
|
55
61
|
|
|
56
62
|
async function shutdown(exitCode: number = 0): Promise<void> {
|
|
57
63
|
// Re-entry: don't restart the teardown, but escalate the pending
|
|
@@ -78,6 +84,13 @@ export function createLifecycle(): Lifecycle {
|
|
|
78
84
|
for (const socket of wss.clients) {
|
|
79
85
|
try { socket.close(1001, 'Server shutting down') } catch {}
|
|
80
86
|
}
|
|
87
|
+
// Same close frame on every SSE+POST session — the adapter
|
|
88
|
+
// translates `close(1001, …)` into a structured `event: close`
|
|
89
|
+
// record on the SSE stream so the client transport can read the
|
|
90
|
+
// code and skip its reconnect backoff (parity with the WS path).
|
|
91
|
+
for (const session of sseSessions()) {
|
|
92
|
+
try { session.close(1001, 'Server shutting down') } catch {}
|
|
93
|
+
}
|
|
81
94
|
// Force-terminate any client that doesn't ack the close frame
|
|
82
95
|
// within a short grace window. `wss.close()` waits for every client
|
|
83
96
|
// to emit `'close'`, and `ws` only TCP-RSTs unresponsive peers
|
|
@@ -92,6 +105,15 @@ export function createLifecycle(): Lifecycle {
|
|
|
92
105
|
try { socket.terminate() } catch {}
|
|
93
106
|
}
|
|
94
107
|
}
|
|
108
|
+
// SSE sessions are tracked separately from `wss.clients`; apply
|
|
109
|
+
// the same terminate-grace policy so a stranded SSE response
|
|
110
|
+
// can't pin the HTTP keep-alive shutdown branch below.
|
|
111
|
+
for (const session of sseSessions()) {
|
|
112
|
+
const rs = session.readyState
|
|
113
|
+
if (rs === session.OPEN || rs === session.CLOSING) {
|
|
114
|
+
try { session.terminate() } catch {}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
95
117
|
// Same grace for HTTP keep-alive sockets that didn't respect the
|
|
96
118
|
// `Connection: close` hint, which would otherwise hold
|
|
97
119
|
// `httpServer.close()` until their TCP timeout.
|