@preventive/triage 1.0.0-alpha.0 → 1.0.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api/reap.ts +79 -0
- package/common/save-error-reason.ts +20 -7
- package/common/server-info.ts +30 -0
- package/out/brotli-fallback.js +1 -1
- package/out/client-admin.js +28 -0
- package/out/client-managed.js +1 -0
- package/out/client-sync.js +17 -10
- package/out/graph.js +5 -4
- package/out/index.html +43 -38
- package/out/prism.js +2 -2
- package/out/terminal.js +32 -28
- package/out/view.css +1 -1
- package/out/view.js +78 -51
- package/package.json +70 -49
- package/{server → server-common}/origin.ts +5 -5
- package/{server → server-e2e}/auth.ts +16 -1
- package/server-e2e/bus-receiver.ts +95 -0
- package/server-e2e/cli.js +22 -0
- package/{server → server-e2e}/config.ts +21 -8
- package/{server → server-e2e}/db-neon.ts +41 -25
- package/{server → server-e2e}/db-revision-sql.ts +15 -9
- package/{server → server-e2e}/db-stmt.ts +2 -2
- package/{server → server-e2e}/db.ts +113 -135
- package/server-e2e/http.ts +266 -0
- package/{server → server-e2e}/hub.ts +27 -8
- package/{server → server-e2e}/index.ts +185 -52
- package/{server → server-e2e}/lifecycle.ts +36 -5
- package/server-e2e/npm-proxy.ts +348 -0
- package/{server → server-e2e}/objstore/blob-fs.ts +6 -8
- package/{server → server-e2e}/objstore/blob-vercel.ts +69 -36
- package/{server → server-e2e}/objstore/blob.ts +24 -9
- package/server-e2e/objstore/fetch-mint-guard.ts +74 -0
- package/{server → server-e2e}/objstore/handlers.ts +25 -15
- package/{server → server-e2e}/objstore/init.ts +52 -12
- package/{server → server-e2e}/objstore/reaper.ts +31 -11
- package/server-e2e/objstore/rest-deny.ts +28 -0
- package/server-e2e/objstore/rest-mint.ts +224 -0
- package/{server → server-e2e}/objstore/rest.ts +119 -84
- package/{server → server-e2e}/objstore/sign.ts +105 -0
- package/{server → server-e2e}/objstore/store-neon.ts +19 -19
- package/{server → server-e2e}/objstore/store.ts +98 -118
- package/{server → server-e2e}/objstore/tokens.ts +9 -12
- package/{server → server-e2e}/peer.ts +7 -9
- package/server-e2e/pubsub.ts +394 -0
- package/{server → server-e2e}/sign.ts +12 -14
- package/server-e2e/sse-server.ts +384 -0
- package/server-e2e/sse-session.ts +216 -0
- package/{server → server-e2e}/static.ts +22 -17
- package/server-e2e/sync-handlers.ts +382 -0
- package/{server → server-e2e}/util.ts +9 -0
- package/server-e2e/ws-server.ts +276 -0
- package/strip-types-loader.js +94 -0
- package/server/http.ts +0 -142
- package/server/sync-handlers.ts +0 -311
- package/server/ws-server.ts +0 -245
- /package/{server → server-e2e}/config.example.json +0 -0
- /package/{server → server-e2e}/neon-driver.ts +0 -0
- /package/{server → server-e2e}/objstore/fs.ts +0 -0
- /package/{server → server-e2e}/validation.ts +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// DeepView triage-sync relay server. WebSocket front-end, SQLite
|
|
3
3
|
// backing store. Implements the protocol described in
|
|
4
|
-
// `client/triage-sync.js` (and `server/sign.ts` for the canonical
|
|
4
|
+
// `client/triage-sync.js` (and `server-e2e/sign.ts` for the canonical
|
|
5
5
|
// signature payloads):
|
|
6
6
|
//
|
|
7
7
|
// server → client challenge { nonce } — emitted on every
|
|
@@ -73,14 +73,16 @@
|
|
|
73
73
|
// and would not reject the attaching socket.
|
|
74
74
|
|
|
75
75
|
import { type WebSocket, WebSocketServer } from 'ws'
|
|
76
|
-
import { errMsg } from './util.ts'
|
|
76
|
+
import { errMsg, errStack } from './util.ts'
|
|
77
77
|
import type { PeerRegistry } from './peer.ts'
|
|
78
|
-
import { LOOPBACK_HOSTS, createOriginGate } from '
|
|
78
|
+
import { LOOPBACK_HOSTS, createOriginGate } from '../server-common/origin.ts'
|
|
79
79
|
import { createHub } from './hub.ts'
|
|
80
80
|
import { createAuth } from './auth.ts'
|
|
81
81
|
import { createSyncHandlers } from './sync-handlers.ts'
|
|
82
82
|
import { WS_UPGRADE_PATH, createHttpServer } from './http.ts'
|
|
83
83
|
import { installWsServer } from './ws-server.ts'
|
|
84
|
+
import { SSE_OPEN_PATH, installSseServer } from './sse-server.ts'
|
|
85
|
+
import type { ServerInfo } from '../common/server-info.ts'
|
|
84
86
|
import { createLifecycle } from './lifecycle.ts'
|
|
85
87
|
import { loadConfig } from './config.ts'
|
|
86
88
|
import { type Handle, openDb } from './db.ts'
|
|
@@ -89,22 +91,27 @@ import { initObjstore } from './objstore/init.ts'
|
|
|
89
91
|
import { type Handle as ObjstoreHandle, listLive, objectMetaWire, openObjstore } from './objstore/store.ts'
|
|
90
92
|
import { openNeonObjstore } from './objstore/store-neon.ts'
|
|
91
93
|
import { openVercelBlobBackend } from './objstore/blob-vercel.ts'
|
|
94
|
+
import {
|
|
95
|
+
type NeonClientCtor, type PubSub,
|
|
96
|
+
createNeonPubSub, createNoopPubSub,
|
|
97
|
+
} from './pubsub.ts'
|
|
98
|
+
import { createBusReceiver } from './bus-receiver.ts'
|
|
92
99
|
|
|
93
100
|
// All external inputs (env vars + optional config.json) are parsed
|
|
94
|
-
// and validated in ./config.ts
|
|
95
|
-
//
|
|
101
|
+
// and validated in ./config.ts; destructure into the uppercase names
|
|
102
|
+
// the rest of this module uses.
|
|
96
103
|
const config = loadConfig()
|
|
97
104
|
const {
|
|
98
105
|
port: PORT, host: HOST, dbPath: DB_PATH, objstoreDir: OBJSTORE_DIR,
|
|
99
|
-
reapIntervalMs: OBJSTORE_REAP_INTERVAL_MS,
|
|
106
|
+
reapIntervalMs: OBJSTORE_REAP_INTERVAL_MS, reapDisabled: OBJSTORE_REAP_DISABLED,
|
|
100
107
|
maxInflightPerSocket: MAX_INFLIGHT_PER_SOCKET, debug: DEBUG,
|
|
101
108
|
neonUrl: NEON_URL, blobToken: BLOB_TOKEN, tokenSecret: TOKEN_SECRET,
|
|
102
109
|
password: CONFIG_PASSWORD, trustProxyEnv: TRUST_PROXY_ENV,
|
|
103
110
|
} = config
|
|
104
111
|
|
|
105
112
|
// Same-origin gate for the WS upgrade and REST data plane (see
|
|
106
|
-
// ./origin.ts). `TRUST_PROXY_ENV`
|
|
107
|
-
//
|
|
113
|
+
// ./origin.ts). `TRUST_PROXY_ENV` also feeds the boot-time
|
|
114
|
+
// misconfiguration fail-fast below.
|
|
108
115
|
const { trustProxy: TRUST_PROXY, isOriginAllowed } = createOriginGate(HOST, TRUST_PROXY_ENV)
|
|
109
116
|
|
|
110
117
|
// Per-socket buffered-bytes cap. `socket.send` returns synchronously
|
|
@@ -114,28 +121,27 @@ const { trustProxy: TRUST_PROXY, isOriginAllowed } = createOriginGate(HOST, TRUS
|
|
|
114
121
|
// MB of fan-out broadcasts in this buffer with no backpressure on
|
|
115
122
|
// the broadcast loop. Drop the message when the buffer crosses the
|
|
116
123
|
// cap; the heartbeat will eventually close a peer that never
|
|
117
|
-
// drains. Transport audit `server/index.ts:225`.
|
|
124
|
+
// drains. Transport audit `server-e2e/index.ts:225`.
|
|
118
125
|
const MAX_BUFFERED_BYTES = 16 * 1024 * 1024
|
|
119
126
|
// Per-socket in-flight async-handler cap (MAX_INFLIGHT_PER_SOCKET,
|
|
120
127
|
// env-validated in config). Each inbound text frame spawns a
|
|
121
128
|
// `track(handler)` IIFE; an authorised peer firing valid frames could
|
|
122
129
|
// otherwise grow the set without bound, stretching SIGTERM drain time.
|
|
123
130
|
// Saves dropped at the cap surface as a typed `busy` NACK. Transport
|
|
124
|
-
// audit `server/index.ts:590`.
|
|
131
|
+
// audit `server-e2e/index.ts:590`.
|
|
125
132
|
|
|
126
133
|
// Per-connection state registry. One `Peer` per accepted socket holds
|
|
127
134
|
// the challenge nonce, auth flag, heartbeat liveness, in-flight count,
|
|
128
|
-
// and subscribed tags (see ./peer.ts)
|
|
129
|
-
//
|
|
130
|
-
//
|
|
131
|
-
// via `peers.get(socket)`.
|
|
135
|
+
// and subscribed tags (see ./peer.ts). The connection handler holds
|
|
136
|
+
// the Peer in a closure for the hot paths; cross-function call sites
|
|
137
|
+
// resolve it via `peers.get(socket)`.
|
|
132
138
|
const peers: PeerRegistry = new WeakMap()
|
|
133
139
|
|
|
134
140
|
// REST PUT idle-body timeout. A slow-loris client trickling bytes
|
|
135
141
|
// within the declared Content-Length holds the staging fd and an
|
|
136
142
|
// inFlightSids slot until the global staging TTL reaps it. Aborting
|
|
137
143
|
// the per-chunk-idle period closes that window. Transport audit
|
|
138
|
-
// `server/objstore/rest.ts:218`.
|
|
144
|
+
// `server-e2e/objstore/rest.ts:218`.
|
|
139
145
|
const REST_PUT_IDLE_TIMEOUT_MS = 30_000
|
|
140
146
|
|
|
141
147
|
// Server-driven WS heartbeat. Every `HEARTBEAT_INTERVAL_MS` we walk
|
|
@@ -157,8 +163,8 @@ const REST_PUT_IDLE_TIMEOUT_MS = 30_000
|
|
|
157
163
|
const HEARTBEAT_INTERVAL_MS = 30_000
|
|
158
164
|
|
|
159
165
|
// Backend selection. Both planes (workspace_revision DB + the
|
|
160
|
-
// v1.objstore byte store) are picked from config at boot. Two
|
|
161
|
-
// pairings:
|
|
166
|
+
// v1.objstore byte store) are picked from config at boot. Two
|
|
167
|
+
// supported pairings:
|
|
162
168
|
// 1. DATABASE_URL set → Neon (workspace_revision + objstore
|
|
163
169
|
// tables) + Vercel Blob Private Storage (bytes). Requires
|
|
164
170
|
// BLOB_READ_WRITE_TOKEN — fail fast at boot if missing, since
|
|
@@ -168,11 +174,10 @@ const HEARTBEAT_INTERVAL_MS = 30_000
|
|
|
168
174
|
// process; the only pairing the SQLite plane supports.
|
|
169
175
|
// The Neon / Vercel files import their peer deps lazily inside the
|
|
170
176
|
// open functions, so static imports here are safe even on a SQLite-
|
|
171
|
-
// only install where the optional peer deps aren't present.
|
|
172
|
-
//
|
|
173
|
-
//
|
|
174
|
-
//
|
|
175
|
-
// without a non-null assertion.
|
|
177
|
+
// only install where the optional peer deps aren't present. Explicit
|
|
178
|
+
// branch (not a ternary) so the SQLite path keeps its `SqliteHandle`
|
|
179
|
+
// narrowing — `sqliteHandle.db` is a non-optional `DatabaseSync` that
|
|
180
|
+
// `openObjstore` accepts without a non-null assertion.
|
|
176
181
|
let handle: Handle
|
|
177
182
|
let objstoreHandle: ObjstoreHandle
|
|
178
183
|
let objstoreBanner: string
|
|
@@ -232,24 +237,65 @@ async function workspaceExists(tag: string): Promise<boolean> {
|
|
|
232
237
|
}
|
|
233
238
|
|
|
234
239
|
// WS fan-out hub: subscriber registry + backpressure-aware send /
|
|
235
|
-
// broadcast (see ./hub.ts).
|
|
236
|
-
// handlers / dispatcher / objstore wiring below read unchanged.
|
|
240
|
+
// broadcast (see ./hub.ts).
|
|
237
241
|
const hub = createHub({ peers, maxBufferedBytes: MAX_BUFFERED_BYTES, debug: DEBUG })
|
|
238
|
-
const { send, broadcast, subscribe, unsubscribeAll } = hub
|
|
242
|
+
const { send, broadcast, subscribe, unsubscribeAll, broadcastLocalRaw } = hub
|
|
243
|
+
|
|
244
|
+
// Cross-instance pub/sub for live broadcasts (see ./pubsub.ts). SQLite
|
|
245
|
+
// mode is single-process by construction so it gets a no-op; Neon mode
|
|
246
|
+
// uses Postgres LISTEN/NOTIFY on a dedicated long-lived Client
|
|
247
|
+
// connection (separate from the HTTP `neon()` callable the queries flow
|
|
248
|
+
// over — LISTEN needs a session-bound socket, the HTTP path is
|
|
249
|
+
// stateless). The bus carries lookup hints, not the full wire payload:
|
|
250
|
+
// the `workspace-state` broadcast's ciphertext alone can exceed the
|
|
251
|
+
// ~8 KB NOTIFY payload cap, so the receiver re-fetches from the shared
|
|
252
|
+
// DB to construct the wire frame. The bus is best-effort fan-out, not a
|
|
253
|
+
// durability layer — the DB itself is the source of truth, and a
|
|
254
|
+
// dropped publish only means peers on other instances miss the live
|
|
255
|
+
// push (they still catch up via the chain on their next subscribe).
|
|
256
|
+
let pubsub: PubSub = createNoopPubSub()
|
|
257
|
+
if (NEON_URL) {
|
|
258
|
+
// Dynamic import: the Client export lives in the same optional peer
|
|
259
|
+
// dep as the HTTP `neon()` callable (see ./neon-driver.ts), but the
|
|
260
|
+
// dep itself is only present on a Neon-mode deploy. The wrapper path
|
|
261
|
+
// also lets tests swap in a PGlite-backed shim (`tests/pubsub.test.js`).
|
|
262
|
+
const mod = (await import('./neon-driver.ts')) as unknown as { Client?: NeonClientCtor }
|
|
263
|
+
if (!mod.Client) {
|
|
264
|
+
console.error('DATABASE_URL is set but the @neondatabase/serverless Client export is not available.')
|
|
265
|
+
console.error('Cross-instance broadcasts require the WebSocket-based Client (the HTTP `neon()` callable cannot LISTEN).')
|
|
266
|
+
console.error('Reinstall the peer dep: pnpm add @neondatabase/serverless')
|
|
267
|
+
process.exit(1)
|
|
268
|
+
}
|
|
269
|
+
const NeonClientImpl = mod.Client
|
|
270
|
+
pubsub = createNeonPubSub({
|
|
271
|
+
newClient: () => new NeonClientImpl(NEON_URL),
|
|
272
|
+
debug: DEBUG,
|
|
273
|
+
})
|
|
274
|
+
}
|
|
239
275
|
|
|
240
276
|
// Password gate (see ./auth.ts) — HMAC derivation + the `authenticate`
|
|
241
|
-
// handshake.
|
|
277
|
+
// handshake.
|
|
242
278
|
const auth = createAuth({ peers, password: CONFIG_PASSWORD, send, debug: DEBUG })
|
|
243
|
-
const { requiresAuth, handleAuthenticate, sendUnauthorized } = auth
|
|
279
|
+
const { requiresAuth, passwordConfigured, handleAuthenticate, sendUnauthorized } = auth
|
|
244
280
|
|
|
245
281
|
// Triage-sync protocol handlers (see ./sync-handlers.ts). `getNonce`
|
|
246
282
|
// resolves a socket's challenge nonce and is shared with the objstore
|
|
247
283
|
// wiring below; `sendSaveError` is reused by the dispatcher's `busy`
|
|
248
284
|
// inflight-cap NACK path.
|
|
249
285
|
const getNonce = (socket: WebSocket): string | undefined => peers.get(socket)?.challenge
|
|
250
|
-
const
|
|
251
|
-
|
|
252
|
-
|
|
286
|
+
const publishRevision = (tag: string, revisionId: string): void => {
|
|
287
|
+
pubsub.publish({ kind: 'rev', tag, id: revisionId })
|
|
288
|
+
}
|
|
289
|
+
const publishObjPut = (tag: string, resourceTag: string): void => {
|
|
290
|
+
pubsub.publish({ kind: 'objput', tag, res: resourceTag })
|
|
291
|
+
}
|
|
292
|
+
const publishObjDeleted = (tag: string, resourceTag: string, version: number): void => {
|
|
293
|
+
pubsub.publish({ kind: 'objdel', tag, res: resourceTag, ver: version })
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
const { handleSave, handleSaveRest, handleSubscribe, sendSaveError } = createSyncHandlers({
|
|
297
|
+
handle, send, broadcast, publishRevision, subscribe, getNonce,
|
|
298
|
+
requiresAuth, passwordConfigured, sendUnauthorized, workspaceExists,
|
|
253
299
|
// Folds the objstore inventory into the `workspace-subscribed` ack.
|
|
254
300
|
// The objstore store keeps its own richer `Handle`, so we wire the
|
|
255
301
|
// query here where both handles exist rather than coupling
|
|
@@ -260,7 +306,9 @@ const { handleSave, handleSubscribe, sendSaveError } = createSyncHandlers({
|
|
|
260
306
|
|
|
261
307
|
const { handlers: objstore, restDeps: objstoreRestDeps, startupReap, stopReaper } = initObjstore({
|
|
262
308
|
handle: objstoreHandle, reapIntervalMs: OBJSTORE_REAP_INTERVAL_MS,
|
|
263
|
-
|
|
309
|
+
reapDisabled: OBJSTORE_REAP_DISABLED,
|
|
310
|
+
send, broadcast, publishObjPut, publishObjDeleted,
|
|
311
|
+
getNonce, debug: DEBUG,
|
|
264
312
|
// Auth gate for the FIRST objstore-put-begin against a workspace
|
|
265
313
|
// that doesn't yet exist on the server. Mirrors handleSave's gate
|
|
266
314
|
// below; handlers.ts calls this AFTER sig verify so the
|
|
@@ -269,10 +317,15 @@ const { handlers: objstore, restDeps: objstoreRestDeps, startupReap, stopReaper
|
|
|
269
317
|
// `unauthorized` frame and bails on `true`.
|
|
270
318
|
authGate: async (socket, tag) => requiresAuth(socket) && !await workspaceExists(tag),
|
|
271
319
|
sendUnauthorized,
|
|
320
|
+
// Socket-less analog of `authGate` for the REST put-begin mint: a REST
|
|
321
|
+
// request can never be operator-authorized, so the gate collapses to
|
|
322
|
+
// "password configured AND workspace new". A deny routes the client to
|
|
323
|
+
// its in-band WS put-begin fallback.
|
|
324
|
+
restPutGate: async (tag) => passwordConfigured && !await workspaceExists(tag),
|
|
272
325
|
// `tokenSecret` is set only when OBJSTORE_TOKEN_SECRET was
|
|
273
326
|
// provided in env (see TOKEN_SECRET resolution above). Omitted
|
|
274
|
-
// → initObjstore mints a fresh per-process secret (
|
|
275
|
-
//
|
|
327
|
+
// → initObjstore mints a fresh per-process secret (fine for
|
|
328
|
+
// single-replica).
|
|
276
329
|
...(TOKEN_SECRET ? { tokenSecret: TOKEN_SECRET } : {}),
|
|
277
330
|
})
|
|
278
331
|
|
|
@@ -289,13 +342,48 @@ const wss = new WebSocketServer({ noServer: true, maxPayload: 4 * 1024 * 1024 })
|
|
|
289
342
|
// every server object exists.
|
|
290
343
|
const { track, isShuttingDown, install: installLifecycle } = createLifecycle()
|
|
291
344
|
|
|
292
|
-
//
|
|
293
|
-
//
|
|
294
|
-
//
|
|
295
|
-
|
|
345
|
+
// The sync protocol this build advertises — emitted as a `server-info` frame
|
|
346
|
+
// right after the challenge on every connection. This is the e2e boot, so it
|
|
347
|
+
// always advertises e2e.
|
|
348
|
+
const SERVER_INFO: ServerInfo = { mode: 'e2e', managed: null }
|
|
349
|
+
|
|
350
|
+
// Shared per-connection dispatch surface. Both the WS plane
|
|
351
|
+
// (installWsServer below) and the SSE+POST fallback (installSseServer
|
|
352
|
+
// below) drive `setupPeerConnection` with this; one Peer per accepted
|
|
353
|
+
// connection, one message-handler tree.
|
|
354
|
+
const peerConnectionDeps = {
|
|
355
|
+
peers, serverInfo: SERVER_INFO, send, unsubscribeAll,
|
|
356
|
+
handleSave, handleSubscribe, handleAuthenticate, sendSaveError, objstore,
|
|
357
|
+
track, isShuttingDown,
|
|
358
|
+
maxInflightPerSocket: MAX_INFLIGHT_PER_SOCKET,
|
|
359
|
+
debug: DEBUG,
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
// SSE+POST fallback transport (see ./sse-server.ts). The WS upgrade is
|
|
363
|
+
// the preferred path; clients fall back to SSE when `new WebSocket(…)`
|
|
364
|
+
// fails to open (corporate proxies that strip the Upgrade header,
|
|
365
|
+
// hostile network middleboxes, etc.). Same protocol on the wire — only
|
|
366
|
+
// the byte transport differs.
|
|
367
|
+
const sseServer = installSseServer({
|
|
368
|
+
peerDeps: peerConnectionDeps, isShuttingDown,
|
|
369
|
+
// Same cap shape as `wss` (no explicit per-process cap; OS FD limits
|
|
370
|
+
// are the upstream bound). 1024 leaves room above the typical web
|
|
371
|
+
// session count without being a soft DoS hammer.
|
|
372
|
+
maxSessions: 1024,
|
|
373
|
+
// Mirror the WS `maxPayload` so the SSE plane can't accept frames
|
|
374
|
+
// the WS plane would reject.
|
|
375
|
+
maxBodyBytes: 4 * 1024 * 1024,
|
|
376
|
+
debug: DEBUG,
|
|
377
|
+
})
|
|
378
|
+
|
|
379
|
+
// HTTP plane: REST byte-transfer routing + SSE fallback + the WS
|
|
380
|
+
// upgrade gate (see ./http.ts). Built after the lifecycle state above
|
|
381
|
+
// because the REST shutdown gate reads `shuttingDown` and the request
|
|
382
|
+
// drain uses `track`. The WS connection handler is wired on `wss`
|
|
383
|
+
// below.
|
|
296
384
|
const httpServer = createHttpServer({
|
|
297
|
-
wss, restDeps: objstoreRestDeps, isOriginAllowed,
|
|
298
|
-
isShuttingDown, track,
|
|
385
|
+
wss, restDeps: objstoreRestDeps, sseServer, serverInfo: SERVER_INFO, isOriginAllowed,
|
|
386
|
+
isShuttingDown, track, handleSaveRest,
|
|
299
387
|
restPutIdleTimeoutMs: REST_PUT_IDLE_TIMEOUT_MS, debug: DEBUG,
|
|
300
388
|
})
|
|
301
389
|
|
|
@@ -303,11 +391,8 @@ const httpServer = createHttpServer({
|
|
|
303
391
|
// sweep (see ./ws-server.ts). Returns the heartbeat timer so shutdown
|
|
304
392
|
// can clear it.
|
|
305
393
|
const { heartbeatTimer } = installWsServer({
|
|
306
|
-
wss,
|
|
307
|
-
|
|
308
|
-
track, isShuttingDown,
|
|
309
|
-
maxInflightPerSocket: MAX_INFLIGHT_PER_SOCKET, heartbeatIntervalMs: HEARTBEAT_INTERVAL_MS,
|
|
310
|
-
debug: DEBUG,
|
|
394
|
+
wss, heartbeatIntervalMs: HEARTBEAT_INTERVAL_MS,
|
|
395
|
+
...peerConnectionDeps,
|
|
311
396
|
})
|
|
312
397
|
|
|
313
398
|
httpServer.on('listening', () => {
|
|
@@ -323,12 +408,40 @@ httpServer.on('listening', () => {
|
|
|
323
408
|
// doesn't claim a misleading DB_PATH under Neon, or a misleading
|
|
324
409
|
// OBJSTORE_DIR under Vercel Blob.
|
|
325
410
|
const dbBanner = NEON_URL ? 'db: neon-postgres' : `db: ${DB_PATH}`
|
|
326
|
-
console.log(`DeepView triage-sync server: ws://${HOST}:${boundPort}${WS_UPGRADE_PATH} http://${HOST}:${boundPort}/api/objstore/{workspaceTag}/{resourceTag} (${dbBanner}, ${objstoreBanner})`)
|
|
411
|
+
console.log(`DeepView triage-sync server: ws://${HOST}:${boundPort}${WS_UPGRADE_PATH} (sse fallback http://${HOST}:${boundPort}${SSE_OPEN_PATH}) http://${HOST}:${boundPort}/api/objstore/{workspaceTag}/{resourceTag} (${dbBanner}, ${objstoreBanner})`)
|
|
412
|
+
})
|
|
413
|
+
|
|
414
|
+
// Cross-instance bus receiver (see ./bus-receiver.ts): a remote NOTIFY
|
|
415
|
+
// lands here, we re-fetch any data the bus payload only hinted at,
|
|
416
|
+
// then broadcast to local peers via `broadcastLocalRaw`. Extracted so
|
|
417
|
+
// the rev / objput / objdel mapping (including the keyframe boolean
|
|
418
|
+
// coercion and the `objectMetaWire` shape) is testable without the
|
|
419
|
+
// full server bring-up.
|
|
420
|
+
const onBusMessage = createBusReceiver({
|
|
421
|
+
handle, objstoreHandle, broadcastLocalRaw, debug: DEBUG,
|
|
422
|
+
})
|
|
423
|
+
|
|
424
|
+
// Kick off the LISTEN loop in the background. `pubsub.start` resolves
|
|
425
|
+
// only after the FIRST successful connect, and the internal reconnect
|
|
426
|
+
// loop retries indefinitely on transport / LISTEN failure — awaiting
|
|
427
|
+
// it here would convert a transient Neon WS hiccup at boot into a
|
|
428
|
+
// server that never binds (and a `.catch` that never runs). Instead
|
|
429
|
+
// the server binds immediately and the bus comes up async; publishes
|
|
430
|
+
// during the down window drop silently (best-effort semantics
|
|
431
|
+
// documented in pubsub.ts). The SQLite no-op resolves immediately,
|
|
432
|
+
// so the behaviour is identical in that mode.
|
|
433
|
+
void pubsub.start(onBusMessage).catch((err) => {
|
|
434
|
+
console.warn('pubsub: startup error:', errStack(err))
|
|
327
435
|
})
|
|
328
436
|
|
|
329
437
|
// App-specific shutdown step (run after the in-flight drain), wired
|
|
330
438
|
// into the lifecycle teardown below.
|
|
331
439
|
const closeDb = async (): Promise<void> => {
|
|
440
|
+
// Stop the bus first so a publish from a still-draining handler
|
|
441
|
+
// can't fire into a half-closed Client. The in-flight drain runs
|
|
442
|
+
// before this (see lifecycle.ts), so by here all `broadcast` →
|
|
443
|
+
// `publish*` calls are settled.
|
|
444
|
+
try { await pubsub.stop() } catch (err) { console.warn('pubsub close error:', errMsg(err)) }
|
|
332
445
|
// objstoreHandle has no close(): SQLite shares this DatabaseSync and
|
|
333
446
|
// Neon has no persistent connection; `handle.close()` covers both.
|
|
334
447
|
try { await handle.close() } catch (err) { console.warn('DB close error:', errMsg(err)) }
|
|
@@ -338,16 +451,36 @@ const closeDb = async (): Promise<void> => {
|
|
|
338
451
|
// and the heartbeat timer all exist.
|
|
339
452
|
installLifecycle({
|
|
340
453
|
httpServer, wss, heartbeatTimer, stopReaper,
|
|
454
|
+
sseSessions: sseServer.sessions,
|
|
455
|
+
sseKeepaliveTimer: sseServer.keepaliveTimer,
|
|
341
456
|
closeDb,
|
|
342
457
|
})
|
|
343
458
|
|
|
344
459
|
// Bind only after the startup orphan sweep finishes — otherwise a
|
|
345
460
|
// fresh boot could serve traffic against tags whose on-disk state
|
|
346
|
-
// still has residue from a prior crash.
|
|
347
|
-
//
|
|
348
|
-
//
|
|
349
|
-
//
|
|
350
|
-
// failure unconditionally and returns void), so no outer `.catch`
|
|
351
|
-
// is needed here.
|
|
461
|
+
// still has residue from a prior crash. `startupReap` already
|
|
462
|
+
// resolves on any error (the reaper's own catch logs the failure
|
|
463
|
+
// unconditionally and returns void), so no outer `.catch` is
|
|
464
|
+
// needed here.
|
|
352
465
|
await startupReap
|
|
353
|
-
|
|
466
|
+
|
|
467
|
+
// Bind the HTTP/WS plane on the configured PORT/HOST. Exported so a
|
|
468
|
+
// launcher (server-e2e/cli.js — the triage-server bin) or any `import`er can
|
|
469
|
+
// start serving. The top-level `await startupReap` above means the server
|
|
470
|
+
// is fully ready — DB open, DDL bootstrapped, objstore reaper swept — by
|
|
471
|
+
// the time the import resolves.
|
|
472
|
+
export function start(): void {
|
|
473
|
+
httpServer.listen(PORT, HOST)
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
export { httpServer, wss }
|
|
477
|
+
|
|
478
|
+
// Library mode: when this module is `import`ed (rather than run as the
|
|
479
|
+
// entry script) skip the auto-start so consumers can own the bind — e.g.
|
|
480
|
+
// wrap CF Access / framework-preset shims around the assembled
|
|
481
|
+
// `httpServer`, or just call `start()` when ready. Direct invocation via
|
|
482
|
+
// `node server-e2e/index.ts` (or the `triage-server` bin) still listens.
|
|
483
|
+
if (import.meta.main) {
|
|
484
|
+
start()
|
|
485
|
+
}
|
|
486
|
+
|
|
@@ -7,14 +7,24 @@
|
|
|
7
7
|
|
|
8
8
|
import type { Server } from 'node:http'
|
|
9
9
|
import type { WebSocketServer } from 'ws'
|
|
10
|
+
import type { SseSession } from './sse-session.ts'
|
|
10
11
|
import { errStack } from './util.ts'
|
|
11
12
|
|
|
12
13
|
export type ShutdownDeps = {
|
|
13
14
|
httpServer: Server
|
|
14
15
|
wss: WebSocketServer
|
|
15
16
|
heartbeatTimer: ReturnType<typeof setInterval>
|
|
17
|
+
// The SSE keepalive-sweep timer (server-e2e/sse-server.ts). Cleared on
|
|
18
|
+
// shutdown alongside `heartbeatTimer` so a tick can't write a keepalive
|
|
19
|
+
// comment to a session the close loop below is already tearing down.
|
|
20
|
+
sseKeepaliveTimer: ReturnType<typeof setInterval>
|
|
16
21
|
// Stops the periodic reaper AND awaits any in-flight sweep.
|
|
17
22
|
stopReaper: () => Promise<void>
|
|
23
|
+
// Live SSE+POST session iterator (mirrors `wss.clients` for the SSE
|
|
24
|
+
// fallback transport). Walked alongside `wss.clients` to send the
|
|
25
|
+
// 1001 close frame and apply the terminate-grace timer to both
|
|
26
|
+
// transports uniformly.
|
|
27
|
+
sseSessions: () => Iterable<SseSession>
|
|
18
28
|
// App-specific teardown, run AFTER the in-flight drain: close the DB.
|
|
19
29
|
// Swallows its own errors (a failed close shouldn't abort the exit).
|
|
20
30
|
closeDb: () => Promise<void>
|
|
@@ -40,7 +50,12 @@ export function createLifecycle(): Lifecycle {
|
|
|
40
50
|
const inFlight = new Set<Promise<unknown>>()
|
|
41
51
|
function track(promise: Promise<unknown>): void {
|
|
42
52
|
inFlight.add(promise)
|
|
43
|
-
|
|
53
|
+
// Trailing `.catch(() => {})` swallows the rejection that `.finally`
|
|
54
|
+
// propagates through its returned promise — without it, a tracked
|
|
55
|
+
// handler that rejects (or whose caller's catch handler itself
|
|
56
|
+
// throws) trips the `unhandledRejection` catchall below and crashes
|
|
57
|
+
// the process via `fireShutdown(1)`.
|
|
58
|
+
promise.finally(() => inFlight.delete(promise)).catch(() => {})
|
|
44
59
|
}
|
|
45
60
|
let shuttingDown = false
|
|
46
61
|
// Live exit code the in-progress shutdown will pass to `process.exit`.
|
|
@@ -51,7 +66,7 @@ export function createLifecycle(): Lifecycle {
|
|
|
51
66
|
let pendingExitCode = 0
|
|
52
67
|
|
|
53
68
|
function install(deps: ShutdownDeps): void {
|
|
54
|
-
const { httpServer, wss, heartbeatTimer, stopReaper, closeDb } = deps
|
|
69
|
+
const { httpServer, wss, heartbeatTimer, sseKeepaliveTimer, stopReaper, sseSessions, closeDb } = deps
|
|
55
70
|
|
|
56
71
|
async function shutdown(exitCode: number = 0): Promise<void> {
|
|
57
72
|
// Re-entry: don't restart the teardown, but escalate the pending
|
|
@@ -66,9 +81,9 @@ export function createLifecycle(): Lifecycle {
|
|
|
66
81
|
shuttingDown = true
|
|
67
82
|
pendingExitCode = exitCode
|
|
68
83
|
console.log('Shutting down…')
|
|
69
|
-
// Stop
|
|
70
|
-
//
|
|
71
|
-
|
|
84
|
+
// Stop both periodic timers (WS heartbeat + SSE keepalive) so neither
|
|
85
|
+
// fires mid-shutdown against a peer the close-loop is tearing down.
|
|
86
|
+
for (const timer of [heartbeatTimer, sseKeepaliveTimer]) clearInterval(timer)
|
|
72
87
|
// Send a 1001 (going away) close frame to every open socket BEFORE
|
|
73
88
|
// shutting the listener. Lets clients distinguish a server-initiated
|
|
74
89
|
// graceful shutdown from a network drop, so they can skip their
|
|
@@ -78,6 +93,13 @@ export function createLifecycle(): Lifecycle {
|
|
|
78
93
|
for (const socket of wss.clients) {
|
|
79
94
|
try { socket.close(1001, 'Server shutting down') } catch {}
|
|
80
95
|
}
|
|
96
|
+
// Same close frame on every SSE+POST session — the adapter
|
|
97
|
+
// translates `close(1001, …)` into a structured `event: close`
|
|
98
|
+
// record on the SSE stream so the client transport can read the
|
|
99
|
+
// code and skip its reconnect backoff (parity with the WS path).
|
|
100
|
+
for (const session of sseSessions()) {
|
|
101
|
+
try { session.close(1001, 'Server shutting down') } catch {}
|
|
102
|
+
}
|
|
81
103
|
// Force-terminate any client that doesn't ack the close frame
|
|
82
104
|
// within a short grace window. `wss.close()` waits for every client
|
|
83
105
|
// to emit `'close'`, and `ws` only TCP-RSTs unresponsive peers
|
|
@@ -92,6 +114,15 @@ export function createLifecycle(): Lifecycle {
|
|
|
92
114
|
try { socket.terminate() } catch {}
|
|
93
115
|
}
|
|
94
116
|
}
|
|
117
|
+
// SSE sessions are tracked separately from `wss.clients`; apply
|
|
118
|
+
// the same terminate-grace policy so a stranded SSE response
|
|
119
|
+
// can't pin the HTTP keep-alive shutdown branch below.
|
|
120
|
+
for (const session of sseSessions()) {
|
|
121
|
+
const rs = session.readyState
|
|
122
|
+
if (rs === session.OPEN || rs === session.CLOSING) {
|
|
123
|
+
try { session.terminate() } catch {}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
95
126
|
// Same grace for HTTP keep-alive sockets that didn't respect the
|
|
96
127
|
// `Connection: close` hint, which would otherwise hold
|
|
97
128
|
// `httpServer.close()` until their TCP timeout.
|