@preventive/triage 1.0.0-alpha.2 → 1.0.0-alpha.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/api/reap.ts +17 -0
  2. package/cli.js +6 -0
  3. package/client/finding-link.js +305 -0
  4. package/client/linked-findings.d.ts +1 -0
  5. package/client/linked-findings.js +111 -0
  6. package/common/bundle-metadata.d.ts +10 -0
  7. package/common/bundle-metadata.js +177 -0
  8. package/common/bundle-reasons.d.ts +2 -0
  9. package/common/bundle-reasons.js +21 -0
  10. package/common/bundle-sources.d.ts +3 -0
  11. package/common/bundle-sources.js +284 -0
  12. package/common/bundle-stats.js +41 -0
  13. package/common/bundle-tabs.js +1 -0
  14. package/common/code-language.js +36 -0
  15. package/common/default-scan-models.ts +30 -0
  16. package/common/finding-id.js +47 -0
  17. package/common/github-pr.ts +56 -0
  18. package/common/managed/comments.ts +34 -0
  19. package/common/managed/permissions.ts +35 -0
  20. package/common/managed/report-content.ts +42 -0
  21. package/common/managed/report-filter.ts +108 -0
  22. package/common/managed/roles.ts +28 -0
  23. package/common/managed/routes.d.ts +2 -0
  24. package/common/managed/routes.js +121 -0
  25. package/common/managed/scan-models.ts +6 -0
  26. package/common/managed/triage.ts +83 -0
  27. package/common/save-error-reason.ts +20 -7
  28. package/common/scan-server.ts +13 -0
  29. package/common/server-info.ts +33 -0
  30. package/common/utf8.d.ts +3 -0
  31. package/common/utf8.js +45 -0
  32. package/out/brotli-fallback.js +3 -3
  33. package/out/client-managed-import.js +81 -0
  34. package/out/client-managed.js +110 -0
  35. package/out/client-sync.js +16 -13
  36. package/out/graph.js +30 -4
  37. package/out/index.html +55 -8
  38. package/out/prism.js +2 -2
  39. package/out/stasis.svg +45 -0
  40. package/out/terminal.js +273 -39
  41. package/out/view.css +1 -1
  42. package/out/view.js +198 -62
  43. package/package.json +179 -55
  44. package/report/index.js +254 -0
  45. package/report/src/finding-id.js +80 -0
  46. package/report/src/finding.js +312 -0
  47. package/report/src/labels.js +33 -0
  48. package/report/src/md-structure.js +471 -0
  49. package/report/src/md-text.js +167 -0
  50. package/report/src/meta.js +76 -0
  51. package/report/src/parse-codex.js +147 -0
  52. package/report/src/parse-deepsec.js +197 -0
  53. package/report/src/parse-deepview-fields.js +375 -0
  54. package/report/src/parse-deepview-md.js +185 -0
  55. package/report/src/parse-md-id.js +137 -0
  56. package/report/src/parse-md.js +322 -0
  57. package/report/src/parse-piolium-id.js +79 -0
  58. package/report/src/parse-piolium-rows.js +131 -0
  59. package/report/src/parse-piolium-tokens.js +175 -0
  60. package/report/src/parse-piolium.js +400 -0
  61. package/report/src/security.js +63 -0
  62. package/report/src/utf8.js +21 -0
  63. package/report/src/write-md-finding.js +273 -0
  64. package/report/src/write-md.js +291 -0
  65. package/server-common/database-config.ts +16 -0
  66. package/server-common/initialize.ts +18 -0
  67. package/server-common/npm-advisories.ts +101 -0
  68. package/{server → server-common}/origin.ts +5 -5
  69. package/server-common/reap.ts +48 -0
  70. package/server-common/scan-config.ts +19 -0
  71. package/server-common/standalone.ts +29 -0
  72. package/server-common/storage-log.ts +34 -0
  73. package/server-common/vercel-blob.ts +110 -0
  74. package/server-e2e/app.ts +485 -0
  75. package/{server → server-e2e}/auth.ts +5 -1
  76. package/{server → server-e2e}/bus-receiver.ts +9 -8
  77. package/{server → server-e2e}/cli.js +9 -4
  78. package/{server → server-e2e}/config.ts +54 -39
  79. package/{server → server-e2e}/db-neon.ts +2 -2
  80. package/{server → server-e2e}/db-revision-sql.ts +7 -10
  81. package/{server → server-e2e}/db-stmt.ts +2 -2
  82. package/{server → server-e2e}/db.ts +96 -135
  83. package/{server → server-e2e}/http.ts +110 -12
  84. package/{server → server-e2e}/hub.ts +44 -14
  85. package/server-e2e/index.ts +17 -0
  86. package/server-e2e/lifecycle.ts +95 -0
  87. package/{server → server-e2e}/neon-driver.ts +2 -2
  88. package/{server → server-e2e}/npm-proxy.ts +11 -144
  89. package/{server → server-e2e}/objstore/blob-fs.ts +6 -8
  90. package/{server → server-e2e}/objstore/blob-vercel.ts +49 -141
  91. package/{server → server-e2e}/objstore/blob.ts +24 -9
  92. package/server-e2e/objstore/fetch-mint-guard.ts +74 -0
  93. package/{server → server-e2e}/objstore/handlers.ts +19 -20
  94. package/{server → server-e2e}/objstore/init.ts +53 -27
  95. package/{server → server-e2e}/objstore/reaper.ts +31 -11
  96. package/server-e2e/objstore/rest-deny.ts +28 -0
  97. package/server-e2e/objstore/rest-mint.ts +224 -0
  98. package/{server → server-e2e}/objstore/rest.ts +110 -93
  99. package/{server → server-e2e}/objstore/sign.ts +105 -0
  100. package/{server → server-e2e}/objstore/store-neon.ts +10 -14
  101. package/{server → server-e2e}/objstore/store.ts +98 -118
  102. package/{server → server-e2e}/objstore/tokens.ts +9 -12
  103. package/{server → server-e2e}/peer.ts +7 -9
  104. package/{server → server-e2e}/pubsub.ts +29 -36
  105. package/{server → server-e2e}/sign.ts +12 -14
  106. package/{server → server-e2e}/sse-server.ts +105 -73
  107. package/{server → server-e2e}/sse-session.ts +30 -16
  108. package/{server → server-e2e}/static.ts +36 -29
  109. package/server-e2e/sync-handlers.ts +408 -0
  110. package/{server → server-e2e}/util.ts +9 -0
  111. package/{server → server-e2e}/ws-server.ts +29 -23
  112. package/server-managed/activity.ts +231 -0
  113. package/server-managed/avatar-store.ts +51 -0
  114. package/server-managed/blob-store.ts +66 -0
  115. package/server-managed/blob-vercel.ts +125 -0
  116. package/server-managed/brotli.ts +10 -0
  117. package/server-managed/bundle-cache.ts +185 -0
  118. package/server-managed/bundle-catalog.ts +29 -0
  119. package/server-managed/bundle-store.ts +28 -0
  120. package/server-managed/bundle-summary-cache.ts +97 -0
  121. package/server-managed/bundle.ts +39 -0
  122. package/server-managed/cache-storage.ts +40 -0
  123. package/server-managed/cli.js +13 -0
  124. package/server-managed/combined.ts +46 -0
  125. package/server-managed/comments.ts +151 -0
  126. package/server-managed/config.ts +144 -0
  127. package/server-managed/content-access.ts +15 -0
  128. package/server-managed/crypto.ts +25 -0
  129. package/server-managed/db-methods.ts +1330 -0
  130. package/server-managed/db-neon.ts +171 -0
  131. package/server-managed/db-schema.ts +203 -0
  132. package/server-managed/db-table-names.ts +22 -0
  133. package/server-managed/db.ts +109 -0
  134. package/server-managed/github-app.ts +332 -0
  135. package/server-managed/github-metadata.ts +65 -0
  136. package/server-managed/github-oauth.ts +215 -0
  137. package/server-managed/github-pulls.ts +115 -0
  138. package/server-managed/http-response.ts +18 -0
  139. package/server-managed/http.ts +2043 -0
  140. package/server-managed/import-triage.ts +48 -0
  141. package/server-managed/index.ts +135 -0
  142. package/server-managed/public-workspace.ts +150 -0
  143. package/server-managed/repo-path.ts +21 -0
  144. package/server-managed/report-migration.ts +35 -0
  145. package/server-managed/report-query.ts +4 -0
  146. package/server-managed/report-response.ts +16 -0
  147. package/server-managed/report-sources.ts +154 -0
  148. package/server-managed/repository-discovery.ts +82 -0
  149. package/server-managed/repository-policy.ts +25 -0
  150. package/server-managed/session.ts +78 -0
  151. package/server-managed/slugs.ts +38 -0
  152. package/server-managed/sql-postgres.ts +30 -0
  153. package/server-managed/sql.ts +61 -0
  154. package/server-managed/static.ts +28 -0
  155. package/server-managed/storage.ts +34 -0
  156. package/server-managed/team-catalog.ts +7 -0
  157. package/server-managed/team-feed.ts +128 -0
  158. package/server-managed/team-reports.ts +156 -0
  159. package/server-managed/triage-response.ts +16 -0
  160. package/server-managed/uploads.ts +47 -0
  161. package/server-managed/workspace-shares.ts +150 -0
  162. package/server.ts +50 -0
  163. package/server/index.ts +0 -481
  164. package/server/lifecycle.ts +0 -204
  165. package/server/sync-handlers.ts +0 -327
  166. /package/{server → server-e2e}/config.example.json +0 -0
  167. /package/{server → server-e2e}/objstore/fs.ts +0 -0
  168. /package/{server → server-e2e}/validation.ts +0 -0
@@ -10,6 +10,7 @@ import { type IncomingMessage as HttpRequest, type Server, type ServerResponse,
10
10
  import { Buffer } from 'node:buffer'
11
11
  import { fileURLToPath } from 'node:url'
12
12
  import type { WebSocketServer } from 'ws'
13
+ import { CONFIG_PATH, type ServerInfo } from '../common/server-info.ts'
13
14
  import { type ObjstoreRestDeps, handleRest, matchRoute } from './objstore/rest.ts'
14
15
  import { SSE_OPEN_PATH, type SseServer } from './sse-server.ts'
15
16
  import { dispatchNpmAdvisories } from './npm-proxy.ts'
@@ -20,6 +21,12 @@ import { errStack } from './util.ts'
20
21
  // similar) can route `/api/*` → this process and `/*` → the static UI
21
22
  // bundle with a single location block.
22
23
  export const WS_UPGRADE_PATH = '/api/sync'
24
+ // Session-independent triage-sync save plane: `POST /api/sync/save`. The
25
+ // SSE-mode alternative to the in-band `workspace-save` frame — a save POSTed
26
+ // here commits + broadcasts WITHOUT taking over the client's SSE event-stream
27
+ // (each in-band POST forces a stream takeover; see sse-server.ts). Sibling of
28
+ // the WS upgrade path so one nginx `/api/*` block routes both.
29
+ export const SAVE_REST_PATH = '/api/sync/save'
23
30
  function isUpgradePath(url: string | undefined): boolean {
24
31
  if (typeof url !== 'string') return false
25
32
  // Strip `?…` so clients can carry build / debug tags. Exact match
@@ -34,6 +41,10 @@ type HasHeaders = { headers: HttpRequest['headers'] }
34
41
  export type HttpServerDeps = {
35
42
  wss: WebSocketServer
36
43
  restDeps: ObjstoreRestDeps
44
+ // This server's `server-info` (mode advertisement), served as JSON at
45
+ // GET /api/config so a client can detect the protocol without a sync
46
+ // connection (the WS connect frame stays the source of truth).
47
+ serverInfo: ServerInfo
37
48
  // SSE+POST fallback. Owns its own session map + lifecycle; we just
38
49
  // give it first crack at requests that match `SSE_OPEN_PATH`. Same
39
50
  // same-origin / shutdown gates run BEFORE the dispatch so the SSE
@@ -42,19 +53,88 @@ export type HttpServerDeps = {
42
53
  isOriginAllowed: (req: HasHeaders) => boolean
43
54
  isShuttingDown: () => boolean
44
55
  track: (promise: Promise<unknown>) => void
56
+ // Handler for `POST /api/sync/save` (see SAVE_REST_PATH). Owns body parse +
57
+ // the save pipeline + JSON response; this module owns the gates (method,
58
+ // same-origin, shutdown, idle-timeout) and the graceful-drain tracking.
59
+ handleSaveRest: (req: HttpRequest, res: ServerResponse) => Promise<void>
45
60
  restPutIdleTimeoutMs: number
46
61
  debug: boolean
47
62
  }
48
63
 
64
+ // `POST /api/sync/save` dispatch. Returns the in-flight handler promise when
65
+ // the request matched the route (the caller `track`s it for graceful drain),
66
+ // or null when it's for a different route. The gate ladder — method →
67
+ // shutdown → same-origin → idle-timeout — mirrors the objstore REST branch; a
68
+ // gate rejection writes its own response and returns an already-settled
69
+ // promise. Kept out of `createHttpServer` so that dispatcher stays compact.
70
+ function dispatchSaveRest(
71
+ req: HttpRequest, res: ServerResponse,
72
+ deps: {
73
+ handleSaveRest: (req: HttpRequest, res: ServerResponse) => Promise<void>
74
+ isOriginAllowed: (req: HasHeaders) => boolean
75
+ isShuttingDown: () => boolean
76
+ restPutIdleTimeoutMs: number
77
+ debug: boolean
78
+ },
79
+ ): Promise<void> | null {
80
+ if (typeof req.url !== 'string' || req.url.split('?', 1)[0] !== SAVE_REST_PATH) return null
81
+ if (req.method !== 'POST') {
82
+ res.writeHead(405, { 'content-type': 'application/json', 'allow': 'POST', 'connection': 'close' })
83
+ res.end(JSON.stringify({ error: 'method-not-allowed' }))
84
+ return Promise.resolve()
85
+ }
86
+ // Shutdown gate — parity with the objstore REST branch (a POST on an
87
+ // existing keep-alive socket after SIGTERM but before close() drains).
88
+ if (deps.isShuttingDown()) {
89
+ res.writeHead(503, { 'content-type': 'application/json', 'connection': 'close' })
90
+ res.end(JSON.stringify({ error: 'shutting-down' }))
91
+ return Promise.resolve()
92
+ }
93
+ // Same-origin gate — a hostile cross-origin page would carry an Origin
94
+ // header (browser-set on fetch); same-origin XHR may omit it (allowed).
95
+ if (!deps.isOriginAllowed(req)) {
96
+ res.writeHead(403, { 'content-type': 'application/json' })
97
+ res.end(JSON.stringify({ error: 'origin-denied' }))
98
+ return Promise.resolve()
99
+ }
100
+ // Idle-body timeout — a slow-loris trickling the JSON body would otherwise
101
+ // hold the connection indefinitely.
102
+ req.setTimeout(deps.restPutIdleTimeoutMs, () => {
103
+ if (deps.debug) console.warn(`sync-save REST idle ${deps.restPutIdleTimeoutMs}ms → abort`)
104
+ try { req.destroy(new Error('idle-timeout')) } catch {}
105
+ })
106
+ // Outer catch is the unhandled-rejection guard for a throw OUTSIDE the
107
+ // handler's own try/catch — logs and terminates the response so the TCP
108
+ // socket doesn't dangle (same policy as the objstore handleRest wrapper).
109
+ return deps.handleSaveRest(req, res).catch((err) => {
110
+ console.warn('sync-save REST handler error:', errStack(err))
111
+ if (res.headersSent) { try { res.destroy() } catch {} }
112
+ else { try { res.writeHead(500, { 'content-type': 'application/json' }); res.end(JSON.stringify({ error: 'internal' })) } catch {} }
113
+ })
114
+ }
115
+
49
116
  export function createHttpServer(deps: HttpServerDeps): Server {
50
- const { wss, restDeps, sseServer, isOriginAllowed, isShuttingDown, track, restPutIdleTimeoutMs, debug } = deps
117
+ const { wss, restDeps, sseServer, serverInfo, isOriginAllowed, isShuttingDown, track, handleSaveRest, restPutIdleTimeoutMs, debug } = deps
51
118
  // Static-file plane (see ./static.ts). The directory is the
52
119
  // `build.js build` output sibling to this file; the loader handles
53
120
  // enumeration, pre-compression, and ETag derivation. Plugged in after
54
121
  // the `/api/objstore/...` REST branch.
55
- const handleStatic = loadStatic(fileURLToPath(new URL('../out', import.meta.url)))
122
+ const handleStatic = loadStatic(fileURLToPath(new URL('../out', import.meta.url)), serverInfo.deepviewScanServer ?? null)
56
123
 
57
124
  const httpServer = createServer((req: HttpRequest, res: ServerResponse) => {
125
+ // Static mode probe: GET /api/config → this server's `server-info` as JSON.
126
+ // A client uses it to detect the protocol up front; the WS connect frame
127
+ // stays the source of truth and catches a later mode change. Public, no body.
128
+ if (typeof req.url === 'string' && req.url.split('?', 1)[0] === CONFIG_PATH) {
129
+ if (req.method !== 'GET') {
130
+ res.writeHead(405, { 'content-type': 'application/json', allow: 'GET' })
131
+ res.end(JSON.stringify({ error: 'method-not-allowed' }))
132
+ return
133
+ }
134
+ res.writeHead(200, { 'content-type': 'application/json', 'cache-control': 'no-store' })
135
+ res.end(JSON.stringify(serverInfo))
136
+ return
137
+ }
58
138
  // SSE+POST fallback plane. Same-origin gate first (Origin header
59
139
  // is set by the browser on cross-origin EventSource + fetch, so
60
140
  // a hostile origin would surface here just like it does on the
@@ -76,6 +156,12 @@ export function createHttpServer(deps: HttpServerDeps): Server {
76
156
  // the lifecycle's sseSessions() close loop.
77
157
  if (sseServer.handle(req, res)) return
78
158
  }
159
+ // Triage-sync save REST plane (see SAVE_REST_PATH) — session-independent
160
+ // `POST /api/sync/save` so an SSE-mode save doesn't take over the
161
+ // event-stream. The dispatch helper owns the gate ladder; we `track` the
162
+ // returned in-flight promise so SIGTERM drains it (mirrors npm-advisories).
163
+ const saveP = dispatchSaveRest(req, res, { handleSaveRest, isOriginAllowed, isShuttingDown, restPutIdleTimeoutMs, debug })
164
+ if (saveP) { track(saveP); return }
79
165
  // npm advisories proxy — same-origin + shutdown gates live in the
80
166
  // helper so this dispatcher stays compact. `dispatchNpmAdvisories`
81
167
  // returns the in-flight promise (or null when the route didn't
@@ -103,20 +189,22 @@ export function createHttpServer(deps: HttpServerDeps): Server {
103
189
  // that holds a valid token (e.g. via XSS that read a freshly-minted
104
190
  // one) would PUT with its own Origin header — caught here.
105
191
  // Same-origin XHR/fetch may omit Origin; that path is allowed (see
106
- // `isOriginAllowed`). Transport audit `server/objstore/rest.ts:103`.
192
+ // `isOriginAllowed`). Transport audit `server-e2e/objstore/rest.ts:103`.
107
193
  if (!isOriginAllowed(req)) {
108
194
  res.writeHead(403, { 'content-type': 'application/json' })
109
195
  res.end(JSON.stringify({ error: 'origin-denied' }))
110
196
  return
111
197
  }
112
- // PUT idle-body timeout — a slow-loris client trickling bytes
113
- // within the declared Content-Length holds the staging fd + an
114
- // inFlightSids slot indefinitely. `req.setTimeout` fires on
115
- // inactivity; we destroy the request, aborting the body pipeline.
116
- // Transport audit `server/objstore/rest.ts:218`.
117
- if (req.method === 'PUT') {
198
+ // Idle-body timeout for the body-bearing REST methods — a slow-loris
199
+ // client trickling bytes holds the connection (and, for PUT, the
200
+ // staging fd + inFlightSids slot) indefinitely. PUT carries the raw
201
+ // blob within its declared Content-Length; POST carries the small
202
+ // fetch-mint JSON body. `req.setTimeout` fires on inactivity; we
203
+ // destroy the request, aborting the body pipeline.
204
+ // Transport audit `server-e2e/objstore/rest.ts:218`.
205
+ if (req.method === 'PUT' || req.method === 'POST') {
118
206
  req.setTimeout(restPutIdleTimeoutMs, () => {
119
- if (debug) console.warn(`REST PUT idle ${restPutIdleTimeoutMs}ms → abort`)
207
+ if (debug) console.warn(`REST ${req.method} idle ${restPutIdleTimeoutMs}ms → abort`)
120
208
  try { req.destroy(new Error('idle-timeout')) } catch {}
121
209
  })
122
210
  }
@@ -142,7 +230,19 @@ export function createHttpServer(deps: HttpServerDeps): Server {
142
230
  res.end(NOT_FOUND_BODY)
143
231
  })
144
232
 
233
+ installUpgradeHandler(httpServer, wss, isOriginAllowed)
234
+
235
+ return httpServer
236
+ }
237
+
238
+ function installUpgradeHandler(httpServer: Server, wss: WebSocketServer, isOriginAllowed: HttpServerDeps['isOriginAllowed']): void {
145
239
  httpServer.on('upgrade', (req, socket, head) => {
240
+ // Combined deployments must not let a managed share capability enter the
241
+ // unscoped sync transport, which bypasses the normal HTTP router.
242
+ if (req.headers['x-deepview-share'] !== undefined) {
243
+ socket.end('HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n')
244
+ return
245
+ }
146
246
  // RFC 6455: the WS upgrade IS an HTTP request; reject with a normal
147
247
  // HTTP response so a misconfigured client sees the JSON body instead
148
248
  // of ECONNRESET. `socket.end(body)` flushes before sending FIN.
@@ -173,6 +273,4 @@ export function createHttpServer(deps: HttpServerDeps): Server {
173
273
  }
174
274
  wss.handleUpgrade(req, socket, head, (ws) => { wss.emit('connection', ws, req) })
175
275
  })
176
-
177
- return httpServer
178
276
  }
@@ -7,9 +7,13 @@
7
7
  // testable and reasoned about on its own.
8
8
 
9
9
  import type { WebSocket } from 'ws'
10
+ import { Buffer } from 'node:buffer'
10
11
  import type { PeerRegistry } from './peer.ts'
11
12
 
12
13
  export type Hub = {
14
+ // Buffer broadcasts during async subscription snapshots. The returned
15
+ // release flushes them after the snapshot/chain replies, preserving order.
16
+ pauseBroadcasts(socket: WebSocket): () => void
13
17
  subscribe(socket: WebSocket, tag: string): void
14
18
  unsubscribeAll(socket: WebSocket): void
15
19
  send(socket: WebSocket, msg: object): void
@@ -21,13 +25,13 @@ export type Hub = {
21
25
  // WS-originated broadcasts pass the originator so it doesn't see its
22
26
  // own message echoed back. Local-only: callers that ALSO want a
23
27
  // cross-instance fan-out are expected to publish to the pubsub bus
24
- // alongside this call (server/pubsub.ts). The hub deliberately stays
28
+ // alongside this call (server-e2e/pubsub.ts). The hub deliberately stays
25
29
  // ignorant of the bus so its transport invariants (backpressure cap,
26
30
  // stringify-once, terminate-on-overflow) are unchanged.
27
31
  broadcast(tag: string, msg: object, except: WebSocket | null): void
28
32
  // Broadcasts an ALREADY-SERIALISED payload to every local subscriber
29
33
  // for `tag`. Used by the pubsub bus receiver
30
- // (server/bus-receiver.ts, wired up from server/index.ts) to relay
34
+ // (server-e2e/bus-receiver.ts, wired up from server-e2e/index.ts) to relay
31
35
  // a remote-instance event into this instance's fan-out. No `except`:
32
36
  // the originator is on a different instance by construction.
33
37
  broadcastLocalRaw(tag: string, payload: string): void
@@ -38,18 +42,34 @@ export function createHub(deps: { peers: PeerRegistry; maxBufferedBytes: number;
38
42
  // workspaceTag → Set<WebSocket>. The per-socket reverse index lives on
39
43
  // `Peer.tags` (see ./peer.ts) and is read by `unsubscribeAll` on close.
40
44
  const subscribers = new Map<string, Set<WebSocket>>()
45
+ const paused = new WeakMap<WebSocket, { depth: number; payloads: string[]; bytes: number }>()
46
+
47
+ function pauseBroadcasts(socket: WebSocket): () => void {
48
+ const pending = paused.get(socket) ?? { depth: 0, payloads: [], bytes: 0 }
49
+ pending.depth++
50
+ paused.set(socket, pending)
51
+ let released = false
52
+ return () => {
53
+ if (released) return
54
+ released = true
55
+ if (--pending.depth > 0 || paused.get(socket) !== pending) return
56
+ paused.delete(socket)
57
+ for (const payload of pending.payloads) sendRaw(socket, payload)
58
+ pending.payloads.length = 0
59
+ }
60
+ }
41
61
 
42
62
  function subscribe(socket: WebSocket, tag: string): void {
43
63
  let set = subscribers.get(tag)
44
- if (!set) {
45
- set = new Set()
46
- subscribers.set(tag, set)
47
- }
64
+ if (!set) { set = new Set(); subscribers.set(tag, set) }
48
65
  set.add(socket)
49
66
  peers.get(socket)?.tags.add(tag)
50
67
  }
51
68
 
52
69
  function unsubscribeAll(socket: WebSocket): void {
70
+ const pending = paused.get(socket)
71
+ if (pending) pending.payloads.length = 0
72
+ paused.delete(socket)
53
73
  const tags = peers.get(socket)?.tags
54
74
  if (!tags) return
55
75
  for (const tag of tags) {
@@ -71,7 +91,7 @@ export function createHub(deps: { peers: PeerRegistry; maxBufferedBytes: number;
71
91
  // kernel yet — a slow / blackholed peer accumulates them unboundedly
72
92
  // during fan-out broadcasts. Drop above the cap and terminate the
73
93
  // socket so the heartbeat doesn't keep it alive on ping/pong while
74
- // every broadcast piles up. Transport audit `server/index.ts:225`.
94
+ // every broadcast piles up. Transport audit `server-e2e/index.ts:225`.
75
95
  if (socket.bufferedAmount > maxBufferedBytes) {
76
96
  if (debug) console.warn(`drop broadcast: socket buffered ${socket.bufferedAmount}B > cap`)
77
97
  try { socket.terminate() } catch {}
@@ -102,17 +122,27 @@ export function createHub(deps: { peers: PeerRegistry; maxBufferedBytes: number;
102
122
  }
103
123
 
104
124
  function fanOut(set: Set<WebSocket>, payload: string, except: WebSocket | null): void {
105
- // Snapshot before iterating — `send`'s try/catch swallows
106
- // socket.send errors, but a socket transitioning to CLOSED
125
+ const payloadBytes = Buffer.byteLength(payload)
126
+ // Snapshot before iterating — a socket transitioning to CLOSED
107
127
  // mid-broadcast triggers `unsubscribeAll` from the 'close' handler,
108
- // which mutates `set` while we're walking it. The snapshot keeps a
109
- // future refactor (different collection, async send) from silently
110
- // skipping subscribers. Audit M4 round-3.
128
+ // mutating `set` while we walk it. The snapshot also keeps a future
129
+ // refactor (different collection, async send) from silently skipping
130
+ // subscribers. Audit M4 round-3.
111
131
  for (const s of [...set]) {
112
132
  if (s === except) continue
113
- sendRaw(s, payload)
133
+ const pending = paused.get(s)
134
+ if (!pending) { sendRaw(s, payload); continue }
135
+ pending.bytes += payloadBytes
136
+ if (pending.bytes > maxBufferedBytes) {
137
+ // Snapshot waits must obey the same memory bound as socket writes.
138
+ pending.payloads.length = 0
139
+ paused.delete(s)
140
+ try { s.terminate() } catch {}
141
+ continue
142
+ }
143
+ pending.payloads.push(payload)
114
144
  }
115
145
  }
116
146
 
117
- return { subscribe, unsubscribeAll, send, sendRaw, broadcast, broadcastLocalRaw }
147
+ return { pauseBroadcasts, subscribe, unsubscribeAll, send, sendRaw, broadcast, broadcastLocalRaw }
118
148
  }
@@ -0,0 +1,17 @@
1
+ // Compatibility entry point for the published e2e server. Importing exposes
2
+ // one unbound instance; the combined launcher uses the factory directly.
3
+ import { startServer } from '../server-common/standalone.ts'
4
+ import { createE2eApp } from './app.ts'
5
+ import { HELP, loadConfig } from './config.ts'
6
+
7
+ if (import.meta.main && (process.argv.includes('--help') || process.argv.includes('-h'))) {
8
+ console.log(HELP)
9
+ process.exit(0)
10
+ }
11
+
12
+ const config = loadConfig()
13
+ export const { httpServer, wss, isShuttingDown, reap, handleRequest, onShutdown } = await createE2eApp(config)
14
+
15
+ export function start(): void { startServer(httpServer, config) }
16
+
17
+ if (import.meta.main) start()
@@ -0,0 +1,95 @@
1
+ // Per-instance request tracking and graceful disposal. Process shutdown belongs
2
+ // to the standalone launcher; embedded hosts retain their signals and errors.
3
+ import type { Server } from 'node:http'
4
+ import type { WebSocketServer } from 'ws'
5
+ import type { SseSession } from './sse-session.ts'
6
+
7
+ export type ShutdownDeps = {
8
+ httpServer: Server
9
+ wss: WebSocketServer
10
+ heartbeatTimer: ReturnType<typeof setInterval>
11
+ sseKeepaliveTimer: ReturnType<typeof setInterval>
12
+ stopReaper: () => Promise<void>
13
+ sseSessions: () => Iterable<SseSession>
14
+ closeDb: () => Promise<void>
15
+ }
16
+
17
+ export type Lifecycle = {
18
+ track: (promise: Promise<unknown>) => void
19
+ isShuttingDown: () => boolean
20
+ install: (deps: ShutdownDeps) => void
21
+ }
22
+
23
+ function closePeers({ wss, sseSessions }: ShutdownDeps): void {
24
+ for (const peer of [...wss.clients, ...sseSessions()]) {
25
+ try { peer.close(1001, 'Server shutting down') } catch {}
26
+ }
27
+ }
28
+
29
+ function terminatePeers({ httpServer, wss, sseSessions }: ShutdownDeps): void {
30
+ for (const peer of [...wss.clients, ...sseSessions()]) {
31
+ if (peer.readyState === peer.OPEN || peer.readyState === peer.CLOSING) {
32
+ try { peer.terminate() } catch {}
33
+ }
34
+ }
35
+ httpServer.closeAllConnections()
36
+ }
37
+
38
+ async function drain(deps: ShutdownDeps, inFlight: Set<Promise<unknown>>): Promise<void> {
39
+ const { httpServer, wss, stopReaper, closeDb } = deps
40
+ // Bound unresponsive WS/SSE peers and HTTP keep-alives to a one-second
41
+ // grace period. Also works when the listeners are mounted on another host.
42
+ const terminateTimer = setTimeout(() => terminatePeers(deps), 1000)
43
+ terminateTimer.unref()
44
+ try {
45
+ await stopReaper()
46
+ httpServer.closeIdleConnections()
47
+ if (httpServer.listening) {
48
+ await new Promise<void>(resolve => { httpServer.close(() => resolve()) })
49
+ }
50
+ await new Promise<void>(resolve => { wss.close(() => resolve()) })
51
+ if (inFlight.size > 0) await Promise.allSettled([...inFlight])
52
+ } finally {
53
+ clearTimeout(terminateTimer)
54
+ await closeDb()
55
+ }
56
+ }
57
+
58
+ export function createLifecycle(): Lifecycle {
59
+ const inFlight = new Set<Promise<unknown>>()
60
+ function track(promise: Promise<unknown>): void {
61
+ inFlight.add(promise)
62
+ promise.finally(() => inFlight.delete(promise)).catch(() => {})
63
+ }
64
+ let shuttingDown = false
65
+
66
+ function install(deps: ShutdownDeps): void {
67
+ const { httpServer, wss, heartbeatTimer, sseKeepaliveTimer } = deps
68
+ let disposal: Promise<void> | undefined
69
+ function dispose(): Promise<void> {
70
+ if (!disposal) {
71
+ shuttingDown = true
72
+ clearInterval(heartbeatTimer)
73
+ clearInterval(sseKeepaliveTimer)
74
+ // Store the promise before any close/error callback can re-enter.
75
+ disposal = Promise.resolve().then(() => drain(deps, inFlight)).finally(() => {
76
+ httpServer.off('close', onClose)
77
+ httpServer.off('error', onClose)
78
+ wss.off('error', onWsError)
79
+ })
80
+ closePeers(deps)
81
+ }
82
+ return disposal
83
+ }
84
+ function onClose(): void {
85
+ void dispose().catch(err => { console.error('Server cleanup failed:', err) })
86
+ }
87
+ function onWsError(err: Error): void { httpServer.emit('error', err) }
88
+ httpServer[Symbol.asyncDispose] = dispose
89
+ httpServer.once('close', onClose)
90
+ httpServer.on('error', onClose)
91
+ wss.on('error', onWsError)
92
+ }
93
+
94
+ return { track, isShuttingDown: () => shuttingDown, install }
95
+ }
@@ -13,10 +13,10 @@
13
13
  // so the absent peer dep is never loaded under test.
14
14
  //
15
15
  // In production the wrapper is only ever imported on the Neon path
16
- // (`DATABASE_URL` set), where the operator has installed the peer dep
16
+ // (an e2e database URL set), where the operator has installed the peer dep
17
17
  // (`pnpm add @neondatabase/serverless`) so the re-export resolves. A
18
18
  // SQLite-only deploy never imports this module — the dynamic `import()`
19
- // sites are gated behind the `DATABASE_URL` branch in `index.ts`.
19
+ // sites are gated behind the configured database URL branch in `index.ts`.
20
20
  //
21
21
  // `@ts-ignore` rather than `@ts-expect-error`: when the peer dep IS
22
22
  // installed the specifier resolves and tsc sees a real type, which
@@ -8,7 +8,7 @@
8
8
  // the request body and answers with the upstream's JSON.
9
9
  //
10
10
  // The route is mounted at `/api/npm-advisories`. The same-origin
11
- // gate runs inside `dispatchNpmAdvisories` below (server/http.ts
11
+ // gate runs inside `dispatchNpmAdvisories` below (server-e2e/http.ts
12
12
  // calls the dispatcher directly without a pre-check), so any
13
13
  // browser request from a foreign origin is rejected with 403 before
14
14
  // we ever issue a fetch — non-browser callers that omit Origin are
@@ -25,37 +25,23 @@
25
25
  //
26
26
  // Outbound fetch carries an `AbortController` so a hung upstream
27
27
  // (slow-loris response, TLS stall) tears down after
28
- // `UPSTREAM_TIMEOUT_MS`, and a client that closes its connection
28
+ // `NPM_ADVISORIES_TIMEOUT_MS`, and a client that closes its connection
29
29
  // mid-fetch propagates an abort through the same controller — so a
30
30
  // stranded in-flight call doesn't block SIGTERM drain.
31
31
 
32
32
  import type { IncomingMessage, ServerResponse } from 'node:http'
33
33
  import { Buffer } from 'node:buffer'
34
34
  import { errStack } from './util.ts'
35
+ import { NPM_ADVISORIES_TIMEOUT_MS, fetchNpmAdvisories } from '../server-common/npm-advisories.ts'
35
36
 
36
37
  type HasHeaders = { headers: IncomingMessage['headers'] }
37
38
 
38
39
  export const NPM_ADVISORIES_PATH = '/api/npm-advisories'
39
- const UPSTREAM_URL = 'https://registry.npmjs.org/-/npm/v1/security/advisories/bulk'
40
-
41
40
  // 1 MiB is generous: the bulk endpoint accepts `{ packageName:
42
41
  // [versions] }` maps, and even a bundle with thousands of pinned
43
42
  // versions serialises to well under this. Anything bigger is almost
44
43
  // certainly malformed input or a probe.
45
44
  const REQUEST_BODY_LIMIT = 1 * 1024 * 1024
46
- // 4 MiB caps the upstream response. The advisories endpoint
47
- // returns at most a few CVEs per package, so even a bundle with
48
- // hundreds of affected packages comes in well under this — a
49
- // runaway / hostile upstream gets cut off before we burn arbitrary
50
- // memory buffering it.
51
- const RESPONSE_BODY_LIMIT = 4 * 1024 * 1024
52
- // Hard deadline on the upstream call. The bulk endpoint typically
53
- // answers in well under a second; 30 s leaves plenty of headroom
54
- // for a slow path but caps a hung TLS / slow-loris connection so
55
- // a stranded fetch can't pin the inflight slot through SIGTERM
56
- // drain. Triggered via AbortController.
57
- const UPSTREAM_TIMEOUT_MS = 30_000
58
-
59
45
  export type NpmProxyDeps = {
60
46
  debug: boolean
61
47
  }
@@ -129,11 +115,12 @@ function deny(res: ServerResponse, status: number, reason: string): void {
129
115
  // Same shape as `deny` but for the richer multi-field envelopes
130
116
  // (upstream-not-json / upstream-too-large) — these can't reuse
131
117
  // `deny` because the body carries more than `{ error }`.
132
- function writeJsonEnvelope(res: ServerResponse, status: number, body: object): void {
118
+ function writeJsonEnvelope(res: ServerResponse, status: number, body: unknown): void {
133
119
  if (!canWrite(res)) return
134
120
  try {
135
- res.writeHead(status, { 'content-type': 'application/json', 'cache-control': 'no-store' })
136
- res.end(JSON.stringify(body))
121
+ const out = JSON.stringify(body)
122
+ res.writeHead(status, { 'content-type': 'application/json', 'cache-control': 'no-store', 'content-length': Buffer.byteLength(out) })
123
+ res.end(out)
137
124
  } catch {}
138
125
  }
139
126
 
@@ -162,7 +149,7 @@ export async function handleNpmAdvisories(
162
149
  if (req.method !== 'POST') { deny(res, 405, 'method-not-allowed'); return }
163
150
  // Single controller drives both the upstream deadline timer AND
164
151
  // the client-disconnect propagation: a fetch hung past
165
- // UPSTREAM_TIMEOUT_MS and a browser tab closed mid-fetch both end
152
+ // NPM_ADVISORIES_TIMEOUT_MS and a browser tab closed mid-fetch both end
166
153
  // up aborting the same signal, which undici threads through into
167
154
  // the body reader. Without this the inflight slot pinned by
168
155
  // `track()` could outlive both a dead client and a wedged
@@ -187,7 +174,7 @@ export async function handleNpmAdvisories(
187
174
  // doesn't fire a no-op abort — itself a no-op on a settled
188
175
  // controller, but skipping the log noise.)
189
176
  const controller = new AbortController()
190
- const timer = setTimeout(() => { try { controller.abort() } catch {} }, UPSTREAM_TIMEOUT_MS)
177
+ const timer = setTimeout(() => { try { controller.abort() } catch {} }, NPM_ADVISORIES_TIMEOUT_MS)
191
178
  const onResClose = (): void => {
192
179
  if (!res.writableEnded) controller.abort()
193
180
  }
@@ -219,130 +206,10 @@ export async function handleNpmAdvisories(
219
206
  try { req.destroy() } catch {}
220
207
  return
221
208
  }
222
- await handleNpmAdvisoriesInner(deps, body, res, controller.signal)
209
+ const result = await fetchNpmAdvisories(body, controller.signal, deps.debug)
210
+ writeJsonEnvelope(res, result.status, result.body)
223
211
  } finally {
224
212
  clearTimeout(timer)
225
213
  res.off('close', onResClose)
226
214
  }
227
215
  }
228
-
229
- async function handleNpmAdvisoriesInner(
230
- deps: NpmProxyDeps,
231
- body: Buffer,
232
- res: ServerResponse,
233
- signal: AbortSignal,
234
- ): Promise<void> {
235
- let upstream: Response
236
- try {
237
- upstream = await fetch(UPSTREAM_URL, {
238
- method: 'POST',
239
- // Force JSON — the bulk endpoint requires it. Drop every
240
- // client-supplied header to keep an upstream fingerprint from
241
- // leaking through (cookies, auth, custom UA, ...). The
242
- // registry's bulk endpoint doesn't need any of them for a
243
- // public lookup.
244
- headers: { 'content-type': 'application/json', 'accept': 'application/json' },
245
- // Re-wrap as a plain Uint8Array — Buffer's underlying
246
- // ArrayBufferLike type doesn't satisfy fetch's BodyInit
247
- // narrowing (it can't statically rule out SharedArrayBuffer),
248
- // but a copy through Uint8Array is zero-cost in practice and
249
- // unambiguously typed.
250
- body: new Uint8Array(body),
251
- signal,
252
- })
253
- } catch (err: unknown) {
254
- if (deps.debug) console.warn('npm-advisories upstream error:', errStack(err))
255
- // Client already gone — `canWrite` (inside `deny`) gates the
256
- // write so a destroyed / writableEnded socket doesn't trip
257
- // ERR_STREAM_DESTROYED on the way out.
258
- deny(res, 502, 'upstream-unreachable')
259
- return
260
- }
261
- // Assert JSON on the upstream body. The Content-Type header is
262
- // unreliable (Cloudflare in front of registry.npmjs.org strips it
263
- // from some responses; a captive portal / WAF can declare HTML on
264
- // a body that's actually JSON or vice-versa), so we don't lean on
265
- // it — instead we buffer the body and parse. A successful
266
- // JSON.parse is the strongest guarantee we can hand the UI's
267
- // `await res.json()`. Buffering is bounded by
268
- // `RESPONSE_BODY_LIMIT`; the advisories endpoint's payloads sit
269
- // well under that.
270
- const upstreamContentType = upstream.headers.get('content-type') ?? ''
271
- let buffered: Buffer | null
272
- try {
273
- buffered = await readUpstreamBody(upstream)
274
- } catch (err: unknown) {
275
- if (deps.debug) console.warn('npm-advisories upstream body error:', errStack(err))
276
- deny(res, 502, 'upstream-unreachable')
277
- return
278
- }
279
- if (buffered === null) {
280
- if (deps.debug) console.warn(`npm-advisories upstream too large: status=${upstream.status}`)
281
- writeJsonEnvelope(res, 502, { error: 'upstream-too-large', upstreamStatus: upstream.status })
282
- return
283
- }
284
- // Treat the body as UTF-8 — `JSON.parse` operates on a string and
285
- // the registry's responses are always UTF-8 in practice. A
286
- // non-UTF-8 byte sequence still decodes (with U+FFFD
287
- // substitution); the subsequent JSON.parse fails and routes
288
- // through the error branch.
289
- const text = buffered.toString('utf8')
290
- let parsed: unknown
291
- try {
292
- parsed = JSON.parse(text)
293
- } catch {
294
- if (deps.debug) console.warn(`npm-advisories upstream non-JSON: status=${upstream.status} ct=${upstreamContentType || '<none>'} bytes=${buffered.byteLength}`)
295
- writeJsonEnvelope(res, 502, {
296
- error: 'upstream-not-json',
297
- upstreamStatus: upstream.status,
298
- upstreamContentType: upstreamContentType || null,
299
- })
300
- return
301
- }
302
- if (!canWrite(res)) return
303
- // Re-stringify rather than echoing `text` so the wire shape we
304
- // emit is canonical (no upstream whitespace / BOM / trailing
305
- // junk after the parsed value rides along), and so the client
306
- // can rely on a single JSON document per response.
307
- const out = JSON.stringify(parsed)
308
- try {
309
- res.writeHead(upstream.status, {
310
- 'content-type': 'application/json',
311
- 'cache-control': 'no-store',
312
- 'content-length': Buffer.byteLength(out),
313
- })
314
- res.end(out)
315
- } catch {}
316
- }
317
-
318
- // Buffer the upstream response body up to RESPONSE_BODY_LIMIT.
319
- // Returns null if the cap is exceeded (caller maps to a 502
320
- // `upstream-too-large`), or the cumulative Buffer otherwise. A
321
- // transport error mid-read (e.g. AbortSignal fired by the deadline
322
- // timer or by `req` close) throws — the caller catches it.
323
- //
324
- // `finally { reader.cancel() }` is load-bearing on the error and
325
- // cap-exceeded paths: leaving the reader locked to the body holds
326
- // the underlying undici TCP socket out of the connection pool until
327
- // GC, and the cap-exceeded path explicitly needs to tear the
328
- // transfer down so we don't keep buffering bytes we'll never use.
329
- // On the clean-drain path (done:true), cancel() is a no-op.
330
- async function readUpstreamBody(upstream: Response): Promise<Buffer | null> {
331
- if (!upstream.body) return Buffer.alloc(0)
332
- const reader = upstream.body.getReader()
333
- const chunks: Uint8Array[] = []
334
- let received = 0
335
- try {
336
- for (;;) {
337
- const { done, value } = await reader.read()
338
- if (done) break
339
- if (!value) continue
340
- received += value.byteLength
341
- if (received > RESPONSE_BODY_LIMIT) return null
342
- chunks.push(value)
343
- }
344
- return Buffer.concat(chunks)
345
- } finally {
346
- try { await reader.cancel() } catch {}
347
- }
348
- }