@preventive/triage 1.0.0-alpha.3 → 1.0.0-alpha.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/LICENSE-APACHE +202 -0
  2. package/api/reap.ts +17 -0
  3. package/cli.js +6 -0
  4. package/client/finding-link.js +305 -0
  5. package/client/linked-findings.d.ts +1 -0
  6. package/client/linked-findings.js +111 -0
  7. package/common/bundle-commit.d.ts +1 -0
  8. package/common/bundle-commit.js +4 -0
  9. package/common/bundle-metadata.d.ts +10 -0
  10. package/common/bundle-metadata.js +182 -0
  11. package/common/bundle-reasons.d.ts +2 -0
  12. package/common/bundle-reasons.js +21 -0
  13. package/common/bundle-source-links.d.ts +5 -0
  14. package/common/bundle-source-links.js +82 -0
  15. package/common/bundle-sources.d.ts +3 -0
  16. package/common/bundle-sources.js +284 -0
  17. package/common/bundle-stats.js +41 -0
  18. package/common/bundle-tabs.js +1 -0
  19. package/common/code-language.js +51 -0
  20. package/common/default-scan-models.ts +30 -0
  21. package/common/finding-id.js +47 -0
  22. package/common/github-issue-labels.d.ts +1 -0
  23. package/common/github-issue-labels.js +11 -0
  24. package/common/github-pr.ts +56 -0
  25. package/common/managed/comments.ts +34 -0
  26. package/common/managed/permissions.ts +35 -0
  27. package/common/managed/report-content.ts +42 -0
  28. package/common/managed/report-filter.ts +108 -0
  29. package/common/managed/roles.ts +28 -0
  30. package/common/managed/routes.d.ts +2 -0
  31. package/common/managed/routes.js +124 -0
  32. package/common/managed/scan-models.ts +6 -0
  33. package/common/managed/triage.ts +83 -0
  34. package/common/save-error-reason.ts +20 -7
  35. package/common/scan-server.ts +13 -0
  36. package/common/server-info.ts +35 -0
  37. package/common/utf8.d.ts +3 -0
  38. package/common/utf8.js +45 -0
  39. package/out/brotli-fallback.js +3 -3
  40. package/out/client-managed-import.js +83 -0
  41. package/out/client-managed.js +169 -0
  42. package/out/client-sync.js +16 -13
  43. package/out/github-issue-authorized.html +3 -0
  44. package/out/github-issue-authorized.js +1 -0
  45. package/out/graph.js +30 -4
  46. package/out/index.html +55 -8
  47. package/out/prism.js +2 -2
  48. package/out/stasis.svg +20 -0
  49. package/out/terminal.js +273 -39
  50. package/out/view.css +1 -1
  51. package/out/view.js +259 -63
  52. package/package.json +205 -57
  53. package/server-common/database-config.ts +16 -0
  54. package/server-common/initialize.ts +18 -0
  55. package/server-common/npm-advisories.ts +101 -0
  56. package/{server → server-common}/origin.ts +5 -5
  57. package/server-common/reap.ts +48 -0
  58. package/server-common/scan-config.ts +19 -0
  59. package/server-common/standalone.ts +29 -0
  60. package/server-common/storage-crypto.ts +182 -0
  61. package/server-common/storage-log.ts +38 -0
  62. package/server-common/vercel-blob.ts +103 -0
  63. package/server-e2e/app.ts +486 -0
  64. package/{server → server-e2e}/auth.ts +5 -1
  65. package/{server → server-e2e}/bus-receiver.ts +2 -1
  66. package/{server → server-e2e}/cli.js +9 -4
  67. package/{server → server-e2e}/config.ts +56 -35
  68. package/{server → server-e2e}/db-neon.ts +2 -2
  69. package/{server → server-e2e}/db-revision-sql.ts +1 -1
  70. package/{server → server-e2e}/db-stmt.ts +2 -2
  71. package/{server → server-e2e}/db.ts +7 -7
  72. package/{server → server-e2e}/http.ts +110 -12
  73. package/{server → server-e2e}/hub.ts +40 -9
  74. package/server-e2e/index.ts +17 -0
  75. package/server-e2e/lifecycle.ts +95 -0
  76. package/{server → server-e2e}/neon-driver.ts +2 -2
  77. package/{server → server-e2e}/npm-proxy.ts +11 -144
  78. package/{server → server-e2e}/objstore/blob-fs.ts +2 -2
  79. package/{server → server-e2e}/objstore/blob-vercel.ts +13 -117
  80. package/{server → server-e2e}/objstore/blob.ts +9 -2
  81. package/server-e2e/objstore/fetch-mint-guard.ts +74 -0
  82. package/{server → server-e2e}/objstore/handlers.ts +9 -8
  83. package/{server → server-e2e}/objstore/init.ts +35 -18
  84. package/{server → server-e2e}/objstore/reaper.ts +31 -11
  85. package/server-e2e/objstore/rest-deny.ts +28 -0
  86. package/server-e2e/objstore/rest-mint.ts +224 -0
  87. package/{server → server-e2e}/objstore/rest.ts +58 -33
  88. package/{server → server-e2e}/objstore/sign.ts +105 -0
  89. package/{server → server-e2e}/objstore/store-neon.ts +6 -10
  90. package/{server → server-e2e}/objstore/store.ts +7 -7
  91. package/{server → server-e2e}/objstore/tokens.ts +1 -1
  92. package/{server → server-e2e}/pubsub.ts +12 -9
  93. package/{server → server-e2e}/sign.ts +4 -4
  94. package/{server → server-e2e}/sse-server.ts +102 -68
  95. package/{server → server-e2e}/sse-session.ts +28 -8
  96. package/{server → server-e2e}/static.ts +31 -21
  97. package/server-e2e/sync-handlers.ts +408 -0
  98. package/{server → server-e2e}/util.ts +9 -0
  99. package/{server → server-e2e}/ws-server.ts +20 -9
  100. package/server-managed/activity.ts +231 -0
  101. package/server-managed/avatar-store.ts +12 -0
  102. package/server-managed/blob-store.ts +17 -0
  103. package/server-managed/brotli.ts +10 -0
  104. package/server-managed/bundle-advisories.ts +53 -0
  105. package/server-managed/bundle-advisory-inventory.ts +75 -0
  106. package/server-managed/bundle-build-diagnostics.d.ts +10 -0
  107. package/server-managed/bundle-build-diagnostics.js +23 -0
  108. package/server-managed/bundle-build-leases.ts +41 -0
  109. package/server-managed/bundle-build-worker.js +54 -0
  110. package/server-managed/bundle-build.ts +168 -0
  111. package/server-managed/bundle-cache.ts +166 -0
  112. package/server-managed/bundle-catalog.ts +29 -0
  113. package/server-managed/bundle-store.ts +24 -0
  114. package/server-managed/bundle-summary-cache.ts +116 -0
  115. package/server-managed/bundle.ts +104 -0
  116. package/server-managed/cache-storage.ts +19 -0
  117. package/server-managed/cli.js +21 -0
  118. package/server-managed/combined.ts +46 -0
  119. package/server-managed/comments.ts +151 -0
  120. package/server-managed/config.ts +149 -0
  121. package/server-managed/content-access.ts +15 -0
  122. package/server-managed/crypto.ts +25 -0
  123. package/server-managed/db-methods.ts +1517 -0
  124. package/server-managed/db-neon.ts +277 -0
  125. package/server-managed/db-schema.ts +206 -0
  126. package/server-managed/db-table-names.ts +22 -0
  127. package/server-managed/db.ts +120 -0
  128. package/server-managed/github-app.ts +385 -0
  129. package/server-managed/github-errors.ts +49 -0
  130. package/server-managed/github-issue-oauth.ts +48 -0
  131. package/server-managed/github-issues.ts +152 -0
  132. package/server-managed/github-metadata.ts +98 -0
  133. package/server-managed/github-oauth.ts +199 -0
  134. package/server-managed/github-pulls.ts +227 -0
  135. package/server-managed/http-response.ts +18 -0
  136. package/server-managed/http.ts +2350 -0
  137. package/server-managed/import-triage.ts +51 -0
  138. package/server-managed/index.ts +175 -0
  139. package/server-managed/managed-issues.ts +47 -0
  140. package/server-managed/management-catalog.ts +45 -0
  141. package/server-managed/management.ts +86 -0
  142. package/server-managed/object-storage-disk.ts +144 -0
  143. package/server-managed/object-storage-vercel.ts +63 -0
  144. package/server-managed/object-storage.ts +71 -0
  145. package/server-managed/package-entry-points.ts +51 -0
  146. package/server-managed/public-workspace.ts +146 -0
  147. package/server-managed/repo-path.ts +21 -0
  148. package/server-managed/report-migration.ts +35 -0
  149. package/server-managed/report-query.ts +10 -0
  150. package/server-managed/report-response.ts +16 -0
  151. package/server-managed/report-sources.ts +166 -0
  152. package/server-managed/repository-browser.ts +139 -0
  153. package/server-managed/repository-discovery.ts +82 -0
  154. package/server-managed/repository-policy.ts +27 -0
  155. package/server-managed/revisions.ts +35 -0
  156. package/server-managed/session.ts +78 -0
  157. package/server-managed/slugs.ts +38 -0
  158. package/server-managed/solidity-entry-points.ts +69 -0
  159. package/server-managed/sql-postgres.ts +31 -0
  160. package/server-managed/sql.ts +72 -0
  161. package/server-managed/static.ts +28 -0
  162. package/server-managed/storage-command.ts +16 -0
  163. package/server-managed/storage-db.ts +175 -0
  164. package/server-managed/storage-encryption.ts +211 -0
  165. package/server-managed/storage-maintenance.ts +223 -0
  166. package/server-managed/storage-payload.ts +72 -0
  167. package/server-managed/storage-stores.ts +81 -0
  168. package/server-managed/storage.ts +58 -0
  169. package/server-managed/team-catalog.ts +7 -0
  170. package/server-managed/team-feed.ts +173 -0
  171. package/server-managed/team-reports.ts +219 -0
  172. package/server-managed/triage-response.ts +16 -0
  173. package/server-managed/uploads.ts +65 -0
  174. package/server-managed/workspace-shares.ts +162 -0
  175. package/server.ts +50 -0
  176. package/server/index.ts +0 -477
  177. package/server/lifecycle.ts +0 -204
  178. package/server/sync-handlers.ts +0 -320
  179. /package/{LICENSE → LICENSE-MIT} +0 -0
  180. /package/{server → server-e2e}/config.example.json +0 -0
  181. /package/{server → server-e2e}/objstore/fs.ts +0 -0
  182. /package/{server → server-e2e}/peer.ts +0 -0
  183. /package/{server → server-e2e}/validation.ts +0 -0
@@ -0,0 +1,95 @@
1
+ // Per-instance request tracking and graceful disposal. Process shutdown belongs
2
+ // to the standalone launcher; embedded hosts retain their signals and errors.
3
+ import type { Server } from 'node:http'
4
+ import type { WebSocketServer } from 'ws'
5
+ import type { SseSession } from './sse-session.ts'
6
+
7
+ export type ShutdownDeps = {
8
+ httpServer: Server
9
+ wss: WebSocketServer
10
+ heartbeatTimer: ReturnType<typeof setInterval>
11
+ sseKeepaliveTimer: ReturnType<typeof setInterval>
12
+ stopReaper: () => Promise<void>
13
+ sseSessions: () => Iterable<SseSession>
14
+ closeDb: () => Promise<void>
15
+ }
16
+
17
+ export type Lifecycle = {
18
+ track: (promise: Promise<unknown>) => void
19
+ isShuttingDown: () => boolean
20
+ install: (deps: ShutdownDeps) => void
21
+ }
22
+
23
+ function closePeers({ wss, sseSessions }: ShutdownDeps): void {
24
+ for (const peer of [...wss.clients, ...sseSessions()]) {
25
+ try { peer.close(1001, 'Server shutting down') } catch {}
26
+ }
27
+ }
28
+
29
+ function terminatePeers({ httpServer, wss, sseSessions }: ShutdownDeps): void {
30
+ for (const peer of [...wss.clients, ...sseSessions()]) {
31
+ if (peer.readyState === peer.OPEN || peer.readyState === peer.CLOSING) {
32
+ try { peer.terminate() } catch {}
33
+ }
34
+ }
35
+ httpServer.closeAllConnections()
36
+ }
37
+
38
+ async function drain(deps: ShutdownDeps, inFlight: Set<Promise<unknown>>): Promise<void> {
39
+ const { httpServer, wss, stopReaper, closeDb } = deps
40
+ // Bound unresponsive WS/SSE peers and HTTP keep-alives to a one-second
41
+ // grace period. Also works when the listeners are mounted on another host.
42
+ const terminateTimer = setTimeout(() => terminatePeers(deps), 1000)
43
+ terminateTimer.unref()
44
+ try {
45
+ await stopReaper()
46
+ httpServer.closeIdleConnections()
47
+ if (httpServer.listening) {
48
+ await new Promise<void>(resolve => { httpServer.close(() => resolve()) })
49
+ }
50
+ await new Promise<void>(resolve => { wss.close(() => resolve()) })
51
+ if (inFlight.size > 0) await Promise.allSettled([...inFlight])
52
+ } finally {
53
+ clearTimeout(terminateTimer)
54
+ await closeDb()
55
+ }
56
+ }
57
+
58
+ export function createLifecycle(): Lifecycle {
59
+ const inFlight = new Set<Promise<unknown>>()
60
+ function track(promise: Promise<unknown>): void {
61
+ inFlight.add(promise)
62
+ promise.finally(() => inFlight.delete(promise)).catch(() => {})
63
+ }
64
+ let shuttingDown = false
65
+
66
+ function install(deps: ShutdownDeps): void {
67
+ const { httpServer, wss, heartbeatTimer, sseKeepaliveTimer } = deps
68
+ let disposal: Promise<void> | undefined
69
+ function dispose(): Promise<void> {
70
+ if (!disposal) {
71
+ shuttingDown = true
72
+ clearInterval(heartbeatTimer)
73
+ clearInterval(sseKeepaliveTimer)
74
+ // Store the promise before any close/error callback can re-enter.
75
+ disposal = Promise.resolve().then(() => drain(deps, inFlight)).finally(() => {
76
+ httpServer.off('close', onClose)
77
+ httpServer.off('error', onClose)
78
+ wss.off('error', onWsError)
79
+ })
80
+ closePeers(deps)
81
+ }
82
+ return disposal
83
+ }
84
+ function onClose(): void {
85
+ void dispose().catch(err => { console.error('Server cleanup failed:', err) })
86
+ }
87
+ function onWsError(err: Error): void { httpServer.emit('error', err) }
88
+ httpServer[Symbol.asyncDispose] = dispose
89
+ httpServer.once('close', onClose)
90
+ httpServer.on('error', onClose)
91
+ wss.on('error', onWsError)
92
+ }
93
+
94
+ return { track, isShuttingDown: () => shuttingDown, install }
95
+ }
@@ -13,10 +13,10 @@
13
13
  // so the absent peer dep is never loaded under test.
14
14
  //
15
15
  // In production the wrapper is only ever imported on the Neon path
16
- // (`DATABASE_URL` set), where the operator has installed the peer dep
16
+ // (an e2e database URL set), where the operator has installed the peer dep
17
17
  // (`pnpm add @neondatabase/serverless`) so the re-export resolves. A
18
18
  // SQLite-only deploy never imports this module — the dynamic `import()`
19
- // sites are gated behind the `DATABASE_URL` branch in `index.ts`.
19
+ // sites are gated behind the configured database URL branch in `index.ts`.
20
20
  //
21
21
  // `@ts-ignore` rather than `@ts-expect-error`: when the peer dep IS
22
22
  // installed the specifier resolves and tsc sees a real type, which
@@ -8,7 +8,7 @@
8
8
  // the request body and answers with the upstream's JSON.
9
9
  //
10
10
  // The route is mounted at `/api/npm-advisories`. The same-origin
11
- // gate runs inside `dispatchNpmAdvisories` below (server/http.ts
11
+ // gate runs inside `dispatchNpmAdvisories` below (server-e2e/http.ts
12
12
  // calls the dispatcher directly without a pre-check), so any
13
13
  // browser request from a foreign origin is rejected with 403 before
14
14
  // we ever issue a fetch — non-browser callers that omit Origin are
@@ -25,37 +25,23 @@
25
25
  //
26
26
  // Outbound fetch carries an `AbortController` so a hung upstream
27
27
  // (slow-loris response, TLS stall) tears down after
28
- // `UPSTREAM_TIMEOUT_MS`, and a client that closes its connection
28
+ // `NPM_ADVISORIES_TIMEOUT_MS`, and a client that closes its connection
29
29
  // mid-fetch propagates an abort through the same controller — so a
30
30
  // stranded in-flight call doesn't block SIGTERM drain.
31
31
 
32
32
  import type { IncomingMessage, ServerResponse } from 'node:http'
33
33
  import { Buffer } from 'node:buffer'
34
34
  import { errStack } from './util.ts'
35
+ import { NPM_ADVISORIES_TIMEOUT_MS, fetchNpmAdvisories } from '../server-common/npm-advisories.ts'
35
36
 
36
37
  type HasHeaders = { headers: IncomingMessage['headers'] }
37
38
 
38
39
  export const NPM_ADVISORIES_PATH = '/api/npm-advisories'
39
- const UPSTREAM_URL = 'https://registry.npmjs.org/-/npm/v1/security/advisories/bulk'
40
-
41
40
  // 1 MiB is generous: the bulk endpoint accepts `{ packageName:
42
41
  // [versions] }` maps, and even a bundle with thousands of pinned
43
42
  // versions serialises to well under this. Anything bigger is almost
44
43
  // certainly malformed input or a probe.
45
44
  const REQUEST_BODY_LIMIT = 1 * 1024 * 1024
46
- // 4 MiB caps the upstream response. The advisories endpoint
47
- // returns at most a few CVEs per package, so even a bundle with
48
- // hundreds of affected packages comes in well under this — a
49
- // runaway / hostile upstream gets cut off before we burn arbitrary
50
- // memory buffering it.
51
- const RESPONSE_BODY_LIMIT = 4 * 1024 * 1024
52
- // Hard deadline on the upstream call. The bulk endpoint typically
53
- // answers in well under a second; 30 s leaves plenty of headroom
54
- // for a slow path but caps a hung TLS / slow-loris connection so
55
- // a stranded fetch can't pin the inflight slot through SIGTERM
56
- // drain. Triggered via AbortController.
57
- const UPSTREAM_TIMEOUT_MS = 30_000
58
-
59
45
  export type NpmProxyDeps = {
60
46
  debug: boolean
61
47
  }
@@ -129,11 +115,12 @@ function deny(res: ServerResponse, status: number, reason: string): void {
129
115
  // Same shape as `deny` but for the richer multi-field envelopes
130
116
  // (upstream-not-json / upstream-too-large) — these can't reuse
131
117
  // `deny` because the body carries more than `{ error }`.
132
- function writeJsonEnvelope(res: ServerResponse, status: number, body: object): void {
118
+ function writeJsonEnvelope(res: ServerResponse, status: number, body: unknown): void {
133
119
  if (!canWrite(res)) return
134
120
  try {
135
- res.writeHead(status, { 'content-type': 'application/json', 'cache-control': 'no-store' })
136
- res.end(JSON.stringify(body))
121
+ const out = JSON.stringify(body)
122
+ res.writeHead(status, { 'content-type': 'application/json', 'cache-control': 'no-store', 'content-length': Buffer.byteLength(out) })
123
+ res.end(out)
137
124
  } catch {}
138
125
  }
139
126
 
@@ -162,7 +149,7 @@ export async function handleNpmAdvisories(
162
149
  if (req.method !== 'POST') { deny(res, 405, 'method-not-allowed'); return }
163
150
  // Single controller drives both the upstream deadline timer AND
164
151
  // the client-disconnect propagation: a fetch hung past
165
- // UPSTREAM_TIMEOUT_MS and a browser tab closed mid-fetch both end
152
+ // NPM_ADVISORIES_TIMEOUT_MS and a browser tab closed mid-fetch both end
166
153
  // up aborting the same signal, which undici threads through into
167
154
  // the body reader. Without this the inflight slot pinned by
168
155
  // `track()` could outlive both a dead client and a wedged
@@ -187,7 +174,7 @@ export async function handleNpmAdvisories(
187
174
  // doesn't fire a no-op abort — itself a no-op on a settled
188
175
  // controller, but skipping the log noise.)
189
176
  const controller = new AbortController()
190
- const timer = setTimeout(() => { try { controller.abort() } catch {} }, UPSTREAM_TIMEOUT_MS)
177
+ const timer = setTimeout(() => { try { controller.abort() } catch {} }, NPM_ADVISORIES_TIMEOUT_MS)
191
178
  const onResClose = (): void => {
192
179
  if (!res.writableEnded) controller.abort()
193
180
  }
@@ -219,130 +206,10 @@ export async function handleNpmAdvisories(
219
206
  try { req.destroy() } catch {}
220
207
  return
221
208
  }
222
- await handleNpmAdvisoriesInner(deps, body, res, controller.signal)
209
+ const result = await fetchNpmAdvisories(body, controller.signal, deps.debug)
210
+ writeJsonEnvelope(res, result.status, result.body)
223
211
  } finally {
224
212
  clearTimeout(timer)
225
213
  res.off('close', onResClose)
226
214
  }
227
215
  }
228
-
229
- async function handleNpmAdvisoriesInner(
230
- deps: NpmProxyDeps,
231
- body: Buffer,
232
- res: ServerResponse,
233
- signal: AbortSignal,
234
- ): Promise<void> {
235
- let upstream: Response
236
- try {
237
- upstream = await fetch(UPSTREAM_URL, {
238
- method: 'POST',
239
- // Force JSON — the bulk endpoint requires it. Drop every
240
- // client-supplied header to keep an upstream fingerprint from
241
- // leaking through (cookies, auth, custom UA, ...). The
242
- // registry's bulk endpoint doesn't need any of them for a
243
- // public lookup.
244
- headers: { 'content-type': 'application/json', 'accept': 'application/json' },
245
- // Re-wrap as a plain Uint8Array — Buffer's underlying
246
- // ArrayBufferLike type doesn't satisfy fetch's BodyInit
247
- // narrowing (it can't statically rule out SharedArrayBuffer),
248
- // but a copy through Uint8Array is zero-cost in practice and
249
- // unambiguously typed.
250
- body: new Uint8Array(body),
251
- signal,
252
- })
253
- } catch (err: unknown) {
254
- if (deps.debug) console.warn('npm-advisories upstream error:', errStack(err))
255
- // Client already gone — `canWrite` (inside `deny`) gates the
256
- // write so a destroyed / writableEnded socket doesn't trip
257
- // ERR_STREAM_DESTROYED on the way out.
258
- deny(res, 502, 'upstream-unreachable')
259
- return
260
- }
261
- // Assert JSON on the upstream body. The Content-Type header is
262
- // unreliable (Cloudflare in front of registry.npmjs.org strips it
263
- // from some responses; a captive portal / WAF can declare HTML on
264
- // a body that's actually JSON or vice-versa), so we don't lean on
265
- // it — instead we buffer the body and parse. A successful
266
- // JSON.parse is the strongest guarantee we can hand the UI's
267
- // `await res.json()`. Buffering is bounded by
268
- // `RESPONSE_BODY_LIMIT`; the advisories endpoint's payloads sit
269
- // well under that.
270
- const upstreamContentType = upstream.headers.get('content-type') ?? ''
271
- let buffered: Buffer | null
272
- try {
273
- buffered = await readUpstreamBody(upstream)
274
- } catch (err: unknown) {
275
- if (deps.debug) console.warn('npm-advisories upstream body error:', errStack(err))
276
- deny(res, 502, 'upstream-unreachable')
277
- return
278
- }
279
- if (buffered === null) {
280
- if (deps.debug) console.warn(`npm-advisories upstream too large: status=${upstream.status}`)
281
- writeJsonEnvelope(res, 502, { error: 'upstream-too-large', upstreamStatus: upstream.status })
282
- return
283
- }
284
- // Treat the body as UTF-8 — `JSON.parse` operates on a string and
285
- // the registry's responses are always UTF-8 in practice. A
286
- // non-UTF-8 byte sequence still decodes (with U+FFFD
287
- // substitution); the subsequent JSON.parse fails and routes
288
- // through the error branch.
289
- const text = buffered.toString('utf8')
290
- let parsed: unknown
291
- try {
292
- parsed = JSON.parse(text)
293
- } catch {
294
- if (deps.debug) console.warn(`npm-advisories upstream non-JSON: status=${upstream.status} ct=${upstreamContentType || '<none>'} bytes=${buffered.byteLength}`)
295
- writeJsonEnvelope(res, 502, {
296
- error: 'upstream-not-json',
297
- upstreamStatus: upstream.status,
298
- upstreamContentType: upstreamContentType || null,
299
- })
300
- return
301
- }
302
- if (!canWrite(res)) return
303
- // Re-stringify rather than echoing `text` so the wire shape we
304
- // emit is canonical (no upstream whitespace / BOM / trailing
305
- // junk after the parsed value rides along), and so the client
306
- // can rely on a single JSON document per response.
307
- const out = JSON.stringify(parsed)
308
- try {
309
- res.writeHead(upstream.status, {
310
- 'content-type': 'application/json',
311
- 'cache-control': 'no-store',
312
- 'content-length': Buffer.byteLength(out),
313
- })
314
- res.end(out)
315
- } catch {}
316
- }
317
-
318
- // Buffer the upstream response body up to RESPONSE_BODY_LIMIT.
319
- // Returns null if the cap is exceeded (caller maps to a 502
320
- // `upstream-too-large`), or the cumulative Buffer otherwise. A
321
- // transport error mid-read (e.g. AbortSignal fired by the deadline
322
- // timer or by `req` close) throws — the caller catches it.
323
- //
324
- // `finally { reader.cancel() }` is load-bearing on the error and
325
- // cap-exceeded paths: leaving the reader locked to the body holds
326
- // the underlying undici TCP socket out of the connection pool until
327
- // GC, and the cap-exceeded path explicitly needs to tear the
328
- // transfer down so we don't keep buffering bytes we'll never use.
329
- // On the clean-drain path (done:true), cancel() is a no-op.
330
- async function readUpstreamBody(upstream: Response): Promise<Buffer | null> {
331
- if (!upstream.body) return Buffer.alloc(0)
332
- const reader = upstream.body.getReader()
333
- const chunks: Uint8Array[] = []
334
- let received = 0
335
- try {
336
- for (;;) {
337
- const { done, value } = await reader.read()
338
- if (done) break
339
- if (!value) continue
340
- received += value.byteLength
341
- if (received > RESPONSE_BODY_LIMIT) return null
342
- chunks.push(value)
343
- }
344
- return Buffer.concat(chunks)
345
- } finally {
346
- try { await reader.cancel() } catch {}
347
- }
348
- }
@@ -48,13 +48,13 @@ async function fsOpenLiveReader(dir: string, tag: string, contentHash: string):
48
48
  const path = liveFilePath(dir, tag, contentHash)
49
49
  let fh
50
50
  try { fh = await open(path, 'r') } catch (err: unknown) {
51
- if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return { ok: false, reason: 'unavailable' }
51
+ if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return { ok: false, reason: 'unavailable', detail: 'fs-enoent' }
52
52
  throw err
53
53
  }
54
54
  let size: number
55
55
  try { size = (await fh.stat()).size } catch {
56
56
  await fh.close().catch(() => {})
57
- return { ok: false, reason: 'unavailable' }
57
+ return { ok: false, reason: 'unavailable', detail: 'fs-stat-failed' }
58
58
  }
59
59
  const stream = fh.createReadStream()
60
60
  let closed = false
@@ -1,5 +1,5 @@
1
1
  // Vercel Blob Private Storage BlobBackend. Paired with the Neon DB
2
- // plane in `server/index.ts` when `BLOB_READ_WRITE_TOKEN` is set;
2
+ // plane in `server-e2e/index.ts` when `BLOB_READ_WRITE_TOKEN` is set;
3
3
  // the combination is the supported multi-replica deployment shape
4
4
  // (Neon for metadata, Vercel Blob for bytes — both serverless, both
5
5
  // HTTP-backed, no shared filesystem required).
@@ -35,108 +35,10 @@
35
35
  // any single resource), and the stale-staging TTL sweep drops orphan
36
36
  // staging blobs.
37
37
 
38
- import { PassThrough, type Readable } from 'node:stream'
39
- import { Buffer } from 'node:buffer'
38
+ import { PassThrough } from 'node:stream'
40
39
  import type { BlobBackend, OpenLiveResult, StagingWriter } from './blob.ts'
41
40
  import { errMsg } from '../util.ts'
42
-
43
- // Minimal structural shape of the bits of `@vercel/blob` we use.
44
- // Kept local so the optional peer dep doesn't have to type-resolve
45
- // for SQLite-only deployments. Real type details live in the
46
- // installed package; the fields/parameters we touch here are stable
47
- // per the SDK v2 public surface.
48
- //
49
- // `put` accepts the SDK's full `PutBody` union (string | Readable |
50
- // Buffer | Blob | ArrayBuffer | ReadableStream | File). We only ever
51
- // pass a Node `PassThrough` (a Readable), but the wider type lets
52
- // callers reuse this signature for future buffer/blob bodies without
53
- // type gymnastics. `copy` accepts `allowOverwrite` — REQUIRED for
54
- // version bumps since the live pathname is reused on re-upload.
55
- type VercelBlobBody = Readable | Buffer | string | Blob | ArrayBuffer | ReadableStream<Uint8Array>
56
- type VercelBlobSdk = {
57
- put: (
58
- pathname: string,
59
- body: VercelBlobBody,
60
- options: {
61
- access: 'private' | 'public'
62
- allowOverwrite?: boolean
63
- contentType?: string
64
- token?: string
65
- multipart?: boolean
66
- abortSignal?: AbortSignal
67
- cacheControlMaxAge?: number
68
- },
69
- ) => Promise<{ url: string; pathname: string }>
70
- head: (
71
- pathname: string,
72
- options?: { token?: string; abortSignal?: AbortSignal },
73
- ) => Promise<{ size: number; pathname: string; url: string }>
74
- get: (
75
- pathname: string,
76
- options: {
77
- access: 'private' | 'public'
78
- token?: string
79
- useCache?: boolean
80
- abortSignal?: AbortSignal
81
- },
82
- ) => Promise<{
83
- statusCode: 200 | 304
84
- stream: ReadableStream<Uint8Array> | null
85
- blob: { size: number | null }
86
- } | null>
87
- copy: (
88
- fromPathname: string,
89
- toPathname: string,
90
- options: {
91
- access: 'private' | 'public'
92
- allowOverwrite?: boolean
93
- token?: string
94
- contentType?: string
95
- cacheControlMaxAge?: number
96
- },
97
- ) => Promise<{ url: string; pathname: string }>
98
- del: (
99
- urlOrPathname: string | string[],
100
- options?: { token?: string; abortSignal?: AbortSignal },
101
- ) => Promise<void>
102
- list: (options: {
103
- prefix?: string
104
- cursor?: string
105
- limit?: number
106
- mode?: 'expanded' | 'folded'
107
- token?: string
108
- }) => Promise<{
109
- // `uploadedAt` (a Date per the SDK v2 surface) is the blob's
110
- // creation time — used by the reaper's GC grace window. Optional
111
- // in the type because the `folded` listing path ignores it (only
112
- // `listLiveBlobs` reads it, and it uses the default expanded mode).
113
- blobs: Array<{ pathname: string; size: number; uploadedAt?: Date | string | number }>
114
- folders?: string[]
115
- cursor?: string
116
- hasMore: boolean
117
- }>
118
- }
119
-
120
- // Recognise "blob is gone" errors uniformly across read/write/delete
121
- // paths so callers can treat them as success (delete) or
122
- // not-found (read). The SDK exposes BlobNotFoundError as a class with
123
- // `.name === 'BlobNotFoundError'`; checking the name string avoids
124
- // importing the class at the top level (which would force the optional
125
- // peer dep to resolve).
126
- //
127
- // Class-name check ONLY — the SDK's internal mapper turns every API
128
- // `not_found` into BlobNotFoundError-by-name, so a bare-404 transport
129
- // leak doesn't reach here. A broader `/does not exist|\b404\b/` match
130
- // is DANGEROUS: it also matches BlobStoreNotFoundError's "This store
131
- // does not exist.", so a config fault (revoked token, deleted store)
132
- // would silently surface as every-blob-missing across reads/unlinks,
133
- // masking the fatal misconfiguration. The tight name check lets
134
- // BlobStoreNotFoundError / other classes propagate as real exceptions.
135
- function isNotFound(err: unknown): boolean {
136
- if (err == null || typeof err !== 'object') return false
137
- const name = (err as { name?: unknown }).name
138
- return typeof name === 'string' && name === 'BlobNotFoundError'
139
- }
41
+ import { type VercelBlobSdk, isNotFound, loadVercelBlobSdk } from '../../server-common/vercel-blob.ts'
140
42
 
141
43
  function liveBlobPath(tag: string, contentHash: string): string {
142
44
  return `${tag}/${contentHash}.bin`
@@ -293,7 +195,7 @@ function buildStatStaging(sdk: VercelBlobSdk, token: string): BlobBackend['statS
293
195
  const h = await sdk.head(stagingBlobPath(tag, stagingId), { token })
294
196
  return h.size
295
197
  } catch (err) {
296
- if (isNotFound(err)) return null
198
+ if (isNotFound(err, sdk)) return null
297
199
  throw err
298
200
  }
299
201
  }
@@ -363,20 +265,20 @@ function buildOpenLiveReader(sdk: VercelBlobSdk, token: string): BlobBackend['op
363
265
  // `not-found` would emit a 404 the FS backend never emits for the
364
266
  // same condition (blob-fs.ts maps ENOENT → `unavailable`), telling
365
267
  // the client the resource is gone for good when it should refetch.
366
- // See server/README.md's GET status table.
367
- if (isNotFound(err)) return { ok: false, reason: 'unavailable' }
268
+ // See server-e2e/README.md's GET status table.
269
+ if (isNotFound(err, sdk)) return { ok: false, reason: 'unavailable', detail: 'vercel-get-not-found' }
368
270
  throw err
369
271
  }
370
272
  // SDK returned null (no blob) — same "bytes missing for a live row"
371
273
  // transient as the BlobNotFoundError branch above → `unavailable`, not
372
274
  // `not-found`.
373
- if (res == null) return { ok: false, reason: 'unavailable' }
275
+ if (res == null) return { ok: false, reason: 'unavailable', detail: 'vercel-get-null' }
374
276
  // statusCode 304 doesn't reach here in practice — the REST
375
277
  // GET layer doesn't pass If-None-Match — but a future call
376
278
  // site could. Treat as unavailable rather than streaming a
377
279
  // null body.
378
280
  if (res.statusCode !== 200 || res.stream == null) {
379
- return { ok: false, reason: 'unavailable' }
281
+ return { ok: false, reason: 'unavailable', detail: `vercel-get-status-${res.statusCode}` }
380
282
  }
381
283
  // `@vercel/blob@2.x`'s streaming `get()` for private blobs returns
382
284
  // the body but does NOT populate `blob.size` nor pass a
@@ -395,11 +297,11 @@ function buildOpenLiveReader(sdk: VercelBlobSdk, token: string): BlobBackend['op
395
297
  // Blob vanished between get() and the head() size fallback (a
396
298
  // racing reaper GC) — still the "live row present, bytes gone"
397
299
  // transient, so `unavailable` (503), matching the get() path above.
398
- if (isNotFound(headErr)) return { ok: false, reason: 'unavailable' }
300
+ if (isNotFound(headErr, sdk)) return { ok: false, reason: 'unavailable', detail: 'vercel-head-not-found' }
399
301
  throw headErr
400
302
  }
401
303
  }
402
- if (size == null) return { ok: false, reason: 'unavailable' }
304
+ if (size == null) return { ok: false, reason: 'unavailable', detail: 'vercel-no-size' }
403
305
  // SDK returns a web ReadableStream<Uint8Array>; the REST layer
404
306
  // expects a Node Readable for pipeline(). Convert via
405
307
  // Readable.fromWeb — built-in and zero-copy where possible.
@@ -426,7 +328,7 @@ function buildOpenLiveReader(sdk: VercelBlobSdk, token: string): BlobBackend['op
426
328
  function buildUnlink(sdk: VercelBlobSdk, token: string, op: 'staging' | 'live', toPath: (tag: string, id: string) => string): (tag: string, id: string) => Promise<void> {
427
329
  return async (tag, id) => {
428
330
  try { await sdk.del(toPath(tag, id), { token }) } catch (err) {
429
- if (isNotFound(err)) return
331
+ if (isNotFound(err, sdk)) return
430
332
  // Don't propagate — the DB-side row drop has already
431
333
  // committed by the time the caller reaches here, and the
432
334
  // reaper picks up the stranded blob on its next sweep. Same
@@ -493,7 +395,7 @@ export type VercelBlobBackendOptions = {
493
395
  // Vercel Blob R/W token, typically from BLOB_READ_WRITE_TOKEN.
494
396
  // The SDK also reads it from process.env, but passing it
495
397
  // explicitly here keeps the env-var → boot config path single-
496
- // sourced through server/index.ts (matches the Neon DATABASE_URL
398
+ // sourced through server-e2e/index.ts (matches the Neon database URL
497
399
  // handling — env-read at boot, threaded as a parameter).
498
400
  token: string
499
401
  // Test seam: inject a stub of the @vercel/blob module to avoid
@@ -507,7 +409,7 @@ export async function openVercelBlobBackend(opts: VercelBlobBackendOptions): Pro
507
409
  // dep. `@ts-ignore` (not `@ts-expect-error`) so an operator who
508
410
  // DOES install `@vercel/blob` doesn't trip TS2578 "unused
509
411
  // directive" — same pattern as db-neon.ts.
510
- const sdk: VercelBlobSdk = opts.sdk ?? (await loadSdk())
412
+ const sdk: VercelBlobSdk = opts.sdk ?? (await loadVercelBlobSdk())
511
413
  const token = opts.token
512
414
  return {
513
415
  // No-op: Vercel Blob has no folder concept. The pathname's
@@ -533,9 +435,3 @@ export async function openVercelBlobBackend(opts: VercelBlobBackendOptions): Pro
533
435
  listStagingIds: buildListStagingIds(sdk, token, (tag) => `${tag}/.staging/`),
534
436
  }
535
437
  }
536
-
537
- async function loadSdk(): Promise<VercelBlobSdk> {
538
- // @ts-ignore optional peer dep: '@vercel/blob'
539
- const mod = (await import('@vercel/blob')) as VercelBlobSdk
540
- return mod
541
- }
@@ -11,7 +11,7 @@
11
11
  // the Neon DB plane for multi-replica
12
12
  // deployments)
13
13
  //
14
- // Selected at boot in `server/index.ts` and passed to `openObjstore`
14
+ // Selected at boot in `server-e2e/index.ts` and passed to `openObjstore`
15
15
  // / `openNeonObjstore`. The DB-plane code in ./store.ts, ./rest.ts,
16
16
  // and ./reaper.ts goes through `handle.blob.*` and is backend-
17
17
  // agnostic — no `if (vercel) … else` branching in consumers.
@@ -94,9 +94,16 @@ export type LiveReader = {
94
94
  // `unavailable`/503 contract, which the client retries. Both backends MUST
95
95
  // map a missing blob to `unavailable` (FS: ENOENT; Vercel: BlobNotFoundError
96
96
  // / null get()). No 404-mapping variant exists here so that bug can't recur.
97
+ //
98
+ // `detail` is a short, NON-SENSITIVE machine tag for the specific cause
99
+ // (e.g. 'vercel-get-not-found', 'fs-enoent', 'vercel-no-size'). Every
100
+ // byte-side failure collapses to the same 503 on the wire, so a permanent
101
+ // loss (reaper GC'd the bytes) and a transient read fault are otherwise
102
+ // indistinguishable — the REST layer logs `detail` so an operator can tell
103
+ // them apart. Purely diagnostic; the REST status is unchanged.
97
104
  export type OpenLiveResult =
98
105
  | { ok: true; reader: LiveReader }
99
- | { ok: false; reason: 'unavailable' }
106
+ | { ok: false; reason: 'unavailable'; detail?: string }
100
107
 
101
108
  export type BlobBackend = {
102
109
  // Per-workspace setup. FS creates the on-disk staging directory;
@@ -0,0 +1,74 @@
1
+ // Anti-replay guard for the REST fetch-mint endpoint (POST
2
+ // /api/objstore/{tag}/{res}, see ./rest.ts). The WS fetch handshake binds
3
+ // the per-connection challenge nonce so a captured frame can't be
4
+ // replayed; the REST mint has no connection, so it binds a client
5
+ // timestamp instead and this guard supplies the matching freshness +
6
+ // dedup the nonce gave for free:
7
+ //
8
+ // - FRESHNESS: reject a request whose `ts` is outside ±`windowMs` of
9
+ // server time (an old captured request, or a future-dated one).
10
+ // - DEDUP: within the window, a signature is accepted at most once — a
11
+ // captured-and-replayed request (same `ts` ⇒ same signature) is
12
+ // rejected on the second presentation.
13
+ //
14
+ // Memory: each accepted signature is held for `windowMs` then pruned.
15
+ // Entries share a uniform TTL and the Map preserves insertion order, so
16
+ // the oldest entries expire first and a cheap front-prune (amortised
17
+ // O(1) per entry) keeps the set bounded; a hard `maxEntries` cap drops
18
+ // the oldest beyond it as a flood backstop.
19
+ //
20
+ // Scope: per-process. In a multi-replica deployment a captured request
21
+ // could be replayed once per replica that hasn't seen its signature yet
22
+ // (bounded by the replica count, within the window) — acceptable because
23
+ // the mint only ever yields a short-TTL GET token over AEAD ciphertext
24
+ // the relay can't read. A shared-store dedup (Redis/Neon) would close
25
+ // that residual gap if ever needed.
26
+
27
+ export type FetchMintVerdict = 'ok' | 'stale' | 'replay'
28
+
29
+ export type FetchMintGuard = {
30
+ // `signature` is the request's Ed25519 signature (unique per
31
+ // (tag, res, ts) tuple, so it doubles as the dedup key). `ts` is the
32
+ // client epoch-ms timestamp the signature commits to; `now` is the
33
+ // server clock (injectable for tests).
34
+ admit: (signature: string, ts: number, now?: number) => FetchMintVerdict
35
+ size: () => number
36
+ }
37
+
38
+ export const DEFAULT_FETCH_MINT_WINDOW_MS = 60_000
39
+ export const DEFAULT_FETCH_MINT_MAX_ENTRIES = 50_000
40
+
41
+ export function createFetchMintGuard(
42
+ { windowMs = DEFAULT_FETCH_MINT_WINDOW_MS, maxEntries = DEFAULT_FETCH_MINT_MAX_ENTRIES }:
43
+ { windowMs?: number; maxEntries?: number } = {},
44
+ ): FetchMintGuard {
45
+ // signature → expiry (ms). Insertion-ordered; uniform TTL ⇒ the head is
46
+ // always the soonest to expire.
47
+ const seen = new Map<string, number>()
48
+
49
+ function admit(signature: string, ts: number, now: number = Date.now()): FetchMintVerdict {
50
+ // Freshness first — a stale/future request never touches the cache, so
51
+ // it can't be used to grow the set.
52
+ if (!Number.isFinite(ts) || Math.abs(now - ts) > windowMs) return 'stale'
53
+ // Front-prune expired entries (contiguous at the head under the
54
+ // uniform TTL). Breaks at the first live entry.
55
+ for (const [key, exp] of seen) {
56
+ if (exp > now) break
57
+ seen.delete(key)
58
+ }
59
+ // Post-prune, any remaining entry is live, so a hit is a genuine replay.
60
+ if (seen.has(signature)) return 'replay'
61
+ seen.set(signature, now + windowMs)
62
+ // Flood backstop: drop the oldest beyond the cap. Those are the
63
+ // closest to expiry anyway; dropping them only shortens their dedup
64
+ // window (still freshness-gated).
65
+ while (seen.size > maxEntries) {
66
+ const oldest = seen.keys().next().value
67
+ if (oldest === undefined) break
68
+ seen.delete(oldest)
69
+ }
70
+ return 'ok'
71
+ }
72
+
73
+ return { admit, size: () => seen.size }
74
+ }