@preventive/triage 1.0.0-alpha.17 → 1.0.0-alpha.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api/reap.ts +13 -75
- package/common/server-info.ts +6 -4
- package/out/client-managed-import.js +81 -0
- package/out/client-managed.js +68 -41
- package/out/client-sync.js +14 -14
- package/out/graph.js +10 -10
- package/out/index.html +1 -0
- package/out/view.css +1 -1
- package/out/view.js +179 -142
- package/package.json +16 -6
- package/report/index.js +2 -1
- package/report/src/finding.js +12 -0
- package/report/src/security.js +63 -0
- package/server-common/database-config.ts +16 -0
- package/server-common/initialize.ts +18 -0
- package/server-common/npm-advisories.ts +101 -0
- package/server-common/reap.ts +48 -0
- package/server-common/standalone.ts +29 -0
- package/server-common/storage-log.ts +34 -0
- package/server-common/vercel-blob.ts +110 -0
- package/server-e2e/app.ts +485 -0
- package/server-e2e/cli.js +7 -2
- package/server-e2e/config.ts +36 -40
- package/server-e2e/db-neon.ts +1 -1
- package/server-e2e/db.ts +1 -1
- package/server-e2e/http.ts +12 -2
- package/server-e2e/index.ts +12 -481
- package/server-e2e/lifecycle.ts +59 -172
- package/server-e2e/neon-driver.ts +2 -2
- package/server-e2e/npm-proxy.ts +10 -143
- package/server-e2e/objstore/blob-vercel.ts +4 -108
- package/server-e2e/objstore/init.ts +15 -23
- package/server-e2e/objstore/store-neon.ts +5 -9
- package/server-e2e/static.ts +11 -17
- package/out/provider-icons/claude.svg +0 -1
- package/out/provider-icons/deepseek.svg +0 -1
- package/out/provider-icons/google.svg +0 -1
- package/out/provider-icons/grok.svg +0 -1
- package/out/provider-icons/moonshot.svg +0 -1
- package/out/provider-icons/nvidia.svg +0 -1
- package/out/provider-icons/openai.svg +0 -1
- package/out/provider-icons/openrouter.svg +0 -1
- package/out/provider-icons/qwen.svg +0 -1
- package/out/provider-icons/zai.svg +0 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@preventive/triage",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.19",
|
|
4
4
|
"description": "Client & relay server for triaging of automated reports",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": {
|
|
@@ -40,6 +40,7 @@
|
|
|
40
40
|
},
|
|
41
41
|
"files": [
|
|
42
42
|
"api/reap.ts",
|
|
43
|
+
"server-e2e/app.ts",
|
|
43
44
|
"server-e2e/auth.ts",
|
|
44
45
|
"server-e2e/bus-receiver.ts",
|
|
45
46
|
"server-e2e/cli.js",
|
|
@@ -80,11 +81,19 @@
|
|
|
80
81
|
"server-e2e/validation.ts",
|
|
81
82
|
"server-e2e/ws-server.ts",
|
|
82
83
|
"server-e2e/config.example.json",
|
|
84
|
+
"server-common/database-config.ts",
|
|
85
|
+
"server-common/initialize.ts",
|
|
86
|
+
"server-common/reap.ts",
|
|
87
|
+
"server-common/npm-advisories.ts",
|
|
83
88
|
"server-common/origin.ts",
|
|
89
|
+
"server-common/standalone.ts",
|
|
90
|
+
"server-common/storage-log.ts",
|
|
91
|
+
"server-common/vercel-blob.ts",
|
|
84
92
|
"server-common/scan-config.ts",
|
|
85
93
|
"report/index.js",
|
|
86
94
|
"report/src/finding-id.js",
|
|
87
95
|
"report/src/finding.js",
|
|
96
|
+
"report/src/security.js",
|
|
88
97
|
"report/src/labels.js",
|
|
89
98
|
"report/src/md-structure.js",
|
|
90
99
|
"report/src/md-text.js",
|
|
@@ -109,6 +118,7 @@
|
|
|
109
118
|
"common/utf8.js",
|
|
110
119
|
"out/brotli-fallback.js",
|
|
111
120
|
"out/client-managed.js",
|
|
121
|
+
"out/client-managed-import.js",
|
|
112
122
|
"out/client-sync.js",
|
|
113
123
|
"out/graph.js",
|
|
114
124
|
"out/icon-maskable.svg",
|
|
@@ -116,7 +126,6 @@
|
|
|
116
126
|
"out/index.html",
|
|
117
127
|
"out/manifest.webmanifest",
|
|
118
128
|
"out/prism.js",
|
|
119
|
-
"out/provider-icons",
|
|
120
129
|
"out/stasis.svg",
|
|
121
130
|
"out/terminal.js",
|
|
122
131
|
"out/view.css",
|
|
@@ -126,20 +135,20 @@
|
|
|
126
135
|
],
|
|
127
136
|
"scripts": {
|
|
128
137
|
"lint": "node --run lint:js && node --run lint:css && node --run lint:ts",
|
|
129
|
-
"lint:js": "oxlint -c .oxlintrc.json --ignore-pattern=ui/logs.js ./tests/ ./common/ ./report/ ./server-common/ ./server-e2e/ ./server-managed/ ./client/ ./ui/ ./api/",
|
|
138
|
+
"lint:js": "oxlint -c .oxlintrc.json --ignore-pattern=ui/logs.js ./cli.js ./server.ts ./tests/ ./common/ ./report/ ./server-common/ ./server-e2e/ ./server-managed/ ./client/ ./ui/ ./api/",
|
|
130
139
|
"lint:css": "stylelint 'ui/**/*.css'",
|
|
131
140
|
"lint:ts": "tsc --noEmit",
|
|
132
141
|
"prepack": "node --run build",
|
|
133
142
|
"build": "node build.js build",
|
|
134
143
|
"serve": "node build.js serve",
|
|
135
|
-
"server": "node server-e2e
|
|
144
|
+
"server": "node --run server-e2e --",
|
|
145
|
+
"server-e2e": "node server-e2e/index.ts",
|
|
136
146
|
"server-managed": "node server-managed/index.ts",
|
|
137
147
|
"server-managed-test": "node server-managed/test-server.ts",
|
|
138
|
-
"test": "node --js-base-64 --experimental-test-module-mocks --test-timeout=120000 --test ./tests/*.test.js ./report/tests/*.test.js"
|
|
148
|
+
"test": "node --js-base-64 --import=./tests/_resource-loader.js --experimental-test-module-mocks --test-timeout=120000 --test ./tests/*.test.js ./report/tests/*.test.js"
|
|
139
149
|
},
|
|
140
150
|
"devDependencies": {
|
|
141
151
|
"@electric-sql/pglite": "^0.5.5",
|
|
142
|
-
"@exodus/stasis-core": "^1.0.0-beta.4",
|
|
143
152
|
"@noble/ciphers": "^2.3.0",
|
|
144
153
|
"@preventive/diff": "^1.0.0",
|
|
145
154
|
"@preventive/terminal": "2.0.0-beta.1",
|
|
@@ -158,6 +167,7 @@
|
|
|
158
167
|
"typescript": "^7.0.2"
|
|
159
168
|
},
|
|
160
169
|
"dependencies": {
|
|
170
|
+
"@exodus/stasis-core": "^1.0.0-beta.4",
|
|
161
171
|
"ws": "^8.20.1"
|
|
162
172
|
},
|
|
163
173
|
"peerDependencies": {
|
package/report/index.js
CHANGED
|
@@ -89,6 +89,7 @@ import { deriveFindingId } from './src/finding-id.js'
|
|
|
89
89
|
export { parseCodexCsvToScans } from './src/parse-codex.js'
|
|
90
90
|
export { computeFileHash, deriveFindingId, findingId } from './src/finding-id.js'
|
|
91
91
|
export { META_FIELDS, inheritReportMeta, repoDirectory, reportRepoGithub } from './src/meta.js'
|
|
92
|
+
export { stampSecurityGroups } from './src/security.js'
|
|
92
93
|
// The writing side: the document writer, and the label tables it
|
|
93
94
|
// spells the app's enumerations with, for the viewer's surfaces that
|
|
94
95
|
// describe the same things in prose.
|
|
@@ -104,7 +105,7 @@ export { COLOR_LABELS, SEVERITY_LABELS, SOURCE_LABELS, TRIAGE_LABELS, severityLa
|
|
|
104
105
|
export {
|
|
105
106
|
REVALIDATE_KINDS, SEVERITIES, SEVERITY_ORDER, correctedVariants, descriptionSections,
|
|
106
107
|
displayedSeverity, effectiveSeverity, evidenceNote, findingDisplayName, findingTitle,
|
|
107
|
-
firstLine, hasSeverityCorrection, isAppFinding, locationLabel, prettyModel, revalidateKindOf,
|
|
108
|
+
firstLine, hasSeverityCorrection, isAppFinding, isSecurityFinding, locationLabel, prettyModel, revalidateKindOf,
|
|
108
109
|
runMetaLine, splitDescription, stripExportMarker, titledDescription,
|
|
109
110
|
} from './src/finding.js'
|
|
110
111
|
|
package/report/src/finding.js
CHANGED
|
@@ -105,6 +105,18 @@ export function isAppFinding(f, source = f?.source) {
|
|
|
105
105
|
return Object.hasOwn(SOURCE_LABELS, source) || revalidateKindOf(f) === 'revalidation'
|
|
106
106
|
}
|
|
107
107
|
|
|
108
|
+
// Security involvement is independent of the App/code layer and the displayed
|
|
109
|
+
// (possibly corrected) severity. External reports use their original severity,
|
|
110
|
+
// not the synthetic report-level `type: security` supplied by their parsers.
|
|
111
|
+
// `security: false` opts out of the analyzer default, not the other OR rules.
|
|
112
|
+
export function isSecurityFinding(f, source = f?._source ?? f?.source) {
|
|
113
|
+
const external = typeof source === 'string' && source !== '' && source !== 'deepview'
|
|
114
|
+
const analyzer = external ? source : f?.analyzer ?? f?.type ?? f?._analyzer
|
|
115
|
+
return f?.security === true
|
|
116
|
+
|| ((analyzer === 'security' || analyzer === 'dependencies') && f?.security !== false)
|
|
117
|
+
|| (external && ['critical', 'high', 'medium', 'low'].includes(f?.severity))
|
|
118
|
+
}
|
|
119
|
+
|
|
108
120
|
// ── Run meta ─────────────────────────────────────────────────────────
|
|
109
121
|
export function prettyModel(model) {
|
|
110
122
|
if (!model) return model
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { isSecurityFinding } from './finding.js'
|
|
2
|
+
|
|
3
|
+
function securityGraph() {
|
|
4
|
+
const ids = [], nodes = new Map()
|
|
5
|
+
const nodeFor = (key) => {
|
|
6
|
+
if (!nodes.has(key)) {
|
|
7
|
+
const node = { security: false }
|
|
8
|
+
node.parent = node
|
|
9
|
+
nodes.set(key, node)
|
|
10
|
+
if (typeof key === 'string') ids.push(key)
|
|
11
|
+
}
|
|
12
|
+
return nodes.get(key)
|
|
13
|
+
}
|
|
14
|
+
const root = (node) => {
|
|
15
|
+
let current = node
|
|
16
|
+
while (current.parent !== current) current = current.parent
|
|
17
|
+
while (node.parent !== node) { const next = node.parent; node.parent = current; node = next }
|
|
18
|
+
return current
|
|
19
|
+
}
|
|
20
|
+
const join = (a, b) => {
|
|
21
|
+
a = root(a); b = root(b)
|
|
22
|
+
if (a !== b) { b.parent = a; a.security ||= b.security }
|
|
23
|
+
}
|
|
24
|
+
return { nodeFor, root, join, ids }
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// Stamp the complete row/link component, before any display lens hides tabs.
|
|
28
|
+
// The callbacks are synchronous lookups of already-known links and index rows;
|
|
29
|
+
// this never loads reports. Only the supplied findings are mutated. Recompute
|
|
30
|
+
// from intrinsic evidence so removing a sibling/link does not latch a true flag.
|
|
31
|
+
export function stampSecurityGroups(groups, { source, linkedIds = () => [], knownRows = () => [] } = {}) {
|
|
32
|
+
const { nodeFor, root, join, ids } = securityGraph()
|
|
33
|
+
const findings = []
|
|
34
|
+
const addRow = (group, known = false) => {
|
|
35
|
+
let first
|
|
36
|
+
for (const f of group) {
|
|
37
|
+
if (!f || typeof f !== 'object') continue
|
|
38
|
+
const node = nodeFor(f.id || f)
|
|
39
|
+
root(node).security ||= known ? f.isSecurity === true : isSecurityFinding(f, f._source ?? f.source ?? source)
|
|
40
|
+
if (first) join(first, node)
|
|
41
|
+
else first = node
|
|
42
|
+
if (!known) findings.push([f, node])
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
for (const group of groups) addRow(group)
|
|
46
|
+
const seenRows = new Set()
|
|
47
|
+
// Growing queue: a known linked row can have siblings with links of their own.
|
|
48
|
+
for (let i = 0; i < ids.length; i++) {
|
|
49
|
+
const id = ids[i], node = nodeFor(id)
|
|
50
|
+
for (const other of linkedIds(id)) join(node, nodeFor(other))
|
|
51
|
+
for (const row of knownRows([id])) {
|
|
52
|
+
if (seenRows.has(row)) continue
|
|
53
|
+
seenRows.add(row)
|
|
54
|
+
addRow(row.members, true)
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
let changed = false
|
|
58
|
+
for (const [f, node] of findings) {
|
|
59
|
+
const security = root(node).security
|
|
60
|
+
if (f.isSecurity !== security) { f.isSecurity = security; changed = true }
|
|
61
|
+
}
|
|
62
|
+
return changed
|
|
63
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { env } from 'node:process'
|
|
2
|
+
|
|
3
|
+
// One shared URL or per-mode URLs. Validate before opening either store so
|
|
4
|
+
// an ambiguous configuration cannot send application traffic and GC elsewhere.
|
|
5
|
+
export function databaseUrls({ combined = false } = {}) {
|
|
6
|
+
const shared = env['DATABASE_URL'] || null
|
|
7
|
+
const e2e = env['E2E_DATABASE_URL'] || null
|
|
8
|
+
const managed = env['MANAGED_DATABASE_URL'] || null
|
|
9
|
+
if (shared && (e2e || managed)) {
|
|
10
|
+
throw new Error('DATABASE_URL cannot be combined with E2E_DATABASE_URL or MANAGED_DATABASE_URL. Use the shared URL alone, or only the per-mode URLs.')
|
|
11
|
+
}
|
|
12
|
+
if (combined && Boolean(e2e) !== Boolean(managed)) {
|
|
13
|
+
throw new Error('Combined mode requires both E2E_DATABASE_URL and MANAGED_DATABASE_URL, DATABASE_URL alone, or no database URLs. Mixing Neon and SQLite is not supported.')
|
|
14
|
+
}
|
|
15
|
+
return { e2e: shared ?? e2e, managed: shared ?? managed }
|
|
16
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// Register resources as they are acquired. Failed assembly rolls them back in
|
|
2
|
+
// reverse order; successful assembly transfers ownership to the returned app.
|
|
3
|
+
export async function initializeApp<T>(assemble: (rollback: AsyncDisposableStack) => T | Promise<T>): Promise<T> {
|
|
4
|
+
const rollback = new AsyncDisposableStack()
|
|
5
|
+
try {
|
|
6
|
+
const app = await assemble(rollback)
|
|
7
|
+
rollback.move()
|
|
8
|
+
return app
|
|
9
|
+
} catch (error) {
|
|
10
|
+
try { await rollback.disposeAsync() }
|
|
11
|
+
catch (cleanupError) {
|
|
12
|
+
// AsyncDisposableStack runs every cleanup even if one fails. Preserve
|
|
13
|
+
// both failures so cleanup errors cannot hide the initialization error.
|
|
14
|
+
throw new SuppressedError(cleanupError, error, 'App initialization and cleanup failed')
|
|
15
|
+
}
|
|
16
|
+
throw error
|
|
17
|
+
}
|
|
18
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer'
|
|
2
|
+
|
|
3
|
+
const UPSTREAM_URL = 'https://registry.npmjs.org/-/npm/v1/security/advisories/bulk'
|
|
4
|
+
const RESPONSE_BODY_LIMIT = 4 * 1024 * 1024
|
|
5
|
+
export const NPM_ADVISORIES_TIMEOUT_MS = 30_000
|
|
6
|
+
|
|
7
|
+
export async function fetchNpmAdvisories(body: Buffer, signal: AbortSignal, debug = false): Promise<{ status: number; body: unknown }> {
|
|
8
|
+
let upstream: Response
|
|
9
|
+
try {
|
|
10
|
+
upstream = await fetch(UPSTREAM_URL, {
|
|
11
|
+
method: 'POST',
|
|
12
|
+
// Force JSON — the bulk endpoint requires it. Drop every
|
|
13
|
+
// client-supplied header to keep an upstream fingerprint from
|
|
14
|
+
// leaking through (cookies, auth, custom UA, ...). The
|
|
15
|
+
// registry's bulk endpoint doesn't need any of them for a
|
|
16
|
+
// public lookup.
|
|
17
|
+
headers: { 'content-type': 'application/json', 'accept': 'application/json' },
|
|
18
|
+
// Re-wrap as a plain Uint8Array — Buffer's underlying
|
|
19
|
+
// ArrayBufferLike type doesn't satisfy fetch's BodyInit
|
|
20
|
+
// narrowing (it can't statically rule out SharedArrayBuffer),
|
|
21
|
+
// but a copy through Uint8Array is zero-cost in practice and
|
|
22
|
+
// unambiguously typed.
|
|
23
|
+
body: new Uint8Array(body),
|
|
24
|
+
signal,
|
|
25
|
+
})
|
|
26
|
+
} catch (err: unknown) {
|
|
27
|
+
if (debug) console.warn('npm-advisories upstream error:', err)
|
|
28
|
+
return { status: 502, body: { error: 'upstream-unreachable' } }
|
|
29
|
+
}
|
|
30
|
+
// Assert JSON on the upstream body. The Content-Type header is
|
|
31
|
+
// unreliable (Cloudflare in front of registry.npmjs.org strips it
|
|
32
|
+
// from some responses; a captive portal / WAF can declare HTML on
|
|
33
|
+
// a body that's actually JSON or vice-versa), so we don't lean on
|
|
34
|
+
// it — instead we buffer the body and parse. A successful
|
|
35
|
+
// JSON.parse is the strongest guarantee we can hand the UI's
|
|
36
|
+
// `await res.json()`. Buffering is bounded by
|
|
37
|
+
// `RESPONSE_BODY_LIMIT`; the advisories endpoint's payloads sit
|
|
38
|
+
// well under that.
|
|
39
|
+
const upstreamContentType = upstream.headers.get('content-type') ?? ''
|
|
40
|
+
let buffered: Buffer | null
|
|
41
|
+
try {
|
|
42
|
+
buffered = await readUpstreamBody(upstream)
|
|
43
|
+
} catch (err: unknown) {
|
|
44
|
+
if (debug) console.warn('npm-advisories upstream body error:', err)
|
|
45
|
+
return { status: 502, body: { error: 'upstream-unreachable' } }
|
|
46
|
+
}
|
|
47
|
+
if (buffered === null) {
|
|
48
|
+
if (debug) console.warn(`npm-advisories upstream too large: status=${upstream.status}`)
|
|
49
|
+
return { status: 502, body: { error: 'upstream-too-large', upstreamStatus: upstream.status } }
|
|
50
|
+
}
|
|
51
|
+
// Treat the body as UTF-8 — `JSON.parse` operates on a string and
|
|
52
|
+
// the registry's responses are always UTF-8 in practice. A
|
|
53
|
+
// non-UTF-8 byte sequence still decodes (with U+FFFD
|
|
54
|
+
// substitution); the subsequent JSON.parse fails and routes
|
|
55
|
+
// through the error branch.
|
|
56
|
+
const text = buffered.toString('utf8')
|
|
57
|
+
let parsed: unknown
|
|
58
|
+
try {
|
|
59
|
+
parsed = JSON.parse(text)
|
|
60
|
+
} catch {
|
|
61
|
+
if (debug) console.warn(`npm-advisories upstream non-JSON: status=${upstream.status} ct=${upstreamContentType || '<none>'} bytes=${buffered.byteLength}`)
|
|
62
|
+
return { status: 502, body: {
|
|
63
|
+
error: 'upstream-not-json',
|
|
64
|
+
upstreamStatus: upstream.status,
|
|
65
|
+
upstreamContentType: upstreamContentType || null,
|
|
66
|
+
} }
|
|
67
|
+
}
|
|
68
|
+
return { status: upstream.status, body: parsed }
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Buffer the upstream response body up to RESPONSE_BODY_LIMIT.
|
|
72
|
+
// Returns null if the cap is exceeded (caller maps to a 502
|
|
73
|
+
// `upstream-too-large`), or the cumulative Buffer otherwise. A
|
|
74
|
+
// transport error mid-read (e.g. AbortSignal fired by the deadline
|
|
75
|
+
// timer or by `req` close) throws — the caller catches it.
|
|
76
|
+
//
|
|
77
|
+
// `finally { reader.cancel() }` is load-bearing on the error and
|
|
78
|
+
// cap-exceeded paths: leaving the reader locked to the body holds
|
|
79
|
+
// the underlying undici TCP socket out of the connection pool until
|
|
80
|
+
// GC, and the cap-exceeded path explicitly needs to tear the
|
|
81
|
+
// transfer down so we don't keep buffering bytes we'll never use.
|
|
82
|
+
// On the clean-drain path (done:true), cancel() is a no-op.
|
|
83
|
+
async function readUpstreamBody(upstream: Response): Promise<Buffer | null> {
|
|
84
|
+
if (!upstream.body) return Buffer.alloc(0)
|
|
85
|
+
const reader = upstream.body.getReader()
|
|
86
|
+
const chunks: Uint8Array[] = []
|
|
87
|
+
let received = 0
|
|
88
|
+
try {
|
|
89
|
+
for (;;) {
|
|
90
|
+
const { done, value } = await reader.read()
|
|
91
|
+
if (done) break
|
|
92
|
+
if (!value) continue
|
|
93
|
+
received += value.byteLength
|
|
94
|
+
if (received > RESPONSE_BODY_LIMIT) return null
|
|
95
|
+
chunks.push(value)
|
|
96
|
+
}
|
|
97
|
+
return Buffer.concat(chunks)
|
|
98
|
+
} finally {
|
|
99
|
+
try { await reader.cancel() } catch {}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto'
|
|
2
|
+
import type { IncomingMessage, ServerResponse } from 'node:http'
|
|
3
|
+
|
|
4
|
+
type Handler = (req: IncomingMessage, res: ServerResponse) => void | Promise<void>
|
|
5
|
+
type Reapers = Record<string, () => Promise<unknown>>
|
|
6
|
+
type Options = { secret?: string; isShuttingDown?: () => boolean }
|
|
7
|
+
|
|
8
|
+
// Start every cleanup and wait for all of them, including when one fails.
|
|
9
|
+
// The caller must not close storage or return success with work still running.
|
|
10
|
+
export async function runReapers(reapers: Reapers): Promise<void> {
|
|
11
|
+
const results = await Promise.allSettled(Object.values(reapers).map(reap => Promise.resolve().then(reap)))
|
|
12
|
+
const errors = results.filter(result => result.status === 'rejected').map(result => result.reason)
|
|
13
|
+
if (errors.length > 0) throw new AggregateError(errors, 'Cleanup failed')
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function createReapHandler(reapers: Reapers, { secret = process.env['CRON_SECRET'], isShuttingDown = () => false }: Options = {}) {
|
|
17
|
+
const key = randomBytes(32)
|
|
18
|
+
const hash = (value: string) => createHmac('sha256', key).update(value).digest()
|
|
19
|
+
const expected = secret ? hash(`Bearer ${secret}`) : null
|
|
20
|
+
return async (req: IncomingMessage, res: ServerResponse): Promise<void> => {
|
|
21
|
+
const send = (status: number, body: object, headers = {}) => {
|
|
22
|
+
res.writeHead(status, { 'content-type': 'application/json', 'cache-control': 'no-store', ...headers })
|
|
23
|
+
res.end(JSON.stringify(body))
|
|
24
|
+
}
|
|
25
|
+
const authorization = req.headers.authorization
|
|
26
|
+
if (req.headers['x-deepview-share'] !== undefined) { send(403, { error: 'share-scope-required' }); return }
|
|
27
|
+
if (!expected || typeof authorization !== 'string' || !timingSafeEqual(hash(authorization), expected)) {
|
|
28
|
+
send(401, { error: 'unauthorized' }); return
|
|
29
|
+
}
|
|
30
|
+
if (req.method !== 'GET') { send(405, { error: 'method-not-allowed' }, { allow: 'GET' }); return }
|
|
31
|
+
if (isShuttingDown()) { send(503, { error: 'shutting-down' }); return }
|
|
32
|
+
const startedAt = Date.now()
|
|
33
|
+
try {
|
|
34
|
+
await runReapers(reapers)
|
|
35
|
+
send(200, { ok: true, reaped: Object.keys(reapers), ms: Date.now() - startedAt })
|
|
36
|
+
} catch (err) {
|
|
37
|
+
console.error('reap failed:', err)
|
|
38
|
+
send(500, { error: 'reap-failed' })
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// Only top-level servers compose this route. Domain routers stay unaware of
|
|
44
|
+
// other modes, and cleanup uses the stores those servers have already opened.
|
|
45
|
+
export function withReap(next: Handler, reapers: Reapers, options: Options = {}): Handler {
|
|
46
|
+
const reap = createReapHandler(reapers, options)
|
|
47
|
+
return (req, res) => req.url?.split('?', 1)[0] === '/api/reap' ? reap(req, res) : next(req, res)
|
|
48
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { Server } from 'node:http'
|
|
2
|
+
|
|
3
|
+
// Only CLI entry points own process signals and termination. Embedded servers
|
|
4
|
+
// expose async disposal and error events to their host instead.
|
|
5
|
+
export function startServer(server: Server, config: { port: number; host: string }): void {
|
|
6
|
+
let shuttingDown = false
|
|
7
|
+
let exitCode = 0
|
|
8
|
+
function shutdown(code: number): void {
|
|
9
|
+
if (code !== 0) exitCode = code
|
|
10
|
+
if (shuttingDown) return
|
|
11
|
+
shuttingDown = true
|
|
12
|
+
console.log('Shutting down…')
|
|
13
|
+
void server[Symbol.asyncDispose]().catch(err => {
|
|
14
|
+
console.error('Shutdown failed:', err)
|
|
15
|
+
exitCode = 1
|
|
16
|
+
}).finally(() => process.exit(exitCode))
|
|
17
|
+
}
|
|
18
|
+
function onSignal(): void { shutdown(0) }
|
|
19
|
+
function onError(err: unknown): void {
|
|
20
|
+
console.error('Server error:', err)
|
|
21
|
+
shutdown(1)
|
|
22
|
+
}
|
|
23
|
+
server.on('error', onError)
|
|
24
|
+
process.on('SIGINT', onSignal)
|
|
25
|
+
process.on('SIGTERM', onSignal)
|
|
26
|
+
process.on('unhandledRejection', onError)
|
|
27
|
+
process.on('uncaughtException', onError)
|
|
28
|
+
server.listen(config.port, config.host)
|
|
29
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { dirname, resolve } from 'node:path'
|
|
2
|
+
|
|
3
|
+
type DatabaseConfig = { dbPath: string; neonUrl?: string | null }
|
|
4
|
+
|
|
5
|
+
function databaseLocation({ dbPath, neonUrl }: DatabaseConfig): string {
|
|
6
|
+
if (!neonUrl) return `SQLite ${dbPath === ':memory:' ? dbPath : resolve(dbPath)}`
|
|
7
|
+
// Report the destination, never URL credentials or query-string secrets.
|
|
8
|
+
try {
|
|
9
|
+
const url = new URL(neonUrl)
|
|
10
|
+
return `Neon Postgres ${url.host}${url.pathname}`
|
|
11
|
+
} catch { return 'Neon Postgres' }
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function e2eStorageLines(config: DatabaseConfig & { objstoreDir: string }): string[] {
|
|
15
|
+
return [
|
|
16
|
+
` E2E database: ${databaseLocation(config)}`,
|
|
17
|
+
` E2E objects: ${config.neonUrl ? 'Vercel Blob (private), {workspaceTag}/' : resolve(config.objstoreDir)}`,
|
|
18
|
+
` E2E staging: ${config.neonUrl ? 'Vercel Blob (private), {workspaceTag}/.staging/' : resolve(config.objstoreDir, '{workspaceTag}', '.staging')}`,
|
|
19
|
+
]
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function managedStorageLines(config: DatabaseConfig): string[] {
|
|
23
|
+
const dir = dirname(config.dbPath)
|
|
24
|
+
const location = (path: string) => config.neonUrl ? `Vercel Blob (private), .managed/${path}/` : resolve(dir, path)
|
|
25
|
+
return [
|
|
26
|
+
` Managed database: ${databaseLocation(config)}`,
|
|
27
|
+
` Managed reports: ${location('reports')}`,
|
|
28
|
+
` Managed bundles: ${location('bundles')}`,
|
|
29
|
+
` Managed avatars: ${location('avatars')}`,
|
|
30
|
+
` Managed bundle cache: ${location('cache/bundles')}`,
|
|
31
|
+
` Managed report sources cache: ${location('cache/report-sources')}`,
|
|
32
|
+
...(config.neonUrl ? [` Managed upload parts: ${location('uploads')}`] : []),
|
|
33
|
+
]
|
|
34
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// Shared optional Vercel Blob SDK boundary for both server modes.
|
|
2
|
+
import type { Readable } from 'node:stream'
|
|
3
|
+
import type { Buffer } from 'node:buffer'
|
|
4
|
+
|
|
5
|
+
// Minimal structural shape of the bits of `@vercel/blob` we use.
|
|
6
|
+
// Shared so the optional peer dep doesn't have to type-resolve
|
|
7
|
+
// for SQLite-only deployments. Real type details live in the
|
|
8
|
+
// installed package; the fields/parameters we touch here are stable
|
|
9
|
+
// per the SDK v2 public surface.
|
|
10
|
+
//
|
|
11
|
+
// `put` accepts the SDK's full `PutBody` union (string | Readable |
|
|
12
|
+
// Buffer | Blob | ArrayBuffer | ReadableStream | File). We only ever
|
|
13
|
+
// pass a Node `PassThrough` (a Readable), but the wider type lets
|
|
14
|
+
// callers reuse this signature for future buffer/blob bodies without
|
|
15
|
+
// type gymnastics. `copy` accepts `allowOverwrite` — REQUIRED for
|
|
16
|
+
// version bumps since the live pathname is reused on re-upload.
|
|
17
|
+
|
|
18
|
+
type VercelBlobBody = Readable | Buffer | string | Blob | ArrayBuffer | ReadableStream<Uint8Array>
|
|
19
|
+
export type VercelBlobSdk = {
|
|
20
|
+
put: (
|
|
21
|
+
pathname: string,
|
|
22
|
+
body: VercelBlobBody,
|
|
23
|
+
options: {
|
|
24
|
+
access: 'private' | 'public'
|
|
25
|
+
addRandomSuffix?: boolean
|
|
26
|
+
allowOverwrite?: boolean
|
|
27
|
+
contentType?: string
|
|
28
|
+
token?: string
|
|
29
|
+
multipart?: boolean
|
|
30
|
+
abortSignal?: AbortSignal
|
|
31
|
+
cacheControlMaxAge?: number
|
|
32
|
+
},
|
|
33
|
+
) => Promise<{ url: string; pathname: string }>
|
|
34
|
+
head: (
|
|
35
|
+
pathname: string,
|
|
36
|
+
options?: { token?: string; abortSignal?: AbortSignal },
|
|
37
|
+
) => Promise<{ size: number; pathname: string; url: string }>
|
|
38
|
+
get: (
|
|
39
|
+
pathname: string,
|
|
40
|
+
options: {
|
|
41
|
+
access: 'private' | 'public'
|
|
42
|
+
token?: string
|
|
43
|
+
useCache?: boolean
|
|
44
|
+
abortSignal?: AbortSignal
|
|
45
|
+
},
|
|
46
|
+
) => Promise<{
|
|
47
|
+
statusCode: 200 | 304
|
|
48
|
+
stream: ReadableStream<Uint8Array> | null
|
|
49
|
+
blob: { size: number | null }
|
|
50
|
+
} | null>
|
|
51
|
+
copy: (
|
|
52
|
+
fromPathname: string,
|
|
53
|
+
toPathname: string,
|
|
54
|
+
options: {
|
|
55
|
+
access: 'private' | 'public'
|
|
56
|
+
addRandomSuffix?: boolean
|
|
57
|
+
allowOverwrite?: boolean
|
|
58
|
+
token?: string
|
|
59
|
+
contentType?: string
|
|
60
|
+
cacheControlMaxAge?: number
|
|
61
|
+
},
|
|
62
|
+
) => Promise<{ url: string; pathname: string }>
|
|
63
|
+
del: (
|
|
64
|
+
urlOrPathname: string | string[],
|
|
65
|
+
options?: { token?: string; abortSignal?: AbortSignal },
|
|
66
|
+
) => Promise<void>
|
|
67
|
+
list: (options: {
|
|
68
|
+
prefix?: string
|
|
69
|
+
cursor?: string
|
|
70
|
+
limit?: number
|
|
71
|
+
mode?: 'expanded' | 'folded'
|
|
72
|
+
token?: string
|
|
73
|
+
}) => Promise<{
|
|
74
|
+
// `uploadedAt` (a Date per the SDK v2 surface) is the blob's
|
|
75
|
+
// creation time — used by the reaper's GC grace window. Optional
|
|
76
|
+
// in the type because the `folded` listing path ignores it (only
|
|
77
|
+
// `listLiveBlobs` reads it, and it uses the default expanded mode).
|
|
78
|
+
blobs: Array<{ pathname: string; size: number; uploadedAt?: Date | string | number }>
|
|
79
|
+
folders?: string[]
|
|
80
|
+
cursor?: string
|
|
81
|
+
hasMore: boolean
|
|
82
|
+
}>
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Recognise "blob is gone" errors uniformly across read/write/delete
|
|
86
|
+
// paths so callers can treat them as success (delete) or
|
|
87
|
+
// not-found (read). The SDK exposes BlobNotFoundError as a class with
|
|
88
|
+
// `.name === 'BlobNotFoundError'`; checking the name string avoids
|
|
89
|
+
// importing the class at the top level (which would force the optional
|
|
90
|
+
// peer dep to resolve).
|
|
91
|
+
//
|
|
92
|
+
// Class-name check ONLY — the SDK's internal mapper turns every API
|
|
93
|
+
// `not_found` into BlobNotFoundError-by-name, so a bare-404 transport
|
|
94
|
+
// leak doesn't reach here. A broader `/does not exist|\b404\b/` match
|
|
95
|
+
// is DANGEROUS: it also matches BlobStoreNotFoundError's "This store
|
|
96
|
+
// does not exist.", so a config fault (revoked token, deleted store)
|
|
97
|
+
// would silently surface as every-blob-missing across reads/unlinks,
|
|
98
|
+
// masking the fatal misconfiguration. The tight name check lets
|
|
99
|
+
// BlobStoreNotFoundError / other classes propagate as real exceptions.
|
|
100
|
+
export function isNotFound(err: unknown): boolean {
|
|
101
|
+
if (err == null || typeof err !== 'object') return false
|
|
102
|
+
const name = (err as { name?: unknown }).name
|
|
103
|
+
return typeof name === 'string' && name === 'BlobNotFoundError'
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export async function loadVercelBlobSdk(): Promise<VercelBlobSdk> {
|
|
107
|
+
// @ts-ignore optional peer dep: '@vercel/blob'
|
|
108
|
+
const mod = (await import('@vercel/blob')) as VercelBlobSdk
|
|
109
|
+
return mod
|
|
110
|
+
}
|