@preventive/triage 1.0.0-alpha.3 → 1.0.0-alpha.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-APACHE +202 -0
- package/api/reap.ts +17 -0
- package/cli.js +6 -0
- package/client/finding-link.js +305 -0
- package/client/linked-findings.d.ts +1 -0
- package/client/linked-findings.js +111 -0
- package/common/bundle-commit.d.ts +1 -0
- package/common/bundle-commit.js +4 -0
- package/common/bundle-metadata.d.ts +10 -0
- package/common/bundle-metadata.js +182 -0
- package/common/bundle-reasons.d.ts +2 -0
- package/common/bundle-reasons.js +21 -0
- package/common/bundle-source-links.d.ts +5 -0
- package/common/bundle-source-links.js +82 -0
- package/common/bundle-sources.d.ts +3 -0
- package/common/bundle-sources.js +284 -0
- package/common/bundle-stats.js +41 -0
- package/common/bundle-tabs.js +1 -0
- package/common/code-language.js +51 -0
- package/common/default-scan-models.ts +30 -0
- package/common/finding-id.js +47 -0
- package/common/github-issue-labels.d.ts +1 -0
- package/common/github-issue-labels.js +11 -0
- package/common/github-pr.ts +56 -0
- package/common/managed/comments.ts +34 -0
- package/common/managed/permissions.ts +35 -0
- package/common/managed/report-content.ts +42 -0
- package/common/managed/report-filter.ts +108 -0
- package/common/managed/roles.ts +28 -0
- package/common/managed/routes.d.ts +2 -0
- package/common/managed/routes.js +124 -0
- package/common/managed/scan-models.ts +6 -0
- package/common/managed/triage.ts +83 -0
- package/common/save-error-reason.ts +20 -7
- package/common/scan-server.ts +13 -0
- package/common/server-info.ts +35 -0
- package/common/utf8.d.ts +3 -0
- package/common/utf8.js +45 -0
- package/out/brotli-fallback.js +3 -3
- package/out/client-managed-import.js +83 -0
- package/out/client-managed.js +169 -0
- package/out/client-sync.js +16 -13
- package/out/github-issue-authorized.html +3 -0
- package/out/github-issue-authorized.js +1 -0
- package/out/graph.js +30 -4
- package/out/index.html +55 -8
- package/out/prism.js +2 -2
- package/out/stasis.svg +20 -0
- package/out/terminal.js +273 -39
- package/out/view.css +1 -1
- package/out/view.js +259 -63
- package/package.json +205 -57
- package/server-common/database-config.ts +16 -0
- package/server-common/initialize.ts +18 -0
- package/server-common/npm-advisories.ts +101 -0
- package/{server → server-common}/origin.ts +5 -5
- package/server-common/reap.ts +48 -0
- package/server-common/scan-config.ts +19 -0
- package/server-common/standalone.ts +29 -0
- package/server-common/storage-crypto.ts +182 -0
- package/server-common/storage-log.ts +38 -0
- package/server-common/vercel-blob.ts +103 -0
- package/server-e2e/app.ts +486 -0
- package/{server → server-e2e}/auth.ts +5 -1
- package/{server → server-e2e}/bus-receiver.ts +2 -1
- package/{server → server-e2e}/cli.js +9 -4
- package/{server → server-e2e}/config.ts +56 -35
- package/{server → server-e2e}/db-neon.ts +2 -2
- package/{server → server-e2e}/db-revision-sql.ts +1 -1
- package/{server → server-e2e}/db-stmt.ts +2 -2
- package/{server → server-e2e}/db.ts +7 -7
- package/{server → server-e2e}/http.ts +110 -12
- package/{server → server-e2e}/hub.ts +40 -9
- package/server-e2e/index.ts +17 -0
- package/server-e2e/lifecycle.ts +95 -0
- package/{server → server-e2e}/neon-driver.ts +2 -2
- package/{server → server-e2e}/npm-proxy.ts +11 -144
- package/{server → server-e2e}/objstore/blob-fs.ts +2 -2
- package/{server → server-e2e}/objstore/blob-vercel.ts +13 -117
- package/{server → server-e2e}/objstore/blob.ts +9 -2
- package/server-e2e/objstore/fetch-mint-guard.ts +74 -0
- package/{server → server-e2e}/objstore/handlers.ts +9 -8
- package/{server → server-e2e}/objstore/init.ts +35 -18
- package/{server → server-e2e}/objstore/reaper.ts +31 -11
- package/server-e2e/objstore/rest-deny.ts +28 -0
- package/server-e2e/objstore/rest-mint.ts +224 -0
- package/{server → server-e2e}/objstore/rest.ts +58 -33
- package/{server → server-e2e}/objstore/sign.ts +105 -0
- package/{server → server-e2e}/objstore/store-neon.ts +6 -10
- package/{server → server-e2e}/objstore/store.ts +7 -7
- package/{server → server-e2e}/objstore/tokens.ts +1 -1
- package/{server → server-e2e}/pubsub.ts +12 -9
- package/{server → server-e2e}/sign.ts +4 -4
- package/{server → server-e2e}/sse-server.ts +102 -68
- package/{server → server-e2e}/sse-session.ts +28 -8
- package/{server → server-e2e}/static.ts +31 -21
- package/server-e2e/sync-handlers.ts +408 -0
- package/{server → server-e2e}/util.ts +9 -0
- package/{server → server-e2e}/ws-server.ts +20 -9
- package/server-managed/activity.ts +231 -0
- package/server-managed/avatar-store.ts +12 -0
- package/server-managed/blob-store.ts +17 -0
- package/server-managed/brotli.ts +10 -0
- package/server-managed/bundle-advisories.ts +53 -0
- package/server-managed/bundle-advisory-inventory.ts +75 -0
- package/server-managed/bundle-build-diagnostics.d.ts +10 -0
- package/server-managed/bundle-build-diagnostics.js +23 -0
- package/server-managed/bundle-build-leases.ts +41 -0
- package/server-managed/bundle-build-worker.js +54 -0
- package/server-managed/bundle-build.ts +168 -0
- package/server-managed/bundle-cache.ts +166 -0
- package/server-managed/bundle-catalog.ts +29 -0
- package/server-managed/bundle-store.ts +24 -0
- package/server-managed/bundle-summary-cache.ts +116 -0
- package/server-managed/bundle.ts +104 -0
- package/server-managed/cache-storage.ts +19 -0
- package/server-managed/cli.js +21 -0
- package/server-managed/combined.ts +46 -0
- package/server-managed/comments.ts +151 -0
- package/server-managed/config.ts +149 -0
- package/server-managed/content-access.ts +15 -0
- package/server-managed/crypto.ts +25 -0
- package/server-managed/db-methods.ts +1517 -0
- package/server-managed/db-neon.ts +277 -0
- package/server-managed/db-schema.ts +206 -0
- package/server-managed/db-table-names.ts +22 -0
- package/server-managed/db.ts +120 -0
- package/server-managed/github-app.ts +385 -0
- package/server-managed/github-errors.ts +49 -0
- package/server-managed/github-issue-oauth.ts +48 -0
- package/server-managed/github-issues.ts +152 -0
- package/server-managed/github-metadata.ts +98 -0
- package/server-managed/github-oauth.ts +199 -0
- package/server-managed/github-pulls.ts +227 -0
- package/server-managed/http-response.ts +18 -0
- package/server-managed/http.ts +2350 -0
- package/server-managed/import-triage.ts +51 -0
- package/server-managed/index.ts +175 -0
- package/server-managed/managed-issues.ts +47 -0
- package/server-managed/management-catalog.ts +45 -0
- package/server-managed/management.ts +86 -0
- package/server-managed/object-storage-disk.ts +144 -0
- package/server-managed/object-storage-vercel.ts +63 -0
- package/server-managed/object-storage.ts +71 -0
- package/server-managed/package-entry-points.ts +51 -0
- package/server-managed/public-workspace.ts +146 -0
- package/server-managed/repo-path.ts +21 -0
- package/server-managed/report-migration.ts +35 -0
- package/server-managed/report-query.ts +10 -0
- package/server-managed/report-response.ts +16 -0
- package/server-managed/report-sources.ts +166 -0
- package/server-managed/repository-browser.ts +139 -0
- package/server-managed/repository-discovery.ts +82 -0
- package/server-managed/repository-policy.ts +27 -0
- package/server-managed/revisions.ts +35 -0
- package/server-managed/session.ts +78 -0
- package/server-managed/slugs.ts +38 -0
- package/server-managed/solidity-entry-points.ts +69 -0
- package/server-managed/sql-postgres.ts +31 -0
- package/server-managed/sql.ts +72 -0
- package/server-managed/static.ts +28 -0
- package/server-managed/storage-command.ts +16 -0
- package/server-managed/storage-db.ts +175 -0
- package/server-managed/storage-encryption.ts +211 -0
- package/server-managed/storage-maintenance.ts +223 -0
- package/server-managed/storage-payload.ts +72 -0
- package/server-managed/storage-stores.ts +81 -0
- package/server-managed/storage.ts +58 -0
- package/server-managed/team-catalog.ts +7 -0
- package/server-managed/team-feed.ts +173 -0
- package/server-managed/team-reports.ts +219 -0
- package/server-managed/triage-response.ts +16 -0
- package/server-managed/uploads.ts +65 -0
- package/server-managed/workspace-shares.ts +162 -0
- package/server.ts +50 -0
- package/server/index.ts +0 -477
- package/server/lifecycle.ts +0 -204
- package/server/sync-handlers.ts +0 -320
- /package/{LICENSE → LICENSE-MIT} +0 -0
- /package/{server → server-e2e}/config.example.json +0 -0
- /package/{server → server-e2e}/objstore/fs.ts +0 -0
- /package/{server → server-e2e}/peer.ts +0 -0
- /package/{server → server-e2e}/validation.ts +0 -0
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer'
|
|
2
|
+
import { createHash, randomUUID } from 'node:crypto'
|
|
3
|
+
import { posix } from 'node:path'
|
|
4
|
+
import { Worker } from 'node:worker_threads'
|
|
5
|
+
import { BUNDLE_BUILD_TIMEOUT_MS, type BundleBuildLeaseStore } from './bundle-build-leases.ts'
|
|
6
|
+
import { type BundleBuildDiagnostic, type BundleBuildStage, bundleBuildDiagnostic } from './bundle-build-diagnostics.js'
|
|
7
|
+
|
|
8
|
+
export class BundleBuildError extends Error {
|
|
9
|
+
status: number
|
|
10
|
+
code: string
|
|
11
|
+
constructor(status: number, code: string) { super(code); this.status = status; this.code = code }
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface BundleBuildInput {
|
|
15
|
+
repoId: number
|
|
16
|
+
commit: string
|
|
17
|
+
entries: string[]
|
|
18
|
+
directory: string
|
|
19
|
+
options: { conditions?: string[]; mainFields?: string[]; metro?: boolean; platforms?: string[]; typescript?: boolean; jsx?: boolean }
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function path(value: unknown): value is string {
|
|
23
|
+
return typeof value === 'string' && value.length > 0 && value.length <= 1024 && !/[\\\p{Cc}]/u.test(value)
|
|
24
|
+
&& value.split('/').every(part => part !== '' && part !== '.' && part !== '..')
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function parseBundleBuild(value: unknown): BundleBuildInput {
|
|
28
|
+
const fail = (code = 'bad-request'): never => { throw new BundleBuildError(400, code) }
|
|
29
|
+
if (value == null || typeof value !== 'object' || Array.isArray(value)) return fail()
|
|
30
|
+
const { repoId, commit, entries, conditions } = value as Record<string, unknown>
|
|
31
|
+
if (typeof repoId !== 'number' || !Number.isSafeInteger(repoId) || repoId <= 0
|
|
32
|
+
|| typeof commit !== 'string' || !/^[a-f\d]{40}$/u.test(commit)
|
|
33
|
+
|| !Array.isArray(entries) || entries.length === 0 || entries.length > 100 || !entries.every(path)) return fail()
|
|
34
|
+
const selected = [...new Set(entries as string[])]
|
|
35
|
+
const scripts = selected.every(entry => /\.[mc]?[jt]s$/u.test(entry))
|
|
36
|
+
if (!scripts && !selected.every(entry => entry.endsWith('.sol'))) return fail('unsupported-entries')
|
|
37
|
+
const parts = posix.dirname(selected[0]!).split('/').filter(part => part !== '.')
|
|
38
|
+
for (const entry of selected) {
|
|
39
|
+
const parent = posix.dirname(entry).split('/')
|
|
40
|
+
while (parts.some((part, i) => parent[i] !== part)) parts.pop()
|
|
41
|
+
}
|
|
42
|
+
const options: BundleBuildInput['options'] = {}
|
|
43
|
+
if (scripts) {
|
|
44
|
+
if (conditions == null || typeof conditions !== 'object' || Array.isArray(conditions)) return fail('bad-conditions')
|
|
45
|
+
const { preset, conditions: names, platforms } = conditions as Record<string, unknown>
|
|
46
|
+
if (!['node', 'browser', 'metro'].includes(String(preset)) || !Array.isArray(names) || names.length > 16
|
|
47
|
+
|| !names.every(name => typeof name === 'string' && name.length > 0 && name.length <= 64
|
|
48
|
+
&& !/^\.|^\d+$|[\s,/\\\p{Cc}]/u.test(name) && !['default', 'import', 'require'].includes(name))) return fail('bad-conditions')
|
|
49
|
+
if (preset === 'metro') {
|
|
50
|
+
if (names.length !== 1 || names[0] !== 'react-native') return fail('metro-conditions')
|
|
51
|
+
if (!Array.isArray(platforms) || platforms.length === 0 || platforms.length > 2
|
|
52
|
+
|| !platforms.every(platform => ['ios', 'android'].includes(platform))) return fail('bad-conditions')
|
|
53
|
+
Object.assign(options, { metro: true, platforms: [...new Set(platforms)] })
|
|
54
|
+
} else {
|
|
55
|
+
options.conditions = [...new Set(names)]
|
|
56
|
+
if (preset === 'browser') options.mainFields = ['browser', 'module', 'main']
|
|
57
|
+
}
|
|
58
|
+
options.typescript = true
|
|
59
|
+
options.jsx = true
|
|
60
|
+
}
|
|
61
|
+
return { repoId, commit, entries: selected, directory: parts.join('/'), options }
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Same portable name and 255-character limit as Stasis's githubBundleFile
|
|
65
|
+
// (src/cmd/github-bundle.js), which is not a public package export.
|
|
66
|
+
export function githubBundleFilename(github: string, directory: string, commit: string): string {
|
|
67
|
+
const portable = (value: string) => value.replaceAll(/[^\w.-]/gu, '_')
|
|
68
|
+
const repo = portable(github.replace('/', '-'))
|
|
69
|
+
const tail = `.${portable(commit.slice(0, 7))}.stasis.code.br`
|
|
70
|
+
if (!directory) return `${repo}${tail}`
|
|
71
|
+
const room = 255 - `${repo}.${tail}`.length
|
|
72
|
+
let dir = portable(directory.replaceAll('/', '-'))
|
|
73
|
+
if (dir.length > room) dir = `${dir.slice(0, room - 9)}_${createHash('sha256').update(directory).digest('hex').slice(0, 8)}`
|
|
74
|
+
return `${repo}.${dir}${tail}`
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export interface BuiltBundle { bytes: Uint8Array; directory: string; filename: string }
|
|
78
|
+
export interface BuildRequest { input: BundleBuildInput; github: string; token: string | null; maxBytes: number; scopes: (string | null)[] }
|
|
79
|
+
// An extra per-process ceiling; admission must also hold the shared DB lease.
|
|
80
|
+
const active = new Set<string>()
|
|
81
|
+
|
|
82
|
+
export async function withBundleBuildLease<T>(db: BundleBuildLeaseStore, userId: string, signal: AbortSignal,
|
|
83
|
+
work: (signal: AbortSignal) => Promise<T>): Promise<T> {
|
|
84
|
+
signal.throwIfAborted()
|
|
85
|
+
const owner = randomUUID(), started = performance.now(), timeout = new AbortController()
|
|
86
|
+
const expired = () => timeout.abort(new BundleBuildError(504, 'build-timeout'))
|
|
87
|
+
const timer = setTimeout(expired, BUNDLE_BUILD_TIMEOUT_MS)
|
|
88
|
+
const combined = AbortSignal.any([signal, timeout.signal])
|
|
89
|
+
let claimed = false
|
|
90
|
+
try {
|
|
91
|
+
claimed = await db.claimBundleBuildLease(userId, owner)
|
|
92
|
+
if (!claimed) throw new BundleBuildError(429, 'build-busy')
|
|
93
|
+
// Account for a slow/lost claim response or a suspended invocation before
|
|
94
|
+
// starting work. The three-minute budget starts before lease acquisition.
|
|
95
|
+
if (performance.now() - started >= BUNDLE_BUILD_TIMEOUT_MS) expired()
|
|
96
|
+
combined.throwIfAborted()
|
|
97
|
+
return await work(combined)
|
|
98
|
+
} catch (error) {
|
|
99
|
+
if (timeout.signal.aborted && !signal.aborted) throw timeout.signal.reason
|
|
100
|
+
throw error
|
|
101
|
+
} finally {
|
|
102
|
+
clearTimeout(timer)
|
|
103
|
+
// buildRepositoryBundle waits for worker termination before settling.
|
|
104
|
+
// Unknown claim/release outcomes expire safely without starting more work.
|
|
105
|
+
if (claimed) await db.releaseBundleBuildLease(owner)
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// Parsing repositories and compressing bundles stays off the HTTP event loop.
|
|
110
|
+
// Cap concurrent builds, terminate disconnected/timed-out work, and share no
|
|
111
|
+
// credentials or mutable Stasis state between requests.
|
|
112
|
+
export async function buildRepositoryBundle(userId: string, request: BuildRequest, signal: AbortSignal): Promise<BuiltBundle> {
|
|
113
|
+
if (active.has(userId) || active.size >= 2) throw new BundleBuildError(429, 'build-busy')
|
|
114
|
+
signal.throwIfAborted()
|
|
115
|
+
active.add(userId)
|
|
116
|
+
let worker: Worker | undefined
|
|
117
|
+
let workerUrl: URL | undefined
|
|
118
|
+
let stage: BundleBuildStage = 'worker-start'
|
|
119
|
+
const buildId = randomUUID(), started = performance.now()
|
|
120
|
+
const context = { buildId, repoId: request.input.repoId, github: request.github, commit: request.input.commit,
|
|
121
|
+
directory: request.input.directory, entryCount: request.input.entries.length, node: process.version }
|
|
122
|
+
const log = (event: string, details: Record<string, unknown> = {}) => {
|
|
123
|
+
const write = event === 'failed' ? console.error : console.info
|
|
124
|
+
write('managed-bundle-build:', JSON.stringify({ ...context, workerUrl: workerUrl?.href, event, stage, elapsedMs: Math.round(performance.now() - started), ...details }))
|
|
125
|
+
}
|
|
126
|
+
try {
|
|
127
|
+
workerUrl = new URL('./bundle-build-worker.js', import.meta.url)
|
|
128
|
+
log('started')
|
|
129
|
+
worker = new Worker(workerUrl, {
|
|
130
|
+
workerData: { ...request, type: 'managed-bundle-build' }, env: {}, execArgv: [], resourceLimits: { maxOldGenerationSizeMb: 512 },
|
|
131
|
+
})
|
|
132
|
+
const running = worker
|
|
133
|
+
return await new Promise((resolve, reject) => {
|
|
134
|
+
let settled = false
|
|
135
|
+
const cleanup = () => { clearTimeout(timer); signal.removeEventListener('abort', abort) }
|
|
136
|
+
const fail = (status: number, code: string, details: Record<string, unknown> = {}) => {
|
|
137
|
+
if (settled) return
|
|
138
|
+
settled = true
|
|
139
|
+
cleanup()
|
|
140
|
+
log(code === 'build-cancelled' ? 'cancelled' : 'failed', { code, ...details })
|
|
141
|
+
reject(new BundleBuildError(status, code))
|
|
142
|
+
}
|
|
143
|
+
const timer = setTimeout(() => fail(504, 'build-timeout'), BUNDLE_BUILD_TIMEOUT_MS)
|
|
144
|
+
const abort = () => fail(signal.reason instanceof BundleBuildError ? signal.reason.status : 499,
|
|
145
|
+
signal.reason instanceof BundleBuildError ? signal.reason.code : 'build-cancelled')
|
|
146
|
+
signal.addEventListener('abort', abort, { once: true })
|
|
147
|
+
running.on('message', (message: BuiltBundle & { type?: string; stage?: BundleBuildStage; error?: string; status?: number; diagnostic?: BundleBuildDiagnostic }) => {
|
|
148
|
+
if (settled) return
|
|
149
|
+
if (message.type === 'progress' && message.stage) { stage = message.stage; log('progress'); return }
|
|
150
|
+
if (message.error) { fail(message.status ?? 422, message.error, { diagnostic: message.diagnostic }); return }
|
|
151
|
+
settled = true
|
|
152
|
+
cleanup()
|
|
153
|
+
log('completed', { byteSize: message.bytes.length })
|
|
154
|
+
resolve({ bytes: Buffer.from(message.bytes), directory: message.directory, filename: message.filename })
|
|
155
|
+
})
|
|
156
|
+
running.once('error', error => fail(422, 'build-failed', { diagnostic: bundleBuildDiagnostic(error, request.token) }))
|
|
157
|
+
running.once('exit', exitCode => fail(422, 'build-failed', { exitCode }))
|
|
158
|
+
if (signal.aborted) abort()
|
|
159
|
+
})
|
|
160
|
+
} catch (error) {
|
|
161
|
+
if (!(error instanceof BundleBuildError)) {
|
|
162
|
+
log('failed', { code: 'build-failed', diagnostic: bundleBuildDiagnostic(error, request.token) })
|
|
163
|
+
}
|
|
164
|
+
throw error
|
|
165
|
+
} finally {
|
|
166
|
+
try { await worker?.terminate() } finally { active.delete(userId) }
|
|
167
|
+
}
|
|
168
|
+
}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
// Metadata is cached as Brotli. Contents use the stored Brotli bytes directly:
|
|
2
|
+
// unchanged Stasis uploads or sourcemaps compressed once at upload.
|
|
3
|
+
import { Buffer } from 'node:buffer'
|
|
4
|
+
import { brotliDecompress } from 'node:zlib'
|
|
5
|
+
import { promisify } from 'node:util'
|
|
6
|
+
import { BUNDLE_METADATA_VERSION, type BundleDetails, createBundleMetadata, createBundleSummary, parseBundleContents } from '../common/bundle-metadata.js'
|
|
7
|
+
import { bundleReasons } from '../common/bundle-reasons.js'
|
|
8
|
+
import { type BundleAdvisoryInventory, bundleAdvisoryInventory } from './bundle-advisory-inventory.ts'
|
|
9
|
+
import { decodeUtf8 } from '../common/utf8.js'
|
|
10
|
+
import type { OpenedBlob } from './blob-store.ts'
|
|
11
|
+
import type { BundleStore } from './bundle-store.ts'
|
|
12
|
+
import { encodeBrotli } from './brotli.ts'
|
|
13
|
+
import type { ManagedBundle, ManagedDb } from './db.ts'
|
|
14
|
+
import { CacheMissError } from './cache-storage.ts'
|
|
15
|
+
import { SUMMARY_FILENAME, createBundleSummaryCache } from './bundle-summary-cache.ts'
|
|
16
|
+
|
|
17
|
+
const decompress = promisify(brotliDecompress)
|
|
18
|
+
const MAX_DECODED_BYTES = 512 * 1024 * 1024
|
|
19
|
+
export const MAX_PACKAGE_INVENTORY_BYTES = 1024 * 1024
|
|
20
|
+
export type BundleCachePart = 'metadata' | 'contents'
|
|
21
|
+
export type BundleCacheRecord = Pick<ManagedBundle, 'id' | 'integrity' | 'filename' | 'kind' | 'byteSize'>
|
|
22
|
+
export interface BundleSummary { files: number; codeFiles: number; lines: number; commit?: string }
|
|
23
|
+
|
|
24
|
+
export async function readBundleDetails(record: BundleCacheRecord, store: BundleStore) {
|
|
25
|
+
const bytes = await store.get(record.id, record.kind)
|
|
26
|
+
if (!bytes) return null
|
|
27
|
+
const decoded = record.kind === 'stasis' || record.kind === 'sourcemap' ? await decompress(bytes, { maxOutputLength: MAX_DECODED_BYTES }) : bytes
|
|
28
|
+
if (decoded.length > MAX_DECODED_BYTES) throw new Error('Decoded bundle too large')
|
|
29
|
+
return parseBundleContents(decodeUtf8(decoded), { integrity: record.integrity, kind: record.kind, size: record.byteSize })
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface BundleCacheStorage {
|
|
33
|
+
exists(id: string, file: string): Promise<boolean>
|
|
34
|
+
put(id: string, file: string, bytes: Buffer): Promise<void>
|
|
35
|
+
open(id: string, file: string): Promise<OpenedBlob>
|
|
36
|
+
// Remove all cached versions for this bundle, including legacy derivatives.
|
|
37
|
+
delete(id: string): Promise<void>
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const filename = `v${BUNDLE_METADATA_VERSION}-metadata.json.br`
|
|
41
|
+
const packagesFilename = 'v4-advisory-inventory.json'
|
|
42
|
+
|
|
43
|
+
// All scopes share one bounded derivative. Never decode full bundle metadata
|
|
44
|
+
// on advisory requests, including when selecting a reason. Persist null when
|
|
45
|
+
// the combined inventory exceeds the budget, without disabling metadata.
|
|
46
|
+
function encodePackageInventory(details: BundleDetails): Buffer {
|
|
47
|
+
const parts: string[] = []
|
|
48
|
+
let size = 0
|
|
49
|
+
function append(part: string) {
|
|
50
|
+
size += Buffer.byteLength(part)
|
|
51
|
+
if (size > MAX_PACKAGE_INVENTORY_BYTES) return false
|
|
52
|
+
parts.push(part)
|
|
53
|
+
return true
|
|
54
|
+
}
|
|
55
|
+
function entries(rows: unknown[]) {
|
|
56
|
+
if (!append('[')) return false
|
|
57
|
+
let first = true
|
|
58
|
+
for (const row of rows) {
|
|
59
|
+
if (!append(`${first ? '' : ','}${JSON.stringify(row)}`)) return false
|
|
60
|
+
first = false
|
|
61
|
+
}
|
|
62
|
+
return append(']')
|
|
63
|
+
}
|
|
64
|
+
function inventory(paths: Set<string> | null) {
|
|
65
|
+
const { packages, skipped } = bundleAdvisoryInventory(details, paths)
|
|
66
|
+
return append('{"packages":') && entries(packages) && append(',"skipped":') && entries(skipped) && append('}')
|
|
67
|
+
}
|
|
68
|
+
append('{"all":')
|
|
69
|
+
if (!inventory(null) || !append(',"reasons":{')) return Buffer.from('null')
|
|
70
|
+
let first = true
|
|
71
|
+
for (const [reason, paths] of bundleReasons(details)) {
|
|
72
|
+
if (!append(`${first ? '' : ','}${JSON.stringify(reason)}:`) || !inventory(paths)) return Buffer.from('null')
|
|
73
|
+
first = false
|
|
74
|
+
}
|
|
75
|
+
if (!append('}}')) return Buffer.from('null')
|
|
76
|
+
return Buffer.from(parts.join(''))
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async function build(record: BundleCacheRecord, storage: BundleCacheStorage, db: ManagedDb, store: BundleStore) {
|
|
80
|
+
const details = await readBundleDetails(record, store)
|
|
81
|
+
if (!details) throw new Error('Bundle bytes unavailable')
|
|
82
|
+
const metadata = { ...await createBundleMetadata(details), id: record.id, filename: record.filename }
|
|
83
|
+
const body = await encodeBrotli(Buffer.from(JSON.stringify(metadata)))
|
|
84
|
+
if (!(await db.getBundle(record.id))) throw new Error('Bundle deleted')
|
|
85
|
+
await storage.put(record.id, filename, body)
|
|
86
|
+
if (record.kind === 'stasis') await storage.put(record.id, packagesFilename, encodePackageInventory(details))
|
|
87
|
+
const summary = createBundleSummary(details, metadata)
|
|
88
|
+
await storage.put(record.id, SUMMARY_FILENAME, Buffer.from(JSON.stringify(summary)))
|
|
89
|
+
// A different instance may have deleted the row while these writes ran.
|
|
90
|
+
// Reconcile after publishing so its cleanup cannot be undone by us.
|
|
91
|
+
if (!(await db.getBundle(record.id))) {
|
|
92
|
+
await storage.delete(record.id)
|
|
93
|
+
throw new Error('Bundle deleted')
|
|
94
|
+
}
|
|
95
|
+
return summary
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export function createBundleCache(storage: BundleCacheStorage, db: ManagedDb, store: BundleStore) {
|
|
99
|
+
const pending = new Map<string, Promise<void>>()
|
|
100
|
+
// The database deduplicates by integrity, so each hash has one persistent
|
|
101
|
+
// bundle/cache directory, shared across teams and repeated uploads.
|
|
102
|
+
const summaries = createBundleSummaryCache(storage, async record => {
|
|
103
|
+
const details = await readBundleDetails(record, store)
|
|
104
|
+
if (!details) throw new Error('Bundle bytes unavailable')
|
|
105
|
+
return createBundleSummary(details)
|
|
106
|
+
}, id => db.getBundle(id))
|
|
107
|
+
let queue = Promise.resolve()
|
|
108
|
+
async function ensure(record: BundleCacheRecord): Promise<void> {
|
|
109
|
+
const existing = pending.get(record.id)
|
|
110
|
+
if (existing) return existing
|
|
111
|
+
const job = (async () => {
|
|
112
|
+
if (await storage.exists(record.id, filename) && (record.kind !== 'stasis' || await storage.exists(record.id, packagesFilename))) return
|
|
113
|
+
const work = queue.then(async () => summaries.remember(record, await build(record, storage, db, store)))
|
|
114
|
+
queue = work.catch(() => {})
|
|
115
|
+
await work
|
|
116
|
+
})()
|
|
117
|
+
pending.set(record.id, job)
|
|
118
|
+
try { await job } finally { if (pending.get(record.id) === job) pending.delete(record.id) }
|
|
119
|
+
}
|
|
120
|
+
async function openCached(record: BundleCacheRecord, file: string) {
|
|
121
|
+
try { return await storage.open(record.id, file) }
|
|
122
|
+
catch (error) { if (!(error instanceof CacheMissError)) throw error }
|
|
123
|
+
await ensure(record)
|
|
124
|
+
return storage.open(record.id, file)
|
|
125
|
+
}
|
|
126
|
+
return {
|
|
127
|
+
prebuild: ensure,
|
|
128
|
+
summary: summaries.summary,
|
|
129
|
+
summaryStatus: summaries.summaryStatus,
|
|
130
|
+
backfillSummaries: summaries.backfill,
|
|
131
|
+
async open(record: ManagedBundle, part: BundleCachePart) {
|
|
132
|
+
if (part === 'contents') {
|
|
133
|
+
if (record.kind !== 'stasis' && record.kind !== 'sourcemap') throw new Error('Unsupported bundle')
|
|
134
|
+
const stored = await store.open(record.id, record.kind)
|
|
135
|
+
if (!stored) throw new Error('Bundle bytes unavailable')
|
|
136
|
+
return stored
|
|
137
|
+
}
|
|
138
|
+
return openCached(record, filename)
|
|
139
|
+
},
|
|
140
|
+
async advisoryInventory(record: ManagedBundle, reason = ''): Promise<BundleAdvisoryInventory | null | undefined> {
|
|
141
|
+
if (record.kind !== 'stasis') return { packages: [], skipped: [] }
|
|
142
|
+
const cached = await openCached(record, packagesFilename)
|
|
143
|
+
try {
|
|
144
|
+
if (cached.size != null && cached.size > MAX_PACKAGE_INVENTORY_BYTES) return null
|
|
145
|
+
const chunks: Buffer[] = []
|
|
146
|
+
let size = 0
|
|
147
|
+
for await (const chunk of cached.stream) {
|
|
148
|
+
size += chunk.length
|
|
149
|
+
if (size > MAX_PACKAGE_INVENTORY_BYTES) return null
|
|
150
|
+
chunks.push(Buffer.from(chunk))
|
|
151
|
+
}
|
|
152
|
+
const inventory = JSON.parse(decodeUtf8(Buffer.concat(chunks)))
|
|
153
|
+
if (inventory === null) return null
|
|
154
|
+
return reason ? (Object.hasOwn(inventory.reasons, reason) ? inventory.reasons[reason] : undefined) : inventory.all
|
|
155
|
+
} finally { cached.stream.destroy() }
|
|
156
|
+
},
|
|
157
|
+
async delete(id: string) {
|
|
158
|
+
// Call after deleting the row. Waiting prevents an in-flight builder
|
|
159
|
+
// from recreating its files after deletion has completed.
|
|
160
|
+
await pending.get(id)?.catch(() => {})
|
|
161
|
+
await summaries.forget(id)
|
|
162
|
+
await storage.delete(id)
|
|
163
|
+
},
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
export type BundleCache = ReturnType<typeof createBundleCache>
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { BundleCache, BundleSummary } from './bundle-cache.ts'
|
|
2
|
+
import type { AdminBundle, UserTeamBundle } from './db.ts'
|
|
3
|
+
|
|
4
|
+
const SUMMARY_READ_CONCURRENCY = 8
|
|
5
|
+
|
|
6
|
+
// Read only existing summaries: cold/malformed bundles must never trigger
|
|
7
|
+
// parsing or wait for metadata builds before the catalog response is sent.
|
|
8
|
+
export async function bundleSummaries(bundles: readonly (AdminBundle | UserTeamBundle)[], cache?: BundleCache) {
|
|
9
|
+
const summaries = new Map<string, { summary: BundleSummary | null; summaryRetryAt: number | null }>()
|
|
10
|
+
if (cache) {
|
|
11
|
+
const unique = [...new Map(bundles.map(bundle => [bundle.integrity, bundle])).values()]
|
|
12
|
+
let next = 0
|
|
13
|
+
// Cold reads may each access remote storage. Bound those requests as well
|
|
14
|
+
// as the subsequent backfill, without dropping entries from large catalogs.
|
|
15
|
+
await Promise.all(Array.from({ length: Math.min(SUMMARY_READ_CONCURRENCY, unique.length) }, async () => {
|
|
16
|
+
while (next < unique.length) {
|
|
17
|
+
const bundle = unique[next++]!
|
|
18
|
+
summaries.set(bundle.integrity, await cache.summaryStatus(bundle).catch(() => ({ summary: null, summaryRetryAt: null })))
|
|
19
|
+
}
|
|
20
|
+
}))
|
|
21
|
+
}
|
|
22
|
+
return summaries
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// Called after sending the response. Each cache permits one small batch at a
|
|
26
|
+
// time, deduplicated by content hash, and skips persisted retry-backoff markers.
|
|
27
|
+
export async function backfillBundleSummaries(bundles: readonly (AdminBundle | UserTeamBundle)[], cache?: BundleCache) {
|
|
28
|
+
await cache?.backfillSummaries([...new Map(bundles.map(bundle => [bundle.integrity, bundle])).values()])
|
|
29
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Sourcemaps are stored once as Brotli. The DB retains the uploaded filename,
|
|
2
|
+
// byte size and integrity so compression does not change report links/dedup.
|
|
3
|
+
import type { Buffer } from 'node:buffer'
|
|
4
|
+
import type { BlobStore, OpenedBlob } from './blob-store.ts'
|
|
5
|
+
import { encodeBrotli } from './brotli.ts'
|
|
6
|
+
|
|
7
|
+
export interface BundleStore {
|
|
8
|
+
put(id: string, bytes: Buffer, kind: string | null): Promise<string | null>
|
|
9
|
+
get(id: string, kind: string | null): Promise<Buffer | null>
|
|
10
|
+
open(id: string, kind: string | null): Promise<OpenedBlob | null>
|
|
11
|
+
delete(id: string): Promise<void>
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function createBundleStore(archives: BlobStore, sourcemaps: BlobStore): BundleStore {
|
|
15
|
+
const storage = (kind: string | null) => kind === 'sourcemap' ? sourcemaps : archives
|
|
16
|
+
return {
|
|
17
|
+
async put(id, bytes, kind) {
|
|
18
|
+
return storage(kind).put(id, kind === 'sourcemap' ? await encodeBrotli(bytes) : bytes)
|
|
19
|
+
},
|
|
20
|
+
get: (id, kind) => storage(kind).get(id),
|
|
21
|
+
open: (id, kind) => storage(kind).open(id),
|
|
22
|
+
async delete(id) { await Promise.all([archives.delete(id), sourcemaps.delete(id)]) },
|
|
23
|
+
}
|
|
24
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer'
|
|
2
|
+
import { setImmediate } from 'node:timers/promises'
|
|
3
|
+
import { CacheMissError } from './cache-storage.ts'
|
|
4
|
+
import { decodeUtf8 } from '../common/utf8.js'
|
|
5
|
+
import { bundleCommitHash } from '../common/bundle-commit.js'
|
|
6
|
+
import type { BundleCacheRecord, BundleCacheStorage, BundleSummary } from './bundle-cache.ts'
|
|
7
|
+
|
|
8
|
+
// Version 3 adds the optional source commit; older bundles use bounded backfill.
|
|
9
|
+
export const SUMMARY_FILENAME = 'v3-summary.json'
|
|
10
|
+
const RETRY_MS = 5 * 60_000
|
|
11
|
+
const BACKFILL_LIMIT = 4
|
|
12
|
+
type CachedSummary = BundleSummary | { retryAt: number }
|
|
13
|
+
|
|
14
|
+
async function readSummary(storage: BundleCacheStorage, id: string): Promise<CachedSummary> {
|
|
15
|
+
const cached = await storage.open(id, SUMMARY_FILENAME)
|
|
16
|
+
try {
|
|
17
|
+
const chunks: Buffer[] = []
|
|
18
|
+
let size = 0
|
|
19
|
+
for await (const chunk of cached.stream) {
|
|
20
|
+
size += chunk.length
|
|
21
|
+
if (size > 1024) throw new Error('Invalid bundle summary')
|
|
22
|
+
chunks.push(Buffer.from(chunk))
|
|
23
|
+
}
|
|
24
|
+
const summary = JSON.parse(decodeUtf8(Buffer.concat(chunks))) as CachedSummary
|
|
25
|
+
if (summary && 'retryAt' in summary && Number.isSafeInteger(summary.retryAt) && summary.retryAt > 0) return summary
|
|
26
|
+
const counts = summary as BundleSummary
|
|
27
|
+
if (!counts || ![counts.files, counts.codeFiles, counts.lines].every(value => Number.isSafeInteger(value) && value >= 0)
|
|
28
|
+
|| counts.codeFiles > counts.files || (counts.commit !== undefined && !bundleCommitHash(counts.commit))) throw new Error('Invalid bundle summary')
|
|
29
|
+
return counts
|
|
30
|
+
} finally { cached.stream.destroy() }
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function createBundleSummaryCache(storage: BundleCacheStorage, build: (record: BundleCacheRecord) => Promise<BundleSummary>, exists: (id: string) => Promise<unknown>) {
|
|
34
|
+
const entries = new Map<string, { id: string; value: CachedSummary }>()
|
|
35
|
+
const reads = new Map<string, { id: string; job: Promise<CachedSummary | null> }>()
|
|
36
|
+
const active = new Map<string, Promise<void>>()
|
|
37
|
+
let pending: Promise<void> | null = null
|
|
38
|
+
function remember(record: BundleCacheRecord, value: CachedSummary) {
|
|
39
|
+
// A completed build supersedes any older storage read still in flight.
|
|
40
|
+
reads.delete(record.integrity)
|
|
41
|
+
// Summaries are tiny and immutable by integrity. Refuse overflow admission
|
|
42
|
+
// instead of evicting the next hit during a scan larger than the cache.
|
|
43
|
+
if (entries.size >= 4096 && !entries.has(record.integrity)) return
|
|
44
|
+
entries.set(record.integrity, { id: record.id, value })
|
|
45
|
+
}
|
|
46
|
+
function read(record: BundleCacheRecord): Promise<CachedSummary | null> {
|
|
47
|
+
const known = entries.get(record.integrity)
|
|
48
|
+
if (known) return Promise.resolve(known.value)
|
|
49
|
+
const existing = reads.get(record.integrity)
|
|
50
|
+
if (existing) return existing.job
|
|
51
|
+
// Concurrent catalogs share the decoded summary, never the response stream.
|
|
52
|
+
// Only in-flight work is shared: misses and failures remain retryable.
|
|
53
|
+
const job: Promise<CachedSummary | null> = readSummary(storage, record.id).then(value => {
|
|
54
|
+
if (reads.get(record.integrity)?.job === job) remember(record, value)
|
|
55
|
+
return entries.get(record.integrity)?.value ?? value
|
|
56
|
+
}).catch(error => {
|
|
57
|
+
if (error instanceof CacheMissError) return null
|
|
58
|
+
throw error
|
|
59
|
+
}).finally(() => { if (reads.get(record.integrity)?.job === job) reads.delete(record.integrity) })
|
|
60
|
+
reads.set(record.integrity, { id: record.id, job })
|
|
61
|
+
return job
|
|
62
|
+
}
|
|
63
|
+
async function publish(record: BundleCacheRecord, value: CachedSummary) {
|
|
64
|
+
if (!await exists(record.id)) return
|
|
65
|
+
await storage.put(record.id, SUMMARY_FILENAME, Buffer.from(JSON.stringify(value)))
|
|
66
|
+
if (!await exists(record.id)) { await storage.delete(record.id); return }
|
|
67
|
+
remember(record, value)
|
|
68
|
+
}
|
|
69
|
+
async function buildOne(record: BundleCacheRecord) {
|
|
70
|
+
try { await publish(record, await build(record)) }
|
|
71
|
+
catch { await publish(record, { retryAt: Date.now() + RETRY_MS }).catch(() => {}) }
|
|
72
|
+
}
|
|
73
|
+
async function summaryStatus(record: BundleCacheRecord) {
|
|
74
|
+
const value = ['stasis', 'sourcemap'].includes(record.kind ?? '') ? await read(record) : null
|
|
75
|
+
return { summary: value && !('retryAt' in value) ? value : null,
|
|
76
|
+
summaryRetryAt: value && 'retryAt' in value ? value.retryAt : null }
|
|
77
|
+
}
|
|
78
|
+
async function backfill(records: readonly BundleCacheRecord[]) {
|
|
79
|
+
// Start after the catalog has been sent. Keep this work awaited by the
|
|
80
|
+
// request owner (including serverless), with no detached tasks or backlog.
|
|
81
|
+
await setImmediate()
|
|
82
|
+
let attempts = 0
|
|
83
|
+
for (const record of records) {
|
|
84
|
+
if (!['stasis', 'sourcemap'].includes(record.kind ?? '')) continue
|
|
85
|
+
const cached = await read(record).catch(() => null)
|
|
86
|
+
if (cached && (!('retryAt' in cached) || cached.retryAt > Date.now())) continue
|
|
87
|
+
if (!await exists(record.id)) continue
|
|
88
|
+
if (attempts++ >= BACKFILL_LIMIT) break
|
|
89
|
+
const job = buildOne(record)
|
|
90
|
+
active.set(record.id, job)
|
|
91
|
+
try { await job } finally { active.delete(record.id) }
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return {
|
|
95
|
+
remember,
|
|
96
|
+
summaryStatus,
|
|
97
|
+
async summary(record: BundleCacheRecord): Promise<BundleSummary | null> {
|
|
98
|
+
return (await summaryStatus(record)).summary
|
|
99
|
+
},
|
|
100
|
+
backfill(records: readonly BundleCacheRecord[]) {
|
|
101
|
+
if (pending) return Promise.resolve() // Another catalog already owns the bounded batch.
|
|
102
|
+
const job = backfill(records).finally(() => { if (pending === job) pending = null })
|
|
103
|
+
pending = job
|
|
104
|
+
return job
|
|
105
|
+
},
|
|
106
|
+
async forget(id: string) {
|
|
107
|
+
// Only this bundle can republish its cache during deletion. A stalled
|
|
108
|
+
// backfill elsewhere must not delay removing these bytes or responding.
|
|
109
|
+
await active.get(id)
|
|
110
|
+
for (const [hash, entry] of entries) if (entry.id === id) entries.delete(hash)
|
|
111
|
+
// Do not wait for remote reads during deletion. Their callers still own
|
|
112
|
+
// and drain them, but a late response must not repopulate this cache.
|
|
113
|
+
for (const [hash, entry] of reads) if (entry.id === id) reads.delete(hash)
|
|
114
|
+
},
|
|
115
|
+
}
|
|
116
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// Bundle helpers for the managed server: the content-addressed identity used to
|
|
2
|
+
// dedupe stored bundles and to auto-link reports to them.
|
|
3
|
+
//
|
|
4
|
+
// A report's top-level `bundleHashes` lists the `sha512-<base64>` integrities
|
|
5
|
+
// the analyzer ran against (see ui/view/ingest.js); a stored bundle's identity
|
|
6
|
+
// is that same integrity computed from its bytes. Matching the two is how a
|
|
7
|
+
// report auto-links to its bundle.
|
|
8
|
+
import { createHash } from 'node:crypto'
|
|
9
|
+
import type { Buffer } from 'node:buffer'
|
|
10
|
+
import { StringDecoder } from 'node:string_decoder'
|
|
11
|
+
import { createBrotliDecompress } from 'node:zlib'
|
|
12
|
+
|
|
13
|
+
// `sha512-<base64>` identity for a bundle's bytes. MUST stay byte-identical to
|
|
14
|
+
// the client's common/integrity.js (SHA-512 → standard base64 WITH padding) so
|
|
15
|
+
// a report's `bundleHashes` entries match a stored bundle's computed integrity.
|
|
16
|
+
// (The client uses `Uint8Array.toBase64()`, whose default is standard base64 +
|
|
17
|
+
// padding — the same bytes node:crypto's base64 digest emits.)
|
|
18
|
+
export function bundleIntegrity(bytes: Buffer): string {
|
|
19
|
+
return `sha512-${createHash('sha512').update(bytes).digest('base64')}`
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// Classify an uploaded bundle by filename — mirrors ui/view/ingest.js
|
|
23
|
+
// `bundleKind`: sourcemap (.map) or stasis (stasis.code.br / .stasis.code.br).
|
|
24
|
+
// null for anything else (stored anyway; the kind is informational).
|
|
25
|
+
export function bundleKind(filename: string): 'sourcemap' | 'stasis' | null {
|
|
26
|
+
// Browsers insert a duplicate-download counter before the final extension.
|
|
27
|
+
const lower = filename.toLowerCase().replace(/ \(\d+\)(?=\.[^.]*$)/u, '')
|
|
28
|
+
if (lower.endsWith('.map')) return 'sourcemap'
|
|
29
|
+
if (lower === 'stasis.code.br' || lower.endsWith('.stasis.code.br')) return 'stasis'
|
|
30
|
+
return null
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
interface BundleRepo { github?: unknown; directory?: unknown }
|
|
34
|
+
const HEADER_KEYS = new Set(['version', 'config', 'repo', 'package'])
|
|
35
|
+
const MAX_HEADER_BYTES = 64 * 1024
|
|
36
|
+
|
|
37
|
+
// Stasis writes origin metadata before entries, formats and source bodies.
|
|
38
|
+
// undefined means another chunk is needed; null means the header has no repo.
|
|
39
|
+
function headerRepo(text: string): BundleRepo | null | undefined {
|
|
40
|
+
let depth = 0, escaped = false, expectKey = false, quoted = false
|
|
41
|
+
let key = '', start = 0, valueStart = 0
|
|
42
|
+
for (let i = 0; i < text.length; i++) {
|
|
43
|
+
const c = text[i]
|
|
44
|
+
if (quoted) {
|
|
45
|
+
if (escaped) { escaped = false; continue }
|
|
46
|
+
if (c === '\\') { escaped = true; continue }
|
|
47
|
+
if (c !== '"') continue
|
|
48
|
+
quoted = false
|
|
49
|
+
if (depth === 1 && expectKey) {
|
|
50
|
+
key = JSON.parse(text.slice(start, i + 1)) as string
|
|
51
|
+
if (!HEADER_KEYS.has(key)) return null
|
|
52
|
+
expectKey = false
|
|
53
|
+
}
|
|
54
|
+
continue
|
|
55
|
+
}
|
|
56
|
+
if (c === '"') { quoted = true; start = i; continue }
|
|
57
|
+
if (c === '{' || c === '[') {
|
|
58
|
+
if (depth === 0 && c !== '{') return null
|
|
59
|
+
depth++
|
|
60
|
+
if (depth === 1) expectKey = true
|
|
61
|
+
} else if (depth === 1 && (c === ',' || c === '}')) {
|
|
62
|
+
if (key === 'repo') {
|
|
63
|
+
const repo: unknown = JSON.parse(text.slice(valueStart, i))
|
|
64
|
+
return repo != null && typeof repo === 'object' && !Array.isArray(repo) ? repo as BundleRepo : null
|
|
65
|
+
}
|
|
66
|
+
if (c === '}') return null
|
|
67
|
+
expectKey = true
|
|
68
|
+
} else if (c === '}' || c === ']') depth--
|
|
69
|
+
else if (depth === 1 && c === ':') valueStart = i + 1
|
|
70
|
+
else if (depth === 0 && c?.trim()) return null
|
|
71
|
+
}
|
|
72
|
+
return undefined
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Decode only a bounded prefix, stopping as soon as repo is read or the
|
|
76
|
+
// header ends. Unknown/malformed archives remain uploadable as opaque bytes.
|
|
77
|
+
export async function bundleRepo(bytes: Buffer): Promise<BundleRepo | null> {
|
|
78
|
+
const stream = createBrotliDecompress({ chunkSize: 4096 })
|
|
79
|
+
const decoder = new StringDecoder('utf8')
|
|
80
|
+
let size = 0, text = ''
|
|
81
|
+
stream.end(bytes)
|
|
82
|
+
try {
|
|
83
|
+
for await (const chunk of stream) {
|
|
84
|
+
size += chunk.length
|
|
85
|
+
if (size > MAX_HEADER_BYTES) return null
|
|
86
|
+
text += decoder.write(chunk)
|
|
87
|
+
const repo = headerRepo(text)
|
|
88
|
+
if (repo !== undefined) return repo
|
|
89
|
+
}
|
|
90
|
+
return null
|
|
91
|
+
} catch { return null }
|
|
92
|
+
finally { stream.destroy() }
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Extract the bundle integrities a report declares (its top-level
|
|
96
|
+
// `bundleHashes`). Non-JSON reports (markdown / CSV) or a missing field → [].
|
|
97
|
+
export function reportBundleHashes(bytes: Buffer): string[] {
|
|
98
|
+
let data: unknown
|
|
99
|
+
try { data = JSON.parse(bytes.toString('utf8')) } catch { return [] }
|
|
100
|
+
if (data == null || typeof data !== 'object') return []
|
|
101
|
+
const hashes = (data as { bundleHashes?: unknown }).bundleHashes
|
|
102
|
+
if (!Array.isArray(hashes)) return []
|
|
103
|
+
return hashes.filter((x): x is string => typeof x === 'string' && x !== '')
|
|
104
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { Buffer } from 'node:buffer'
|
|
2
|
+
import type { OpenedBlob } from './blob-store.ts'
|
|
3
|
+
|
|
4
|
+
export class CacheMissError extends Error {
|
|
5
|
+
constructor() { super('Managed cache unavailable') }
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
// Keys are internal relative paths; delete removes a whole directory of variants.
|
|
9
|
+
export interface CacheStorage {
|
|
10
|
+
exists(key: string): Promise<boolean>
|
|
11
|
+
put(key: string, bytes: Buffer): Promise<void>
|
|
12
|
+
open(key: string): Promise<OpenedBlob>
|
|
13
|
+
delete(prefix: string): Promise<void>
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function validateCacheKey(key: string): string {
|
|
17
|
+
if (!key.split('/').every(part => /^[a-z0-9][a-z0-9.-]*$/iu.test(part))) throw new Error('Invalid cache key')
|
|
18
|
+
return key
|
|
19
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Executable entry for the managed-server bin. See server-e2e/cli.js for the
|
|
3
|
+
// full rationale: Node refuses to strip TypeScript types from files under
|
|
4
|
+
// node_modules, so once this package is installed as a dependency
|
|
5
|
+
// server-managed/index.ts can't be executed directly. Register the in-thread
|
|
6
|
+
// type-stripping hook FIRST, then load the server via a dynamic import (a
|
|
7
|
+
// static import would be fetched before the hook is active) and start it.
|
|
8
|
+
//
|
|
9
|
+
// Imported here rather than run as the process entry, so index.ts's
|
|
10
|
+
// `import.meta.main` auto-start gate stays off and we call `start()` ourselves.
|
|
11
|
+
import '../strip-types-loader.js'
|
|
12
|
+
const command = process.argv[2]
|
|
13
|
+
if (command === '--storage-encryption-status') {
|
|
14
|
+
const { storageCommand } = await import('./storage-command.ts')
|
|
15
|
+
await storageCommand(process.argv[3])
|
|
16
|
+
} else if (command) {
|
|
17
|
+
throw new Error(`Unknown command: ${command}`)
|
|
18
|
+
} else {
|
|
19
|
+
const { start } = await import('./index.ts')
|
|
20
|
+
await start()
|
|
21
|
+
}
|