@preventive/triage 1.0.0-alpha.27 → 1.0.0-alpha.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/LICENSE-APACHE +202 -0
  2. package/client/linked-findings.js +1 -1
  3. package/common/bundle-metadata.js +12 -10
  4. package/common/code-language.js +17 -2
  5. package/common/default-scan-models.ts +3 -3
  6. package/common/finding-id.js +1 -1
  7. package/common/managed/report-content.ts +1 -1
  8. package/common/managed/report-filter.ts +1 -1
  9. package/out/client-managed-import.js +16 -16
  10. package/out/client-managed.js +93 -68
  11. package/out/graph.js +11 -11
  12. package/out/index.html +1 -1
  13. package/out/prism.js +2 -2
  14. package/out/stasis.svg +0 -25
  15. package/out/terminal.js +163 -163
  16. package/out/view.css +1 -1
  17. package/out/view.js +223 -190
  18. package/package.json +18 -28
  19. package/server-e2e/static.ts +3 -4
  20. package/server-managed/bundle-advisories.ts +53 -0
  21. package/server-managed/bundle-advisory-inventory.ts +75 -0
  22. package/server-managed/bundle-build-leases.ts +41 -0
  23. package/server-managed/bundle-build-worker.js +44 -0
  24. package/server-managed/bundle-build.ts +141 -0
  25. package/server-managed/bundle-cache.ts +22 -13
  26. package/server-managed/bundle-summary-cache.ts +27 -9
  27. package/server-managed/bundle.ts +66 -1
  28. package/server-managed/cache-storage.ts +4 -0
  29. package/server-managed/content-access.ts +1 -1
  30. package/server-managed/db-methods.ts +122 -20
  31. package/server-managed/db-neon.ts +121 -32
  32. package/server-managed/db-schema.ts +2 -0
  33. package/server-managed/db.ts +5 -1
  34. package/server-managed/github-app.ts +10 -15
  35. package/server-managed/github-errors.ts +21 -0
  36. package/server-managed/github-issues.ts +1 -1
  37. package/server-managed/github-metadata.ts +34 -1
  38. package/server-managed/github-pulls.ts +142 -30
  39. package/server-managed/http.ts +244 -131
  40. package/server-managed/index.ts +25 -9
  41. package/server-managed/management-catalog.ts +45 -0
  42. package/server-managed/object-storage-vercel.ts +1 -1
  43. package/server-managed/public-workspace.ts +20 -14
  44. package/server-managed/report-migration.ts +1 -1
  45. package/server-managed/report-sources.ts +14 -4
  46. package/server-managed/repository-browser.ts +3 -0
  47. package/server-managed/repository-discovery.ts +1 -1
  48. package/server-managed/revisions.ts +35 -0
  49. package/server-managed/sql-postgres.ts +1 -0
  50. package/server-managed/sql.ts +14 -3
  51. package/server-managed/storage-db.ts +12 -5
  52. package/server-managed/storage-maintenance.ts +37 -22
  53. package/server-managed/storage-payload.ts +16 -4
  54. package/server-managed/storage-stores.ts +2 -2
  55. package/server-managed/storage.ts +6 -4
  56. package/server-managed/team-feed.ts +50 -5
  57. package/server-managed/team-reports.ts +85 -22
  58. package/server-managed/workspace-shares.ts +15 -3
  59. package/report/index.js +0 -260
  60. package/report/src/finding-id.js +0 -80
  61. package/report/src/finding.js +0 -314
  62. package/report/src/labels.js +0 -35
  63. package/report/src/md-structure.js +0 -471
  64. package/report/src/md-text.js +0 -167
  65. package/report/src/meta.js +0 -76
  66. package/report/src/parse-codex.js +0 -149
  67. package/report/src/parse-deepsec.js +0 -199
  68. package/report/src/parse-deepview-fields.js +0 -375
  69. package/report/src/parse-deepview-md.js +0 -186
  70. package/report/src/parse-md-id.js +0 -137
  71. package/report/src/parse-md.js +0 -323
  72. package/report/src/parse-piolium-id.js +0 -79
  73. package/report/src/parse-piolium-rows.js +0 -126
  74. package/report/src/parse-piolium-tokens.js +0 -185
  75. package/report/src/parse-piolium.js +0 -399
  76. package/report/src/security.js +0 -63
  77. package/report/src/severity.js +0 -18
  78. package/report/src/utf8.js +0 -21
  79. package/report/src/write-md-finding.js +0 -273
  80. package/report/src/write-md.js +0 -291
  81. /package/{LICENSE → LICENSE-MIT} +0 -0
package/package.json CHANGED
@@ -1,8 +1,8 @@
1
1
  {
2
2
  "name": "@preventive/triage",
3
- "version": "1.0.0-alpha.27",
3
+ "version": "1.0.0-alpha.28",
4
4
  "description": "Client & relay server for triaging of automated reports",
5
- "license": "MIT",
5
+ "license": "MIT OR Apache-2.0",
6
6
  "author": {
7
7
  "name": "Nikita Skovoroda",
8
8
  "email": "chalkerx@gmail.com",
@@ -30,7 +30,6 @@
30
30
  },
31
31
  "exports": {
32
32
  "./reap": "./api/reap.ts",
33
- "./report": "./report/index.js",
34
33
  "./server": "./server.ts",
35
34
  "./server-e2e": "./server-e2e/index.ts",
36
35
  "./strip-types-loader": "./strip-types-loader.js",
@@ -42,6 +41,8 @@
42
41
  "triage-managed-server": "server-managed/cli.js"
43
42
  },
44
43
  "files": [
44
+ "LICENSE-APACHE",
45
+ "LICENSE-MIT",
45
46
  "api/reap.ts",
46
47
  "cli.js",
47
48
  "server.ts",
@@ -90,7 +91,12 @@
90
91
  "server-managed/avatar-store.ts",
91
92
  "server-managed/blob-store.ts",
92
93
  "server-managed/brotli.ts",
94
+ "server-managed/bundle-advisories.ts",
95
+ "server-managed/bundle-advisory-inventory.ts",
93
96
  "server-managed/bundle-cache.ts",
97
+ "server-managed/bundle-build.ts",
98
+ "server-managed/bundle-build-leases.ts",
99
+ "server-managed/bundle-build-worker.js",
94
100
  "server-managed/bundle-catalog.ts",
95
101
  "server-managed/bundle-store.ts",
96
102
  "server-managed/bundle-summary-cache.ts",
@@ -119,6 +125,7 @@
119
125
  "server-managed/import-triage.ts",
120
126
  "server-managed/managed-issues.ts",
121
127
  "server-managed/management.ts",
128
+ "server-managed/management-catalog.ts",
122
129
  "server-managed/object-storage.ts",
123
130
  "server-managed/object-storage-disk.ts",
124
131
  "server-managed/object-storage-vercel.ts",
@@ -130,6 +137,7 @@
130
137
  "server-managed/report-query.ts",
131
138
  "server-managed/report-response.ts",
132
139
  "server-managed/report-sources.ts",
140
+ "server-managed/revisions.ts",
133
141
  "server-managed/repository-browser.ts",
134
142
  "server-managed/repository-discovery.ts",
135
143
  "server-managed/repository-policy.ts",
@@ -162,28 +170,6 @@
162
170
  "server-common/storage-crypto.ts",
163
171
  "server-common/vercel-blob.ts",
164
172
  "server-common/scan-config.ts",
165
- "report/index.js",
166
- "report/src/finding-id.js",
167
- "report/src/finding.js",
168
- "report/src/severity.js",
169
- "report/src/security.js",
170
- "report/src/labels.js",
171
- "report/src/md-structure.js",
172
- "report/src/md-text.js",
173
- "report/src/meta.js",
174
- "report/src/parse-codex.js",
175
- "report/src/parse-deepsec.js",
176
- "report/src/parse-deepview-fields.js",
177
- "report/src/parse-deepview-md.js",
178
- "report/src/parse-md-id.js",
179
- "report/src/parse-md.js",
180
- "report/src/parse-piolium-id.js",
181
- "report/src/parse-piolium-rows.js",
182
- "report/src/parse-piolium-tokens.js",
183
- "report/src/parse-piolium.js",
184
- "report/src/utf8.js",
185
- "report/src/write-md-finding.js",
186
- "report/src/write-md.js",
187
173
  "common/bundle-metadata.d.ts",
188
174
  "common/bundle-metadata.js",
189
175
  "common/bundle-reasons.d.ts",
@@ -238,7 +224,7 @@
238
224
  ],
239
225
  "scripts": {
240
226
  "lint": "node --run lint:js && node --run lint:css && node --run lint:ts",
241
- "lint:js": "oxlint -c .oxlintrc.json --ignore-pattern=ui/logs.js ./cli.js ./server.ts ./tests/ ./common/ ./report/ ./server-common/ ./server-e2e/ ./server-managed/ ./client/ ./ui/ ./api/",
227
+ "lint:js": "oxlint -c .oxlintrc.json --ignore-pattern=ui/logs.js ./cli.js ./server.ts ./tests/ ./common/ ./server-common/ ./server-e2e/ ./server-managed/ ./client/ ./ui/ ./api/",
242
228
  "lint:css": "stylelint 'ui/**/*.css'",
243
229
  "lint:ts": "tsc --noEmit",
244
230
  "prepack": "node --run build",
@@ -248,12 +234,13 @@
248
234
  "server-e2e": "node server-e2e/index.ts",
249
235
  "server-managed": "node server-managed/index.ts",
250
236
  "server-managed-test": "node server-managed/test-server.ts",
251
- "test": "node --js-base-64 --import=./tests/_resource-loader.js --experimental-test-module-mocks --test-timeout=120000 --test ./tests/*.test.js ./report/tests/*.test.js"
237
+ "test": "node --js-base-64 --import=./tests/_resource-loader.js --experimental-test-module-mocks --test-timeout=120000 --test ./tests/*.test.js"
252
238
  },
253
239
  "devDependencies": {
254
240
  "@electric-sql/pglite": "^0.5.5",
255
241
  "@noble/ciphers": "^2.3.0",
256
242
  "@preventive/diff": "^1.0.0",
243
+ "@preventive/lockfile": "1.0.0-alpha.5",
257
244
  "@preventive/terminal": "2.0.0-beta.1",
258
245
  "@rray/frontend": "^1.0.0",
259
246
  "@stylistic/stylelint-plugin": "^5.3.0",
@@ -270,7 +257,10 @@
270
257
  "typescript": "^7.0.2"
271
258
  },
272
259
  "dependencies": {
273
- "@exodus/stasis-core": "^1.0.0-beta.4",
260
+ "@exodus/stasis": "1.0.0-beta.5",
261
+ "@exodus/stasis-core": "1.0.0-beta.5",
262
+ "@preventive/report": "1.0.0-alpha.4",
263
+ "@preventive/upstream": "1.0.0-alpha.4",
274
264
  "ws": "^8.20.1"
275
265
  },
276
266
  "peerDependencies": {
@@ -194,13 +194,12 @@ function buildEntry(staticDir: string, name: string, deepviewScanServer: string
194
194
  // collision-resistance against the ~tens of files we load.
195
195
  const etag = `"${createHash('sha256').update(identity).digest('hex').slice(0, 32)}"`
196
196
  const isCompressible = COMPRESSIBLE.has(ext)
197
- // Brotli quality 11 is the maximum — slow at compress time but
198
- // we pay it once at boot and ship the smaller bytes on every
199
- // request thereafter. text mode tells the encoder we're working
197
+ // Brotli quality 9 limits boot-time compression work, including on
198
+ // serverless cold starts. Text mode tells the encoder we're working
200
199
  // on UTF-8 text (every COMPRESSIBLE entry is text-shaped).
201
200
  const br = isCompressible ? brotliCompressSync(identity, {
202
201
  params: {
203
- [zlibConstants.BROTLI_PARAM_QUALITY]: 11,
202
+ [zlibConstants.BROTLI_PARAM_QUALITY]: 9,
204
203
  [zlibConstants.BROTLI_PARAM_MODE]: zlibConstants.BROTLI_MODE_TEXT,
205
204
  },
206
205
  }) : null
@@ -0,0 +1,53 @@
1
+ import { type Advisory, type Package, advisories } from '@preventive/upstream/advisories.js'
2
+ import { type Client, HttpError, createClient } from '@preventive/upstream/github.js'
3
+
4
+ export const ADVISORIES_TIMEOUT_MS = 30_000
5
+
6
+ function advisoryGithubClient(token: string | null, signal: AbortSignal): Client {
7
+ const authenticated = createClient({ token, userAgent: 'deepview-triage' })
8
+ if (token === null) return authenticated
9
+ let client = authenticated
10
+ return {
11
+ ...authenticated,
12
+ async listRepoAdvisories(options) {
13
+ const current = client
14
+ try { return await current.listRepoAdvisories(options) } catch (error) {
15
+ if (current !== authenticated || !(error instanceof HttpError) || error.status !== 401) throw error
16
+ signal.throwIfAborted()
17
+ // Published advisories remain readable after a token is revoked. Reuse
18
+ // anonymous access for subsequent repositories in this audit as well.
19
+ if (client === authenticated) client = createClient({ token: null, userAgent: 'deepview-triage' })
20
+ return client.listRepoAdvisories(options)
21
+ }
22
+ },
23
+ }
24
+ }
25
+
26
+ // The managed caller supplies its user's token when available; public shares
27
+ // remain anonymous. Upstream sends this credential only to GitHub and returns
28
+ // published advisories, matching versions across npm, OSV and repository rows.
29
+ export async function fetchBundleAdvisories(packages: Package[], signal: AbortSignal,
30
+ { debug = false, repoAdvisories = false, githubToken = null }: { debug?: boolean; repoAdvisories?: boolean; githubToken?: string | null } = {}): Promise<
31
+ { status: 200; body: Advisory[] } | { status: 502; body: { error: string } }
32
+ > {
33
+ let onAbort: (() => void) | undefined
34
+ try {
35
+ signal.throwIfAborted()
36
+ if (packages.length === 0) return { status: 200, body: [] }
37
+ // Upstream owns each transport's timeout and exposes no cancellation
38
+ // option. Bound the caller's wait for the multi-request audit as well.
39
+ const deadline = new Promise<never>((_resolve, reject) => {
40
+ onAbort = () => reject(signal.reason)
41
+ signal.addEventListener('abort', onAbort, { once: true })
42
+ })
43
+ const result = await Promise.race([
44
+ advisories(packages, { github: advisoryGithubClient(githubToken, signal), repoAdvisories }), deadline,
45
+ ])
46
+ return { status: 200, body: result }
47
+ } catch (error) {
48
+ if (debug) console.warn('managed: advisories failed:', error)
49
+ return { status: 502, body: { error: 'upstream-unavailable' } }
50
+ } finally {
51
+ if (onAbort) signal.removeEventListener('abort', onAbort)
52
+ }
53
+ }
@@ -0,0 +1,75 @@
1
+ import type { Package } from '@preventive/upstream/advisories.js'
2
+ import { satisfies, valid } from '@preventive/upstream/semver.js'
3
+ import type { BundleDetails } from '../common/bundle-metadata.js'
4
+
5
+ export interface SkippedAdvisoryPackage {
6
+ ecosystem: string
7
+ name: string
8
+ version: string
9
+ because: string
10
+ }
11
+ export interface BundleAdvisoryInventory { packages: Package[]; skipped: SkippedAdvisoryPackage[] }
12
+ interface BundleModule { ecosystem?: string; name?: string; version?: string; files: Record<string, unknown> }
13
+
14
+ // Match Stasis's audit evidence rules, including their verified version bounds:
15
+ // https://github.com/PreventiveMeasures/stasis/blob/2bd4c14354da9888c00ab45fa89740413099b5dc/stasis/src/audit-corrections.js
16
+ // Keep this on the server: upstream's semver implementation uses Node's npm.
17
+ const cargoManifests = ['Cargo.toml', 'Cargo.lock', '.cargo-checksum.json']
18
+ const solidityManifests = ['package.json', 'foundry.toml', 'remappings.txt', 'soldeer.toml']
19
+ const manifests = new Map([
20
+ ['npm', ['package.json']],
21
+ ['cargo', cargoManifests], ['cargo-git', cargoManifests], ['cargo-unknown', cargoManifests],
22
+ ['composer', ['composer.json', 'composer.lock']],
23
+ ['soldeer', solidityManifests], ['github', solidityManifests],
24
+ ])
25
+ const browserStubRanges = new Map([['ws', '<=8.21.1'], ['node-fetch', '<=2.7.0']])
26
+
27
+ function isEvidence(ecosystem: string, name: string, version: string, file: string): boolean {
28
+ if (manifests.get(ecosystem)?.includes(file.slice(file.lastIndexOf('/') + 1))) return false
29
+ const stubRange = ecosystem === 'npm' && file === 'browser.js' ? browserStubRanges.get(name) : undefined
30
+ return !stubRange || !valid(version) || !satisfies(version, stubRange)
31
+ }
32
+
33
+ function supported(ecosystem: string): ecosystem is Package['ecosystem'] {
34
+ return ['npm', 'cargo', 'composer', 'soldeer', 'github'].includes(ecosystem)
35
+ }
36
+
37
+ function skipReason(ecosystem: string, version: string): string | undefined {
38
+ if (ecosystem === 'cargo-git') return 'Crate vendored from git; its identity is not a crates.io package.'
39
+ if (ecosystem === 'cargo-unknown') return 'Crate source is unknown because .cargo-checksum.json is missing.'
40
+ if (ecosystem === 'composer' && /^dev-|-dev$/iu.test(version.replace(/#.*$/su, ''))) return 'Composer dev versions cannot be matched against release advisories.'
41
+ return undefined
42
+ }
43
+
44
+ // Package and reason presence both require a recorded evidence file. Entry and
45
+ // manually added files count even when they have no incoming import edge.
46
+ export function bundleAdvisoryInventory(details: BundleDetails, paths: Iterable<string> | null = null): BundleAdvisoryInventory {
47
+ const modules = (details.bundle as { modules?: ReadonlyMap<string, BundleModule> } | undefined)?.modules
48
+ if (details.kind !== 'stasis' || !modules) return { packages: [], skipped: [] }
49
+ const selected = paths === null ? null : new Set(paths)
50
+ const packages = new Map<string, { ecosystem: Package['ecosystem']; name: string; versions: Set<string> }>()
51
+ const skipped = new Map<string, SkippedAdvisoryPackage>()
52
+ for (const [dir, info] of modules) {
53
+ if (dir === '.') continue
54
+ const ecosystem = info.ecosystem ?? (/(?:^|\/)node_modules\//u.test(dir) ? 'npm' : undefined)
55
+ const { name, version } = info
56
+ if (ecosystem === undefined || typeof name !== 'string' || !name || typeof version !== 'string' || !version) continue
57
+ if (!Object.keys(info.files).some(file => (!selected || selected.has(`${dir}/${file}`)) && isEvidence(ecosystem, name, version, file))) continue
58
+ const because = skipReason(ecosystem, version)
59
+ if (!supported(ecosystem) || because !== undefined) {
60
+ skipped.set(JSON.stringify([ecosystem, name, version]), { ecosystem, name, version, because: because ?? `No advisory source is supported for ${ecosystem}.` })
61
+ continue
62
+ }
63
+ // Foundry's branch '.' means the superproject's branch. As in Stasis,
64
+ // 0.0.0 tells upstream its actual version is unknown, so every range covers it.
65
+ const auditedVersion = ecosystem === 'github' && version === '.' ? '0.0.0' : version
66
+ const key = JSON.stringify([ecosystem, ecosystem === 'github' ? name.toLowerCase() : name])
67
+ if (!packages.has(key)) packages.set(key, { ecosystem, name, versions: new Set() })
68
+ packages.get(key)!.versions.add(auditedVersion)
69
+ }
70
+ const byPackage = (a: { ecosystem: string; name: string }, b: { ecosystem: string; name: string }) => a.ecosystem.localeCompare(b.ecosystem) || a.name.localeCompare(b.name)
71
+ return {
72
+ packages: [...packages.values()].map(pkg => ({ ...pkg, versions: [...pkg.versions].toSorted() })).toSorted(byPackage),
73
+ skipped: [...skipped.values()].toSorted((a, b) => byPackage(a, b) || a.version.localeCompare(b.version)),
74
+ }
75
+ }
@@ -0,0 +1,41 @@
1
+ import type { ManagedSql } from './sql.ts'
2
+
3
+ export const BUNDLE_BUILD_TIMEOUT_MS = 180_000
4
+ // Leave time to terminate a timed-out worker before a crashed owner's slot
5
+ // becomes reusable. Expiration uses the database clock, not an instance's clock.
6
+ const LEASE_MS = BUNDLE_BUILD_TIMEOUT_MS + 60_000
7
+ export const BUNDLE_BUILD_LEASE_SCHEMA = `
8
+ CREATE TABLE IF NOT EXISTS managed_bundle_build_lease (
9
+ slot INTEGER PRIMARY KEY CHECK (slot IN (1, 2)),
10
+ owner TEXT NOT NULL UNIQUE,
11
+ user_id TEXT NOT NULL UNIQUE,
12
+ expires_at INTEGER NOT NULL
13
+ ) STRICT;
14
+ `
15
+
16
+ export interface BundleBuildLeaseStore {
17
+ claimBundleBuildLease(userId: string, owner: string): Promise<boolean>
18
+ releaseBundleBuildLease(owner: string): Promise<void>
19
+ }
20
+
21
+ export function bundleBuildLeaseMethods(db: ManagedSql): BundleBuildLeaseStore {
22
+ return {
23
+ // scopeManagedMethods holds SQLite's write transaction / PostgreSQL's
24
+ // advisory transaction lock across cleanup, inspection, and admission.
25
+ async claimBundleBuildLease(userId, owner) {
26
+ const { now } = await db.prepare("SELECT CAST(unixepoch('subsec') * 1000 AS INTEGER) AS now").get() as { now: number }
27
+ await db.prepare('DELETE FROM managed_bundle_build_lease WHERE expires_at <= ?').run(now)
28
+ const leases = await db.prepare('SELECT slot, user_id AS userId FROM managed_bundle_build_lease').all() as { slot: number; userId: string }[]
29
+ if (leases.length >= 2 || leases.some(lease => lease.userId === userId)) return false
30
+ const slot = leases.some(lease => lease.slot === 1) ? 2 : 1
31
+ await db.prepare('INSERT INTO managed_bundle_build_lease (slot, owner, user_id, expires_at) VALUES (?, ?, ?, ?)')
32
+ .run(slot, owner, userId, now + LEASE_MS)
33
+ return true
34
+ },
35
+ // No user FK: deleting an account must not free a still-running worker's
36
+ // slot. A delayed/crashed owner's cleanup cannot release its successor.
37
+ async releaseBundleBuildLease(owner) {
38
+ await db.prepare('DELETE FROM managed_bundle_build_lease WHERE owner = ?').run(owner)
39
+ },
40
+ }
41
+ }
@@ -0,0 +1,44 @@
1
+ import { parentPort, workerData } from 'node:worker_threads'
2
+ import { Buffer } from 'node:buffer'
3
+ import { posix } from 'node:path'
4
+ import { brotliCompressSync } from 'node:zlib'
5
+ import { buildGitHubBundle } from '@exodus/stasis/vfs-bundle'
6
+ import { brotliOptions } from '@exodus/stasis-core/brotli'
7
+ import { HttpError, createClient } from '@preventive/upstream/github.js'
8
+ import { githubBundleFilename } from './bundle-build.ts'
9
+
10
+ export async function buildStasisBundle({ input, github, token, maxBytes, scopes }, client = createClient({ token })) {
11
+ const allowed = path => scopes.some(scope => !scope || path === scope || path.startsWith(scope + '/'))
12
+ // Stasis/upstream cache writes are disabled by default; this fresh worker
13
+ // never enables a cache directory or inherits application credentials.
14
+ const { bundle } = await buildGitHubBundle({
15
+ github, sha: input.commit, directory: input.directory || undefined,
16
+ entries: input.entries.map(entry => posix.relative(input.directory || '.', entry)),
17
+ ...input.options, client,
18
+ })
19
+ const directory = bundle.repo?.directory ?? ''
20
+ // A workspace lockfile may widen the build root. Never publish it under a
21
+ // narrower team grant: the storage location must cover the actual bundle.
22
+ if (!allowed(directory)) throw new Error('build-scope')
23
+ const serialized = bundle.serialize()
24
+ if (Buffer.byteLength(serialized) > 200 * 1024 * 1024) throw new Error('too-large')
25
+ const bytes = brotliCompressSync(serialized, brotliOptions())
26
+ if (bytes.length > maxBytes) throw new Error('too-large')
27
+ return { bytes, directory, filename: githubBundleFilename(github, input.directory, input.commit) }
28
+ }
29
+
30
+ if (parentPort && workerData?.type === 'managed-bundle-build') {
31
+ try {
32
+ // Node worker messages have no browser target origin.
33
+ // oxlint-disable-next-line unicorn/require-post-message-target-origin
34
+ parentPort.postMessage(await buildStasisBundle(workerData))
35
+ } catch (error) {
36
+ // Do not return upstream bodies or paths outside the caller's grants.
37
+ const code = ['build-scope', 'too-large'].includes(error.message) ? error.message
38
+ : error instanceof HttpError ? (error.status === 429 ? 'github-rate-limited' : 'github-build-failed')
39
+ : /no packageManager given|lockfile installs/u.test(error.message) ? 'build-lockfile'
40
+ : 'build-failed'
41
+ // oxlint-disable-next-line unicorn/require-post-message-target-origin
42
+ parentPort.postMessage({ error: code, status: code === 'too-large' ? 413 : code === 'build-scope' ? 403 : 422 })
43
+ }
44
+ }
@@ -0,0 +1,141 @@
1
+ import { Buffer } from 'node:buffer'
2
+ import { createHash, randomUUID } from 'node:crypto'
3
+ import { posix } from 'node:path'
4
+ import { Worker } from 'node:worker_threads'
5
+ import { BUNDLE_BUILD_TIMEOUT_MS, type BundleBuildLeaseStore } from './bundle-build-leases.ts'
6
+
7
+ export class BundleBuildError extends Error {
8
+ status: number
9
+ code: string
10
+ constructor(status: number, code: string) { super(code); this.status = status; this.code = code }
11
+ }
12
+
13
+ export interface BundleBuildInput {
14
+ repoId: number
15
+ commit: string
16
+ entries: string[]
17
+ directory: string
18
+ options: { conditions?: string[]; mainFields?: string[]; metro?: boolean; platforms?: string[]; typescript?: boolean; jsx?: boolean }
19
+ }
20
+
21
+ function path(value: unknown): value is string {
22
+ return typeof value === 'string' && value.length > 0 && value.length <= 1024 && !/[\\\p{Cc}]/u.test(value)
23
+ && value.split('/').every(part => part !== '' && part !== '.' && part !== '..')
24
+ }
25
+
26
+ export function parseBundleBuild(value: unknown): BundleBuildInput {
27
+ const fail = (code = 'bad-request'): never => { throw new BundleBuildError(400, code) }
28
+ if (value == null || typeof value !== 'object' || Array.isArray(value)) return fail()
29
+ const { repoId, commit, entries, conditions } = value as Record<string, unknown>
30
+ if (typeof repoId !== 'number' || !Number.isSafeInteger(repoId) || repoId <= 0
31
+ || typeof commit !== 'string' || !/^[a-f\d]{40}$/u.test(commit)
32
+ || !Array.isArray(entries) || entries.length === 0 || entries.length > 100 || !entries.every(path)) return fail()
33
+ const selected = [...new Set(entries as string[])]
34
+ const scripts = selected.every(entry => /\.[mc]?[jt]s$/u.test(entry))
35
+ if (!scripts && !selected.every(entry => entry.endsWith('.sol'))) return fail('unsupported-entries')
36
+ const parts = posix.dirname(selected[0]!).split('/').filter(part => part !== '.')
37
+ for (const entry of selected) {
38
+ const parent = posix.dirname(entry).split('/')
39
+ while (parts.some((part, i) => parent[i] !== part)) parts.pop()
40
+ }
41
+ const options: BundleBuildInput['options'] = {}
42
+ if (scripts) {
43
+ if (conditions == null || typeof conditions !== 'object' || Array.isArray(conditions)) return fail('bad-conditions')
44
+ const { preset, conditions: names, platforms } = conditions as Record<string, unknown>
45
+ if (!['node', 'browser', 'metro'].includes(String(preset)) || !Array.isArray(names) || names.length > 16
46
+ || !names.every(name => typeof name === 'string' && name.length > 0 && name.length <= 64
47
+ && !/^\.|^\d+$|[\s,/\\\p{Cc}]/u.test(name) && !['default', 'import', 'require'].includes(name))) return fail('bad-conditions')
48
+ if (preset === 'metro') {
49
+ if (names.length !== 1 || names[0] !== 'react-native') return fail('metro-conditions')
50
+ if (!Array.isArray(platforms) || platforms.length === 0 || platforms.length > 2
51
+ || !platforms.every(platform => ['ios', 'android'].includes(platform))) return fail('bad-conditions')
52
+ Object.assign(options, { metro: true, platforms: [...new Set(platforms)] })
53
+ } else {
54
+ options.conditions = [...new Set(names)]
55
+ if (preset === 'browser') options.mainFields = ['browser', 'module', 'main']
56
+ }
57
+ options.typescript = true
58
+ options.jsx = true
59
+ }
60
+ return { repoId, commit, entries: selected, directory: parts.join('/'), options }
61
+ }
62
+
63
+ // Same portable name and 255-character limit as Stasis's githubBundleFile
64
+ // (src/cmd/github-bundle.js), which is not a public package export.
65
+ export function githubBundleFilename(github: string, directory: string, commit: string): string {
66
+ const portable = (value: string) => value.replaceAll(/[^\w.-]/gu, '_')
67
+ const repo = portable(github.replace('/', '-'))
68
+ const tail = `.${portable(commit.slice(0, 7))}.stasis.code.br`
69
+ if (!directory) return `${repo}${tail}`
70
+ const room = 255 - `${repo}.${tail}`.length
71
+ let dir = portable(directory.replaceAll('/', '-'))
72
+ if (dir.length > room) dir = `${dir.slice(0, room - 9)}_${createHash('sha256').update(directory).digest('hex').slice(0, 8)}`
73
+ return `${repo}.${dir}${tail}`
74
+ }
75
+
76
+ export interface BuiltBundle { bytes: Uint8Array; directory: string; filename: string }
77
+ export interface BuildRequest { input: BundleBuildInput; github: string; token: string | null; maxBytes: number; scopes: (string | null)[] }
78
+ // An extra per-process ceiling; admission must also hold the shared DB lease.
79
+ const active = new Set<string>()
80
+
81
+ export async function withBundleBuildLease<T>(db: BundleBuildLeaseStore, userId: string, signal: AbortSignal,
82
+ work: (signal: AbortSignal) => Promise<T>): Promise<T> {
83
+ signal.throwIfAborted()
84
+ const owner = randomUUID(), started = performance.now(), timeout = new AbortController()
85
+ const expired = () => timeout.abort(new BundleBuildError(504, 'build-timeout'))
86
+ const timer = setTimeout(expired, BUNDLE_BUILD_TIMEOUT_MS)
87
+ const combined = AbortSignal.any([signal, timeout.signal])
88
+ let claimed = false
89
+ try {
90
+ claimed = await db.claimBundleBuildLease(userId, owner)
91
+ if (!claimed) throw new BundleBuildError(429, 'build-busy')
92
+ // Account for a slow/lost claim response or a suspended invocation before
93
+ // starting work. The three-minute budget starts before lease acquisition.
94
+ if (performance.now() - started >= BUNDLE_BUILD_TIMEOUT_MS) expired()
95
+ combined.throwIfAborted()
96
+ return await work(combined)
97
+ } catch (error) {
98
+ if (timeout.signal.aborted && !signal.aborted) throw timeout.signal.reason
99
+ throw error
100
+ } finally {
101
+ clearTimeout(timer)
102
+ // buildRepositoryBundle waits for worker termination before settling.
103
+ // Unknown claim/release outcomes expire safely without starting more work.
104
+ if (claimed) await db.releaseBundleBuildLease(owner)
105
+ }
106
+ }
107
+
108
+ // Parsing repositories and compressing bundles stays off the HTTP event loop.
109
+ // Cap concurrent builds, terminate disconnected/timed-out work, and share no
110
+ // credentials or mutable Stasis state between requests.
111
+ export async function buildRepositoryBundle(userId: string, request: BuildRequest, signal: AbortSignal): Promise<BuiltBundle> {
112
+ if (active.has(userId) || active.size >= 2) throw new BundleBuildError(429, 'build-busy')
113
+ signal.throwIfAborted()
114
+ active.add(userId)
115
+ let worker: Worker | undefined
116
+ try {
117
+ worker = new Worker(new URL('./bundle-build-worker.js', import.meta.url), {
118
+ workerData: { ...request, type: 'managed-bundle-build' }, env: {}, execArgv: [], resourceLimits: { maxOldGenerationSizeMb: 512 },
119
+ })
120
+ const running = worker
121
+ return await new Promise((resolve, reject) => {
122
+ const timer = setTimeout(() => reject(new BundleBuildError(504, 'build-timeout')), BUNDLE_BUILD_TIMEOUT_MS)
123
+ const abort = () => reject(new BundleBuildError(499, 'build-cancelled'))
124
+ signal.addEventListener('abort', abort, { once: true })
125
+ running.once('message', (message: BuiltBundle & { error?: string; status?: number }) => {
126
+ clearTimeout(timer)
127
+ signal.removeEventListener('abort', abort)
128
+ if (message.error) reject(new BundleBuildError(message.status ?? 422, message.error))
129
+ else resolve({ ...message, bytes: Buffer.from(message.bytes) })
130
+ })
131
+ running.once('error', () => reject(new BundleBuildError(422, 'build-failed')))
132
+ running.once('exit', () => {
133
+ clearTimeout(timer)
134
+ signal.removeEventListener('abort', abort)
135
+ reject(new BundleBuildError(422, 'build-failed'))
136
+ })
137
+ })
138
+ } finally {
139
+ try { await worker?.terminate() } finally { active.delete(userId) }
140
+ }
141
+ }
@@ -5,12 +5,13 @@ import { brotliDecompress } from 'node:zlib'
5
5
  import { promisify } from 'node:util'
6
6
  import { BUNDLE_METADATA_VERSION, type BundleDetails, createBundleMetadata, createBundleSummary, parseBundleContents } from '../common/bundle-metadata.js'
7
7
  import { bundleReasons } from '../common/bundle-reasons.js'
8
- import { bundlePackageVersions } from '../common/bundle-sources.js'
8
+ import { type BundleAdvisoryInventory, bundleAdvisoryInventory } from './bundle-advisory-inventory.ts'
9
9
  import { decodeUtf8 } from '../common/utf8.js'
10
10
  import type { OpenedBlob } from './blob-store.ts'
11
11
  import type { BundleStore } from './bundle-store.ts'
12
12
  import { encodeBrotli } from './brotli.ts'
13
13
  import type { ManagedBundle, ManagedDb } from './db.ts'
14
+ import { CacheMissError } from './cache-storage.ts'
14
15
  import { SUMMARY_FILENAME, createBundleSummaryCache } from './bundle-summary-cache.ts'
15
16
 
16
17
  const decompress = promisify(brotliDecompress)
@@ -37,7 +38,7 @@ export interface BundleCacheStorage {
37
38
  }
38
39
 
39
40
  const filename = `v${BUNDLE_METADATA_VERSION}-metadata.json.br`
40
- const packagesFilename = 'v2-package-versions.json'
41
+ const packagesFilename = 'v4-advisory-inventory.json'
41
42
 
42
43
  // All scopes share one bounded derivative. Never decode full bundle metadata
43
44
  // on advisory requests, including when selecting a reason. Persist null when
@@ -51,14 +52,18 @@ function encodePackageInventory(details: BundleDetails): Buffer {
51
52
  parts.push(part)
52
53
  return true
53
54
  }
54
- function inventory(paths: Set<string> | null) {
55
- if (!append('{')) return false
55
+ function entries(rows: unknown[]) {
56
+ if (!append('[')) return false
56
57
  let first = true
57
- for (const [name, versions] of bundlePackageVersions(details, paths)) {
58
- if (!append(`${first ? '' : ','}${JSON.stringify(name)}:${JSON.stringify([...versions].toSorted())}`)) return false
58
+ for (const row of rows) {
59
+ if (!append(`${first ? '' : ','}${JSON.stringify(row)}`)) return false
59
60
  first = false
60
61
  }
61
- return append('}')
62
+ return append(']')
63
+ }
64
+ function inventory(paths: Set<string> | null) {
65
+ const { packages, skipped } = bundleAdvisoryInventory(details, paths)
66
+ return append('{"packages":') && entries(packages) && append(',"skipped":') && entries(skipped) && append('}')
62
67
  }
63
68
  append('{"all":')
64
69
  if (!inventory(null) || !append(',"reasons":{')) return Buffer.from('null')
@@ -112,6 +117,12 @@ export function createBundleCache(storage: BundleCacheStorage, db: ManagedDb, st
112
117
  pending.set(record.id, job)
113
118
  try { await job } finally { if (pending.get(record.id) === job) pending.delete(record.id) }
114
119
  }
120
+ async function openCached(record: BundleCacheRecord, file: string) {
121
+ try { return await storage.open(record.id, file) }
122
+ catch (error) { if (!(error instanceof CacheMissError)) throw error }
123
+ await ensure(record)
124
+ return storage.open(record.id, file)
125
+ }
115
126
  return {
116
127
  prebuild: ensure,
117
128
  summary: summaries.summary,
@@ -124,13 +135,11 @@ export function createBundleCache(storage: BundleCacheStorage, db: ManagedDb, st
124
135
  if (!stored) throw new Error('Bundle bytes unavailable')
125
136
  return stored
126
137
  }
127
- await ensure(record)
128
- return storage.open(record.id, filename)
138
+ return openCached(record, filename)
129
139
  },
130
- async packageVersions(record: ManagedBundle, reason = ''): Promise<Record<string, string[]> | null | undefined> {
131
- if (record.kind !== 'stasis') return {}
132
- await ensure(record)
133
- const cached = await storage.open(record.id, packagesFilename)
140
+ async advisoryInventory(record: ManagedBundle, reason = ''): Promise<BundleAdvisoryInventory | null | undefined> {
141
+ if (record.kind !== 'stasis') return { packages: [], skipped: [] }
142
+ const cached = await openCached(record, packagesFilename)
134
143
  try {
135
144
  if (cached.size != null && cached.size > MAX_PACKAGE_INVENTORY_BYTES) return null
136
145
  const chunks: Buffer[] = []
@@ -1,10 +1,11 @@
1
1
  import { Buffer } from 'node:buffer'
2
2
  import { setImmediate } from 'node:timers/promises'
3
- import { BUNDLE_METADATA_VERSION } from '../common/bundle-metadata.js'
3
+ import { CacheMissError } from './cache-storage.ts'
4
4
  import { decodeUtf8 } from '../common/utf8.js'
5
5
  import type { BundleCacheRecord, BundleCacheStorage, BundleSummary } from './bundle-cache.ts'
6
6
 
7
- export const SUMMARY_FILENAME = `v${BUNDLE_METADATA_VERSION}-summary.json`
7
+ // Origin metadata does not change these counts; retain the byte-sized v2 cache.
8
+ export const SUMMARY_FILENAME = 'v2-summary.json'
8
9
  const RETRY_MS = 5 * 60_000
9
10
  const BACKFILL_LIMIT = 4
10
11
  type CachedSummary = BundleSummary | { retryAt: number }
@@ -30,19 +31,33 @@ async function readSummary(storage: BundleCacheStorage, id: string): Promise<Cac
30
31
 
31
32
  export function createBundleSummaryCache(storage: BundleCacheStorage, build: (record: BundleCacheRecord) => Promise<BundleSummary>, exists: (id: string) => Promise<unknown>) {
32
33
  const entries = new Map<string, { id: string; value: CachedSummary }>()
34
+ const reads = new Map<string, { id: string; job: Promise<CachedSummary | null> }>()
33
35
  const active = new Map<string, Promise<void>>()
34
36
  let pending: Promise<void> | null = null
35
37
  function remember(record: BundleCacheRecord, value: CachedSummary) {
36
- if (entries.size >= 256) entries.delete(entries.keys().next().value!)
38
+ // A completed build supersedes any older storage read still in flight.
39
+ reads.delete(record.integrity)
40
+ // Summaries are tiny and immutable by integrity. Refuse overflow admission
41
+ // instead of evicting the next hit during a scan larger than the cache.
42
+ if (entries.size >= 4096 && !entries.has(record.integrity)) return
37
43
  entries.set(record.integrity, { id: record.id, value })
38
44
  }
39
- async function read(record: BundleCacheRecord) {
45
+ function read(record: BundleCacheRecord): Promise<CachedSummary | null> {
40
46
  const known = entries.get(record.integrity)
41
- if (known) return known.value
42
- if (!await storage.exists(record.id, SUMMARY_FILENAME)) return null
43
- const value = await readSummary(storage, record.id)
44
- remember(record, value)
45
- return value
47
+ if (known) return Promise.resolve(known.value)
48
+ const existing = reads.get(record.integrity)
49
+ if (existing) return existing.job
50
+ // Concurrent catalogs share the decoded summary, never the response stream.
51
+ // Only in-flight work is shared: misses and failures remain retryable.
52
+ const job: Promise<CachedSummary | null> = readSummary(storage, record.id).then(value => {
53
+ if (reads.get(record.integrity)?.job === job) remember(record, value)
54
+ return entries.get(record.integrity)?.value ?? value
55
+ }).catch(error => {
56
+ if (error instanceof CacheMissError) return null
57
+ throw error
58
+ }).finally(() => { if (reads.get(record.integrity)?.job === job) reads.delete(record.integrity) })
59
+ reads.set(record.integrity, { id: record.id, job })
60
+ return job
46
61
  }
47
62
  async function publish(record: BundleCacheRecord, value: CachedSummary) {
48
63
  if (!await exists(record.id)) return
@@ -92,6 +107,9 @@ export function createBundleSummaryCache(storage: BundleCacheStorage, build: (re
92
107
  // backfill elsewhere must not delay removing these bytes or responding.
93
108
  await active.get(id)
94
109
  for (const [hash, entry] of entries) if (entry.id === id) entries.delete(hash)
110
+ // Do not wait for remote reads during deletion. Their callers still own
111
+ // and drain them, but a late response must not repopulate this cache.
112
+ for (const [hash, entry] of reads) if (entry.id === id) reads.delete(hash)
95
113
  },
96
114
  }
97
115
  }