@preventive/triage 1.0.0-alpha.26 → 1.0.0-alpha.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/LICENSE-APACHE +202 -0
  2. package/client/linked-findings.js +1 -1
  3. package/common/bundle-metadata.js +12 -10
  4. package/common/code-language.js +17 -2
  5. package/common/default-scan-models.ts +3 -3
  6. package/common/finding-id.js +1 -1
  7. package/common/managed/report-content.ts +1 -1
  8. package/common/managed/report-filter.ts +1 -1
  9. package/out/client-managed-import.js +24 -24
  10. package/out/client-managed.js +93 -68
  11. package/out/graph.js +11 -11
  12. package/out/index.html +1 -1
  13. package/out/prism.js +2 -2
  14. package/out/stasis.svg +0 -25
  15. package/out/terminal.js +163 -163
  16. package/out/view.css +1 -1
  17. package/out/view.js +223 -190
  18. package/package.json +29 -29
  19. package/server-common/storage-crypto.ts +182 -0
  20. package/server-common/storage-log.ts +5 -1
  21. package/server-common/vercel-blob.ts +6 -3
  22. package/server-e2e/static.ts +3 -4
  23. package/server-managed/avatar-store.ts +1 -40
  24. package/server-managed/blob-store.ts +3 -52
  25. package/server-managed/bundle-advisories.ts +53 -0
  26. package/server-managed/bundle-advisory-inventory.ts +75 -0
  27. package/server-managed/bundle-build-leases.ts +41 -0
  28. package/server-managed/bundle-build-worker.js +44 -0
  29. package/server-managed/bundle-build.ts +141 -0
  30. package/server-managed/bundle-cache.ts +22 -41
  31. package/server-managed/bundle-store.ts +3 -7
  32. package/server-managed/bundle-summary-cache.ts +27 -9
  33. package/server-managed/bundle.ts +66 -1
  34. package/server-managed/cache-storage.ts +4 -25
  35. package/server-managed/cli.js +10 -2
  36. package/server-managed/config.ts +12 -1
  37. package/server-managed/content-access.ts +1 -1
  38. package/server-managed/db-methods.ts +163 -40
  39. package/server-managed/db-neon.ts +124 -30
  40. package/server-managed/db-schema.ts +2 -0
  41. package/server-managed/db.ts +11 -1
  42. package/server-managed/github-app.ts +10 -15
  43. package/server-managed/github-errors.ts +21 -0
  44. package/server-managed/github-issues.ts +1 -1
  45. package/server-managed/github-metadata.ts +34 -1
  46. package/server-managed/github-pulls.ts +142 -30
  47. package/server-managed/http.ts +278 -167
  48. package/server-managed/index.ts +26 -9
  49. package/server-managed/management-catalog.ts +45 -0
  50. package/server-managed/object-storage-disk.ts +144 -0
  51. package/server-managed/object-storage-vercel.ts +60 -0
  52. package/server-managed/object-storage.ts +71 -0
  53. package/server-managed/public-workspace.ts +20 -14
  54. package/server-managed/report-migration.ts +1 -1
  55. package/server-managed/report-sources.ts +14 -4
  56. package/server-managed/repository-browser.ts +8 -1
  57. package/server-managed/repository-discovery.ts +1 -1
  58. package/server-managed/revisions.ts +35 -0
  59. package/server-managed/solidity-entry-points.ts +69 -0
  60. package/server-managed/sql-postgres.ts +1 -0
  61. package/server-managed/sql.ts +14 -8
  62. package/server-managed/storage-command.ts +16 -0
  63. package/server-managed/storage-db.ts +175 -0
  64. package/server-managed/storage-encryption.ts +211 -0
  65. package/server-managed/storage-maintenance.ts +197 -0
  66. package/server-managed/storage-payload.ts +72 -0
  67. package/server-managed/storage-stores.ts +81 -0
  68. package/server-managed/storage.ts +48 -24
  69. package/server-managed/team-feed.ts +50 -5
  70. package/server-managed/team-reports.ts +85 -22
  71. package/server-managed/uploads.ts +2 -2
  72. package/server-managed/workspace-shares.ts +15 -3
  73. package/report/index.js +0 -260
  74. package/report/src/finding-id.js +0 -80
  75. package/report/src/finding.js +0 -314
  76. package/report/src/labels.js +0 -35
  77. package/report/src/md-structure.js +0 -471
  78. package/report/src/md-text.js +0 -167
  79. package/report/src/meta.js +0 -76
  80. package/report/src/parse-codex.js +0 -149
  81. package/report/src/parse-deepsec.js +0 -199
  82. package/report/src/parse-deepview-fields.js +0 -375
  83. package/report/src/parse-deepview-md.js +0 -186
  84. package/report/src/parse-md-id.js +0 -137
  85. package/report/src/parse-md.js +0 -323
  86. package/report/src/parse-piolium-id.js +0 -79
  87. package/report/src/parse-piolium-rows.js +0 -126
  88. package/report/src/parse-piolium-tokens.js +0 -185
  89. package/report/src/parse-piolium.js +0 -399
  90. package/report/src/security.js +0 -63
  91. package/report/src/severity.js +0 -18
  92. package/report/src/utf8.js +0 -21
  93. package/report/src/write-md-finding.js +0 -273
  94. package/report/src/write-md.js +0 -291
  95. package/server-managed/blob-vercel.ts +0 -127
  96. /package/{LICENSE → LICENSE-MIT} +0 -0
@@ -0,0 +1,41 @@
1
+ import type { ManagedSql } from './sql.ts'
2
+
3
+ export const BUNDLE_BUILD_TIMEOUT_MS = 180_000
4
+ // Leave time to terminate a timed-out worker before a crashed owner's slot
5
+ // becomes reusable. Expiration uses the database clock, not an instance's clock.
6
+ const LEASE_MS = BUNDLE_BUILD_TIMEOUT_MS + 60_000
7
+ export const BUNDLE_BUILD_LEASE_SCHEMA = `
8
+ CREATE TABLE IF NOT EXISTS managed_bundle_build_lease (
9
+ slot INTEGER PRIMARY KEY CHECK (slot IN (1, 2)),
10
+ owner TEXT NOT NULL UNIQUE,
11
+ user_id TEXT NOT NULL UNIQUE,
12
+ expires_at INTEGER NOT NULL
13
+ ) STRICT;
14
+ `
15
+
16
+ export interface BundleBuildLeaseStore {
17
+ claimBundleBuildLease(userId: string, owner: string): Promise<boolean>
18
+ releaseBundleBuildLease(owner: string): Promise<void>
19
+ }
20
+
21
+ export function bundleBuildLeaseMethods(db: ManagedSql): BundleBuildLeaseStore {
22
+ return {
23
+ // scopeManagedMethods holds SQLite's write transaction / PostgreSQL's
24
+ // advisory transaction lock across cleanup, inspection, and admission.
25
+ async claimBundleBuildLease(userId, owner) {
26
+ const { now } = await db.prepare("SELECT CAST(unixepoch('subsec') * 1000 AS INTEGER) AS now").get() as { now: number }
27
+ await db.prepare('DELETE FROM managed_bundle_build_lease WHERE expires_at <= ?').run(now)
28
+ const leases = await db.prepare('SELECT slot, user_id AS userId FROM managed_bundle_build_lease').all() as { slot: number; userId: string }[]
29
+ if (leases.length >= 2 || leases.some(lease => lease.userId === userId)) return false
30
+ const slot = leases.some(lease => lease.slot === 1) ? 2 : 1
31
+ await db.prepare('INSERT INTO managed_bundle_build_lease (slot, owner, user_id, expires_at) VALUES (?, ?, ?, ?)')
32
+ .run(slot, owner, userId, now + LEASE_MS)
33
+ return true
34
+ },
35
+ // No user FK: deleting an account must not free a still-running worker's
36
+ // slot. A delayed/crashed owner's cleanup cannot release its successor.
37
+ async releaseBundleBuildLease(owner) {
38
+ await db.prepare('DELETE FROM managed_bundle_build_lease WHERE owner = ?').run(owner)
39
+ },
40
+ }
41
+ }
@@ -0,0 +1,44 @@
1
+ import { parentPort, workerData } from 'node:worker_threads'
2
+ import { Buffer } from 'node:buffer'
3
+ import { posix } from 'node:path'
4
+ import { brotliCompressSync } from 'node:zlib'
5
+ import { buildGitHubBundle } from '@exodus/stasis/vfs-bundle'
6
+ import { brotliOptions } from '@exodus/stasis-core/brotli'
7
+ import { HttpError, createClient } from '@preventive/upstream/github.js'
8
+ import { githubBundleFilename } from './bundle-build.ts'
9
+
10
+ export async function buildStasisBundle({ input, github, token, maxBytes, scopes }, client = createClient({ token })) {
11
+ const allowed = path => scopes.some(scope => !scope || path === scope || path.startsWith(scope + '/'))
12
+ // Stasis/upstream cache writes are disabled by default; this fresh worker
13
+ // never enables a cache directory or inherits application credentials.
14
+ const { bundle } = await buildGitHubBundle({
15
+ github, sha: input.commit, directory: input.directory || undefined,
16
+ entries: input.entries.map(entry => posix.relative(input.directory || '.', entry)),
17
+ ...input.options, client,
18
+ })
19
+ const directory = bundle.repo?.directory ?? ''
20
+ // A workspace lockfile may widen the build root. Never publish it under a
21
+ // narrower team grant: the storage location must cover the actual bundle.
22
+ if (!allowed(directory)) throw new Error('build-scope')
23
+ const serialized = bundle.serialize()
24
+ if (Buffer.byteLength(serialized) > 200 * 1024 * 1024) throw new Error('too-large')
25
+ const bytes = brotliCompressSync(serialized, brotliOptions())
26
+ if (bytes.length > maxBytes) throw new Error('too-large')
27
+ return { bytes, directory, filename: githubBundleFilename(github, input.directory, input.commit) }
28
+ }
29
+
30
+ if (parentPort && workerData?.type === 'managed-bundle-build') {
31
+ try {
32
+ // Node worker messages have no browser target origin.
33
+ // oxlint-disable-next-line unicorn/require-post-message-target-origin
34
+ parentPort.postMessage(await buildStasisBundle(workerData))
35
+ } catch (error) {
36
+ // Do not return upstream bodies or paths outside the caller's grants.
37
+ const code = ['build-scope', 'too-large'].includes(error.message) ? error.message
38
+ : error instanceof HttpError ? (error.status === 429 ? 'github-rate-limited' : 'github-build-failed')
39
+ : /no packageManager given|lockfile installs/u.test(error.message) ? 'build-lockfile'
40
+ : 'build-failed'
41
+ // oxlint-disable-next-line unicorn/require-post-message-target-origin
42
+ parentPort.postMessage({ error: code, status: code === 'too-large' ? 413 : code === 'build-scope' ? 403 : 422 })
43
+ }
44
+ }
@@ -0,0 +1,141 @@
1
+ import { Buffer } from 'node:buffer'
2
+ import { createHash, randomUUID } from 'node:crypto'
3
+ import { posix } from 'node:path'
4
+ import { Worker } from 'node:worker_threads'
5
+ import { BUNDLE_BUILD_TIMEOUT_MS, type BundleBuildLeaseStore } from './bundle-build-leases.ts'
6
+
7
+ export class BundleBuildError extends Error {
8
+ status: number
9
+ code: string
10
+ constructor(status: number, code: string) { super(code); this.status = status; this.code = code }
11
+ }
12
+
13
+ export interface BundleBuildInput {
14
+ repoId: number
15
+ commit: string
16
+ entries: string[]
17
+ directory: string
18
+ options: { conditions?: string[]; mainFields?: string[]; metro?: boolean; platforms?: string[]; typescript?: boolean; jsx?: boolean }
19
+ }
20
+
21
+ function path(value: unknown): value is string {
22
+ return typeof value === 'string' && value.length > 0 && value.length <= 1024 && !/[\\\p{Cc}]/u.test(value)
23
+ && value.split('/').every(part => part !== '' && part !== '.' && part !== '..')
24
+ }
25
+
26
+ export function parseBundleBuild(value: unknown): BundleBuildInput {
27
+ const fail = (code = 'bad-request'): never => { throw new BundleBuildError(400, code) }
28
+ if (value == null || typeof value !== 'object' || Array.isArray(value)) return fail()
29
+ const { repoId, commit, entries, conditions } = value as Record<string, unknown>
30
+ if (typeof repoId !== 'number' || !Number.isSafeInteger(repoId) || repoId <= 0
31
+ || typeof commit !== 'string' || !/^[a-f\d]{40}$/u.test(commit)
32
+ || !Array.isArray(entries) || entries.length === 0 || entries.length > 100 || !entries.every(path)) return fail()
33
+ const selected = [...new Set(entries as string[])]
34
+ const scripts = selected.every(entry => /\.[mc]?[jt]s$/u.test(entry))
35
+ if (!scripts && !selected.every(entry => entry.endsWith('.sol'))) return fail('unsupported-entries')
36
+ const parts = posix.dirname(selected[0]!).split('/').filter(part => part !== '.')
37
+ for (const entry of selected) {
38
+ const parent = posix.dirname(entry).split('/')
39
+ while (parts.some((part, i) => parent[i] !== part)) parts.pop()
40
+ }
41
+ const options: BundleBuildInput['options'] = {}
42
+ if (scripts) {
43
+ if (conditions == null || typeof conditions !== 'object' || Array.isArray(conditions)) return fail('bad-conditions')
44
+ const { preset, conditions: names, platforms } = conditions as Record<string, unknown>
45
+ if (!['node', 'browser', 'metro'].includes(String(preset)) || !Array.isArray(names) || names.length > 16
46
+ || !names.every(name => typeof name === 'string' && name.length > 0 && name.length <= 64
47
+ && !/^\.|^\d+$|[\s,/\\\p{Cc}]/u.test(name) && !['default', 'import', 'require'].includes(name))) return fail('bad-conditions')
48
+ if (preset === 'metro') {
49
+ if (names.length !== 1 || names[0] !== 'react-native') return fail('metro-conditions')
50
+ if (!Array.isArray(platforms) || platforms.length === 0 || platforms.length > 2
51
+ || !platforms.every(platform => ['ios', 'android'].includes(platform))) return fail('bad-conditions')
52
+ Object.assign(options, { metro: true, platforms: [...new Set(platforms)] })
53
+ } else {
54
+ options.conditions = [...new Set(names)]
55
+ if (preset === 'browser') options.mainFields = ['browser', 'module', 'main']
56
+ }
57
+ options.typescript = true
58
+ options.jsx = true
59
+ }
60
+ return { repoId, commit, entries: selected, directory: parts.join('/'), options }
61
+ }
62
+
63
+ // Same portable name and 255-character limit as Stasis's githubBundleFile
64
+ // (src/cmd/github-bundle.js), which is not a public package export.
65
+ export function githubBundleFilename(github: string, directory: string, commit: string): string {
66
+ const portable = (value: string) => value.replaceAll(/[^\w.-]/gu, '_')
67
+ const repo = portable(github.replace('/', '-'))
68
+ const tail = `.${portable(commit.slice(0, 7))}.stasis.code.br`
69
+ if (!directory) return `${repo}${tail}`
70
+ const room = 255 - `${repo}.${tail}`.length
71
+ let dir = portable(directory.replaceAll('/', '-'))
72
+ if (dir.length > room) dir = `${dir.slice(0, room - 9)}_${createHash('sha256').update(directory).digest('hex').slice(0, 8)}`
73
+ return `${repo}.${dir}${tail}`
74
+ }
75
+
76
+ export interface BuiltBundle { bytes: Uint8Array; directory: string; filename: string }
77
+ export interface BuildRequest { input: BundleBuildInput; github: string; token: string | null; maxBytes: number; scopes: (string | null)[] }
78
+ // An extra per-process ceiling; admission must also hold the shared DB lease.
79
+ const active = new Set<string>()
80
+
81
+ export async function withBundleBuildLease<T>(db: BundleBuildLeaseStore, userId: string, signal: AbortSignal,
82
+ work: (signal: AbortSignal) => Promise<T>): Promise<T> {
83
+ signal.throwIfAborted()
84
+ const owner = randomUUID(), started = performance.now(), timeout = new AbortController()
85
+ const expired = () => timeout.abort(new BundleBuildError(504, 'build-timeout'))
86
+ const timer = setTimeout(expired, BUNDLE_BUILD_TIMEOUT_MS)
87
+ const combined = AbortSignal.any([signal, timeout.signal])
88
+ let claimed = false
89
+ try {
90
+ claimed = await db.claimBundleBuildLease(userId, owner)
91
+ if (!claimed) throw new BundleBuildError(429, 'build-busy')
92
+ // Account for a slow/lost claim response or a suspended invocation before
93
+ // starting work. The three-minute budget starts before lease acquisition.
94
+ if (performance.now() - started >= BUNDLE_BUILD_TIMEOUT_MS) expired()
95
+ combined.throwIfAborted()
96
+ return await work(combined)
97
+ } catch (error) {
98
+ if (timeout.signal.aborted && !signal.aborted) throw timeout.signal.reason
99
+ throw error
100
+ } finally {
101
+ clearTimeout(timer)
102
+ // buildRepositoryBundle waits for worker termination before settling.
103
+ // Unknown claim/release outcomes expire safely without starting more work.
104
+ if (claimed) await db.releaseBundleBuildLease(owner)
105
+ }
106
+ }
107
+
108
+ // Parsing repositories and compressing bundles stays off the HTTP event loop.
109
+ // Cap concurrent builds, terminate disconnected/timed-out work, and share no
110
+ // credentials or mutable Stasis state between requests.
111
+ export async function buildRepositoryBundle(userId: string, request: BuildRequest, signal: AbortSignal): Promise<BuiltBundle> {
112
+ if (active.has(userId) || active.size >= 2) throw new BundleBuildError(429, 'build-busy')
113
+ signal.throwIfAborted()
114
+ active.add(userId)
115
+ let worker: Worker | undefined
116
+ try {
117
+ worker = new Worker(new URL('./bundle-build-worker.js', import.meta.url), {
118
+ workerData: { ...request, type: 'managed-bundle-build' }, env: {}, execArgv: [], resourceLimits: { maxOldGenerationSizeMb: 512 },
119
+ })
120
+ const running = worker
121
+ return await new Promise((resolve, reject) => {
122
+ const timer = setTimeout(() => reject(new BundleBuildError(504, 'build-timeout')), BUNDLE_BUILD_TIMEOUT_MS)
123
+ const abort = () => reject(new BundleBuildError(499, 'build-cancelled'))
124
+ signal.addEventListener('abort', abort, { once: true })
125
+ running.once('message', (message: BuiltBundle & { error?: string; status?: number }) => {
126
+ clearTimeout(timer)
127
+ signal.removeEventListener('abort', abort)
128
+ if (message.error) reject(new BundleBuildError(message.status ?? 422, message.error))
129
+ else resolve({ ...message, bytes: Buffer.from(message.bytes) })
130
+ })
131
+ running.once('error', () => reject(new BundleBuildError(422, 'build-failed')))
132
+ running.once('exit', () => {
133
+ clearTimeout(timer)
134
+ signal.removeEventListener('abort', abort)
135
+ reject(new BundleBuildError(422, 'build-failed'))
136
+ })
137
+ })
138
+ } finally {
139
+ try { await worker?.terminate() } finally { active.delete(userId) }
140
+ }
141
+ }
@@ -1,19 +1,17 @@
1
1
  // Metadata is cached as Brotli. Contents use the stored Brotli bytes directly:
2
2
  // unchanged Stasis uploads or sourcemaps compressed once at upload.
3
3
  import { Buffer } from 'node:buffer'
4
- import { mkdir, open, rename, rm, stat, writeFile } from 'node:fs/promises'
5
- import { join } from 'node:path'
6
- import { randomUUID } from 'node:crypto'
7
4
  import { brotliDecompress } from 'node:zlib'
8
5
  import { promisify } from 'node:util'
9
6
  import { BUNDLE_METADATA_VERSION, type BundleDetails, createBundleMetadata, createBundleSummary, parseBundleContents } from '../common/bundle-metadata.js'
10
7
  import { bundleReasons } from '../common/bundle-reasons.js'
11
- import { bundlePackageVersions } from '../common/bundle-sources.js'
8
+ import { type BundleAdvisoryInventory, bundleAdvisoryInventory } from './bundle-advisory-inventory.ts'
12
9
  import { decodeUtf8 } from '../common/utf8.js'
13
10
  import type { OpenedBlob } from './blob-store.ts'
14
11
  import type { BundleStore } from './bundle-store.ts'
15
12
  import { encodeBrotli } from './brotli.ts'
16
13
  import type { ManagedBundle, ManagedDb } from './db.ts'
14
+ import { CacheMissError } from './cache-storage.ts'
17
15
  import { SUMMARY_FILENAME, createBundleSummaryCache } from './bundle-summary-cache.ts'
18
16
 
19
17
  const decompress = promisify(brotliDecompress)
@@ -40,7 +38,7 @@ export interface BundleCacheStorage {
40
38
  }
41
39
 
42
40
  const filename = `v${BUNDLE_METADATA_VERSION}-metadata.json.br`
43
- const packagesFilename = 'v2-package-versions.json'
41
+ const packagesFilename = 'v4-advisory-inventory.json'
44
42
 
45
43
  // All scopes share one bounded derivative. Never decode full bundle metadata
46
44
  // on advisory requests, including when selecting a reason. Persist null when
@@ -54,14 +52,18 @@ function encodePackageInventory(details: BundleDetails): Buffer {
54
52
  parts.push(part)
55
53
  return true
56
54
  }
57
- function inventory(paths: Set<string> | null) {
58
- if (!append('{')) return false
55
+ function entries(rows: unknown[]) {
56
+ if (!append('[')) return false
59
57
  let first = true
60
- for (const [name, versions] of bundlePackageVersions(details, paths)) {
61
- if (!append(`${first ? '' : ','}${JSON.stringify(name)}:${JSON.stringify([...versions].toSorted())}`)) return false
58
+ for (const row of rows) {
59
+ if (!append(`${first ? '' : ','}${JSON.stringify(row)}`)) return false
62
60
  first = false
63
61
  }
64
- return append('}')
62
+ return append(']')
63
+ }
64
+ function inventory(paths: Set<string> | null) {
65
+ const { packages, skipped } = bundleAdvisoryInventory(details, paths)
66
+ return append('{"packages":') && entries(packages) && append(',"skipped":') && entries(skipped) && append('}')
65
67
  }
66
68
  append('{"all":')
67
69
  if (!inventory(null) || !append(',"reasons":{')) return Buffer.from('null')
@@ -115,6 +117,12 @@ export function createBundleCache(storage: BundleCacheStorage, db: ManagedDb, st
115
117
  pending.set(record.id, job)
116
118
  try { await job } finally { if (pending.get(record.id) === job) pending.delete(record.id) }
117
119
  }
120
+ async function openCached(record: BundleCacheRecord, file: string) {
121
+ try { return await storage.open(record.id, file) }
122
+ catch (error) { if (!(error instanceof CacheMissError)) throw error }
123
+ await ensure(record)
124
+ return storage.open(record.id, file)
125
+ }
118
126
  return {
119
127
  prebuild: ensure,
120
128
  summary: summaries.summary,
@@ -127,13 +135,11 @@ export function createBundleCache(storage: BundleCacheStorage, db: ManagedDb, st
127
135
  if (!stored) throw new Error('Bundle bytes unavailable')
128
136
  return stored
129
137
  }
130
- await ensure(record)
131
- return storage.open(record.id, filename)
138
+ return openCached(record, filename)
132
139
  },
133
- async packageVersions(record: ManagedBundle, reason = ''): Promise<Record<string, string[]> | null | undefined> {
134
- if (record.kind !== 'stasis') return {}
135
- await ensure(record)
136
- const cached = await storage.open(record.id, packagesFilename)
140
+ async advisoryInventory(record: ManagedBundle, reason = ''): Promise<BundleAdvisoryInventory | null | undefined> {
141
+ if (record.kind !== 'stasis') return { packages: [], skipped: [] }
142
+ const cached = await openCached(record, packagesFilename)
137
143
  try {
138
144
  if (cached.size != null && cached.size > MAX_PACKAGE_INVENTORY_BYTES) return null
139
145
  const chunks: Buffer[] = []
@@ -158,28 +164,3 @@ export function createBundleCache(storage: BundleCacheStorage, db: ManagedDb, st
158
164
  }
159
165
  }
160
166
  export type BundleCache = ReturnType<typeof createBundleCache>
161
-
162
- export function createDiskBundleCache(dir: string, db: ManagedDb, store: BundleStore): BundleCache {
163
- function directory(id: string) {
164
- if (!/^[a-f\d-]{36}$/iu.test(id)) throw new Error('Invalid bundle id')
165
- return join(dir, id)
166
- }
167
- return createBundleCache({
168
- async exists(id, file) {
169
- try { await stat(join(directory(id), file)); return true }
170
- catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; throw err }
171
- },
172
- async put(id, file, bytes) {
173
- await mkdir(directory(id), { recursive: true })
174
- const target = join(directory(id), file), temp = `${target}.${randomUUID()}.tmp`
175
- try { await writeFile(temp, bytes); await rename(temp, target) }
176
- finally { await rm(temp, { force: true }) }
177
- },
178
- async open(id, name) {
179
- const file = await open(join(directory(id), name), 'r')
180
- try { return { size: (await file.stat()).size, stream: file.createReadStream() } }
181
- catch (err) { await file.close(); throw err }
182
- },
183
- async delete(id) { await rm(directory(id), { recursive: true, force: true }) },
184
- }, db, store)
185
- }
@@ -1,11 +1,11 @@
1
1
  // Sourcemaps are stored once as Brotli. The DB retains the uploaded filename,
2
2
  // byte size and integrity so compression does not change report links/dedup.
3
3
  import type { Buffer } from 'node:buffer'
4
- import { type BlobStore, type OpenedBlob, createDiskBlobStore } from './blob-store.ts'
4
+ import type { BlobStore, OpenedBlob } from './blob-store.ts'
5
5
  import { encodeBrotli } from './brotli.ts'
6
6
 
7
7
  export interface BundleStore {
8
- put(id: string, bytes: Buffer, kind: string | null): Promise<void>
8
+ put(id: string, bytes: Buffer, kind: string | null): Promise<string | null>
9
9
  get(id: string, kind: string | null): Promise<Buffer | null>
10
10
  open(id: string, kind: string | null): Promise<OpenedBlob | null>
11
11
  delete(id: string): Promise<void>
@@ -15,14 +15,10 @@ export function createBundleStore(archives: BlobStore, sourcemaps: BlobStore): B
15
15
  const storage = (kind: string | null) => kind === 'sourcemap' ? sourcemaps : archives
16
16
  return {
17
17
  async put(id, bytes, kind) {
18
- await storage(kind).put(id, kind === 'sourcemap' ? await encodeBrotli(bytes) : bytes)
18
+ return storage(kind).put(id, kind === 'sourcemap' ? await encodeBrotli(bytes) : bytes)
19
19
  },
20
20
  get: (id, kind) => storage(kind).get(id),
21
21
  open: (id, kind) => storage(kind).open(id),
22
22
  async delete(id) { await Promise.all([archives.delete(id), sourcemaps.delete(id)]) },
23
23
  }
24
24
  }
25
-
26
- export function createDiskBundleStore(dir: string): BundleStore {
27
- return createBundleStore(createDiskBlobStore(dir), createDiskBlobStore(dir, '.map.br'))
28
- }
@@ -1,10 +1,11 @@
1
1
  import { Buffer } from 'node:buffer'
2
2
  import { setImmediate } from 'node:timers/promises'
3
- import { BUNDLE_METADATA_VERSION } from '../common/bundle-metadata.js'
3
+ import { CacheMissError } from './cache-storage.ts'
4
4
  import { decodeUtf8 } from '../common/utf8.js'
5
5
  import type { BundleCacheRecord, BundleCacheStorage, BundleSummary } from './bundle-cache.ts'
6
6
 
7
- export const SUMMARY_FILENAME = `v${BUNDLE_METADATA_VERSION}-summary.json`
7
+ // Origin metadata does not change these counts; retain the byte-sized v2 cache.
8
+ export const SUMMARY_FILENAME = 'v2-summary.json'
8
9
  const RETRY_MS = 5 * 60_000
9
10
  const BACKFILL_LIMIT = 4
10
11
  type CachedSummary = BundleSummary | { retryAt: number }
@@ -30,19 +31,33 @@ async function readSummary(storage: BundleCacheStorage, id: string): Promise<Cac
30
31
 
31
32
  export function createBundleSummaryCache(storage: BundleCacheStorage, build: (record: BundleCacheRecord) => Promise<BundleSummary>, exists: (id: string) => Promise<unknown>) {
32
33
  const entries = new Map<string, { id: string; value: CachedSummary }>()
34
+ const reads = new Map<string, { id: string; job: Promise<CachedSummary | null> }>()
33
35
  const active = new Map<string, Promise<void>>()
34
36
  let pending: Promise<void> | null = null
35
37
  function remember(record: BundleCacheRecord, value: CachedSummary) {
36
- if (entries.size >= 256) entries.delete(entries.keys().next().value!)
38
+ // A completed build supersedes any older storage read still in flight.
39
+ reads.delete(record.integrity)
40
+ // Summaries are tiny and immutable by integrity. Refuse overflow admission
41
+ // instead of evicting the next hit during a scan larger than the cache.
42
+ if (entries.size >= 4096 && !entries.has(record.integrity)) return
37
43
  entries.set(record.integrity, { id: record.id, value })
38
44
  }
39
- async function read(record: BundleCacheRecord) {
45
+ function read(record: BundleCacheRecord): Promise<CachedSummary | null> {
40
46
  const known = entries.get(record.integrity)
41
- if (known) return known.value
42
- if (!await storage.exists(record.id, SUMMARY_FILENAME)) return null
43
- const value = await readSummary(storage, record.id)
44
- remember(record, value)
45
- return value
47
+ if (known) return Promise.resolve(known.value)
48
+ const existing = reads.get(record.integrity)
49
+ if (existing) return existing.job
50
+ // Concurrent catalogs share the decoded summary, never the response stream.
51
+ // Only in-flight work is shared: misses and failures remain retryable.
52
+ const job: Promise<CachedSummary | null> = readSummary(storage, record.id).then(value => {
53
+ if (reads.get(record.integrity)?.job === job) remember(record, value)
54
+ return entries.get(record.integrity)?.value ?? value
55
+ }).catch(error => {
56
+ if (error instanceof CacheMissError) return null
57
+ throw error
58
+ }).finally(() => { if (reads.get(record.integrity)?.job === job) reads.delete(record.integrity) })
59
+ reads.set(record.integrity, { id: record.id, job })
60
+ return job
46
61
  }
47
62
  async function publish(record: BundleCacheRecord, value: CachedSummary) {
48
63
  if (!await exists(record.id)) return
@@ -92,6 +107,9 @@ export function createBundleSummaryCache(storage: BundleCacheStorage, build: (re
92
107
  // backfill elsewhere must not delay removing these bytes or responding.
93
108
  await active.get(id)
94
109
  for (const [hash, entry] of entries) if (entry.id === id) entries.delete(hash)
110
+ // Do not wait for remote reads during deletion. Their callers still own
111
+ // and drain them, but a late response must not repopulate this cache.
112
+ for (const [hash, entry] of reads) if (entry.id === id) reads.delete(hash)
95
113
  },
96
114
  }
97
115
  }
@@ -7,6 +7,8 @@
7
7
  // report auto-links to its bundle.
8
8
  import { createHash } from 'node:crypto'
9
9
  import type { Buffer } from 'node:buffer'
10
+ import { StringDecoder } from 'node:string_decoder'
11
+ import { createBrotliDecompress } from 'node:zlib'
10
12
 
11
13
  // `sha512-<base64>` identity for a bundle's bytes. MUST stay byte-identical to
12
14
  // the client's common/integrity.js (SHA-512 → standard base64 WITH padding) so
@@ -21,12 +23,75 @@ export function bundleIntegrity(bytes: Buffer): string {
21
23
  // `bundleKind`: sourcemap (.map) or stasis (stasis.code.br / .stasis.code.br).
22
24
  // null for anything else (stored anyway; the kind is informational).
23
25
  export function bundleKind(filename: string): 'sourcemap' | 'stasis' | null {
24
- const lower = filename.toLowerCase()
26
+ // Browsers insert a duplicate-download counter before the final extension.
27
+ const lower = filename.toLowerCase().replace(/ \(\d+\)(?=\.[^.]*$)/u, '')
25
28
  if (lower.endsWith('.map')) return 'sourcemap'
26
29
  if (lower === 'stasis.code.br' || lower.endsWith('.stasis.code.br')) return 'stasis'
27
30
  return null
28
31
  }
29
32
 
33
+ interface BundleRepo { github?: unknown; directory?: unknown }
34
+ const HEADER_KEYS = new Set(['version', 'config', 'repo', 'package'])
35
+ const MAX_HEADER_BYTES = 64 * 1024
36
+
37
+ // Stasis writes origin metadata before entries, formats and source bodies.
38
+ // undefined means another chunk is needed; null means the header has no repo.
39
+ function headerRepo(text: string): BundleRepo | null | undefined {
40
+ let depth = 0, escaped = false, expectKey = false, quoted = false
41
+ let key = '', start = 0, valueStart = 0
42
+ for (let i = 0; i < text.length; i++) {
43
+ const c = text[i]
44
+ if (quoted) {
45
+ if (escaped) { escaped = false; continue }
46
+ if (c === '\\') { escaped = true; continue }
47
+ if (c !== '"') continue
48
+ quoted = false
49
+ if (depth === 1 && expectKey) {
50
+ key = JSON.parse(text.slice(start, i + 1)) as string
51
+ if (!HEADER_KEYS.has(key)) return null
52
+ expectKey = false
53
+ }
54
+ continue
55
+ }
56
+ if (c === '"') { quoted = true; start = i; continue }
57
+ if (c === '{' || c === '[') {
58
+ if (depth === 0 && c !== '{') return null
59
+ depth++
60
+ if (depth === 1) expectKey = true
61
+ } else if (depth === 1 && (c === ',' || c === '}')) {
62
+ if (key === 'repo') {
63
+ const repo: unknown = JSON.parse(text.slice(valueStart, i))
64
+ return repo != null && typeof repo === 'object' && !Array.isArray(repo) ? repo as BundleRepo : null
65
+ }
66
+ if (c === '}') return null
67
+ expectKey = true
68
+ } else if (c === '}' || c === ']') depth--
69
+ else if (depth === 1 && c === ':') valueStart = i + 1
70
+ else if (depth === 0 && c?.trim()) return null
71
+ }
72
+ return undefined
73
+ }
74
+
75
+ // Decode only a bounded prefix, stopping as soon as repo is read or the
76
+ // header ends. Unknown/malformed archives remain uploadable as opaque bytes.
77
+ export async function bundleRepo(bytes: Buffer): Promise<BundleRepo | null> {
78
+ const stream = createBrotliDecompress({ chunkSize: 4096 })
79
+ const decoder = new StringDecoder('utf8')
80
+ let size = 0, text = ''
81
+ stream.end(bytes)
82
+ try {
83
+ for await (const chunk of stream) {
84
+ size += chunk.length
85
+ if (size > MAX_HEADER_BYTES) return null
86
+ text += decoder.write(chunk)
87
+ const repo = headerRepo(text)
88
+ if (repo !== undefined) return repo
89
+ }
90
+ return null
91
+ } catch { return null }
92
+ finally { stream.destroy() }
93
+ }
94
+
30
95
  // Extract the bundle integrities a report declares (its top-level
31
96
  // `bundleHashes`). Non-JSON reports (markdown / CSV) or a missing field → [].
32
97
  export function reportBundleHashes(bytes: Buffer): string[] {
@@ -1,9 +1,10 @@
1
1
  import type { Buffer } from 'node:buffer'
2
- import { randomUUID } from 'node:crypto'
3
- import { mkdir, open, rename, rm, stat, writeFile } from 'node:fs/promises'
4
- import { dirname, join } from 'node:path'
5
2
  import type { OpenedBlob } from './blob-store.ts'
6
3
 
4
+ export class CacheMissError extends Error {
5
+ constructor() { super('Managed cache unavailable') }
6
+ }
7
+
7
8
  // Keys are internal relative paths; delete removes a whole directory of variants.
8
9
  export interface CacheStorage {
9
10
  exists(key: string): Promise<boolean>
@@ -16,25 +17,3 @@ export function validateCacheKey(key: string): string {
16
17
  if (!key.split('/').every(part => /^[a-z0-9][a-z0-9.-]*$/iu.test(part))) throw new Error('Invalid cache key')
17
18
  return key
18
19
  }
19
-
20
- export function createDiskCacheStorage(dir: string): CacheStorage {
21
- const path = (key: string) => join(dir, validateCacheKey(key))
22
- return {
23
- async exists(key) {
24
- try { await stat(path(key)); return true }
25
- catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; throw err }
26
- },
27
- async put(key, bytes) {
28
- const target = path(key), temp = `${target}.${randomUUID()}.tmp`
29
- await mkdir(dirname(target), { recursive: true })
30
- try { await writeFile(temp, bytes); await rename(temp, target) }
31
- finally { await rm(temp, { force: true }) }
32
- },
33
- async open(key) {
34
- const file = await open(path(key), 'r')
35
- try { return { size: (await file.stat()).size, stream: file.createReadStream() } }
36
- catch (err) { await file.close(); throw err }
37
- },
38
- async delete(prefix) { await rm(path(prefix), { recursive: true, force: true }) },
39
- }
40
- }
@@ -9,5 +9,13 @@
9
9
  // Imported here rather than run as the process entry, so index.ts's
10
10
  // `import.meta.main` auto-start gate stays off and we call `start()` ourselves.
11
11
  import '../strip-types-loader.js'
12
- const { start } = await import('./index.ts')
13
- await start()
12
+ const command = process.argv[2]
13
+ if (command === '--storage-encryption-status') {
14
+ const { storageCommand } = await import('./storage-command.ts')
15
+ await storageCommand(process.argv[3])
16
+ } else if (command) {
17
+ throw new Error(`Unknown command: ${command}`)
18
+ } else {
19
+ const { start } = await import('./index.ts')
20
+ await start()
21
+ }
@@ -4,6 +4,7 @@
4
4
  import { env } from 'node:process'
5
5
  import { databaseUrls } from '../server-common/database-config.ts'
6
6
  import { MAX_UPLOAD_BYTES } from './uploads.ts'
7
+ import { parseStorageKey } from '../server-common/storage-crypto.ts'
7
8
 
8
9
  const LOOPBACK_HOSTS = new Set(['127.0.0.1', '::1', 'localhost'])
9
10
 
@@ -13,6 +14,10 @@ export interface ManagedConfig {
13
14
  dbPath: string
14
15
  neonUrl?: string | null
15
16
  blobToken?: string | null
17
+ storageEncryptionKey?: string | null
18
+ storageEncryptionMigrate?: boolean
19
+ storageEncryptionMigrateMaxMs?: number
20
+ vercelPreview?: boolean
16
21
  serverless?: boolean
17
22
  debug: boolean
18
23
  allowShare: boolean
@@ -88,6 +93,10 @@ export function loadManagedConfig({ combined = false } = {}): ManagedConfig {
88
93
  const serverless = env['VERCEL'] === '1'
89
94
  const neonUrl = databaseUrls({ combined }).managed
90
95
  const blobToken = env['BLOB_READ_WRITE_TOKEN'] || null
96
+ const storageEncryptionKey = env['MANAGED_STORAGE_ENCRYPTION_KEY'] || null
97
+ parseStorageKey(storageEncryptionKey)
98
+ const storageEncryptionMigrate = env['MANAGED_STORAGE_ENCRYPTION_MIGRATE'] === '1'
99
+ if (storageEncryptionMigrate && !storageEncryptionKey) fail('MANAGED_STORAGE_ENCRYPTION_MIGRATE requires MANAGED_STORAGE_ENCRYPTION_KEY.')
91
100
  if (serverless && !neonUrl) fail('Vercel managed mode requires DATABASE_URL or MANAGED_DATABASE_URL.')
92
101
  if (neonUrl && !blobToken) fail('Managed Neon mode requires BLOB_READ_WRITE_TOKEN.')
93
102
  const host = env['HOST'] ?? '127.0.0.1'
@@ -105,7 +114,9 @@ export function loadManagedConfig({ combined = false } = {}): ManagedConfig {
105
114
  fail(`SESSION_COOKIE_NAME=${sessionCookieName} uses the __Host- prefix but the callback is not https. Use a non-prefixed name for loopback http dev.`)
106
115
  }
107
116
  return {
108
- neonUrl, blobToken, serverless,
117
+ neonUrl, blobToken, serverless, storageEncryptionKey, storageEncryptionMigrate,
118
+ storageEncryptionMigrateMaxMs: intEnv('MANAGED_STORAGE_ENCRYPTION_MIGRATE_MAX_MS', 150_000, 1, serverless ? 240_000 : 3_600_000),
119
+ vercelPreview: env['VERCEL_ENV'] === 'preview',
109
120
  port: intEnv('PORT', 8765, 0, 65535),
110
121
  host,
111
122
  // DB_PATH belongs to e2e in a combined process. Keep the two stores apart.
@@ -6,7 +6,7 @@ type Content = { repoId: number | null; repoDirectory?: string }
6
6
  // Read grants for each request: neither the manager role nor a cached page
7
7
  // confers repository access. Bundles have repository scope; reports also have
8
8
  // a directory scope. Unassigned content is only available to administrators.
9
- export async function contentAccess(db: ManagedDb, user: StoredUser) {
9
+ export async function contentAccess(db: Pick<ManagedDb, 'listRepoScopesForUser'>, user: StoredUser) {
10
10
  const scopes = user.role === 'admin' ? null : await db.listRepoScopesForUser(user.id)
11
11
  const bundle = (item: Content) => scopes == null || scopes.some(scope => scope.repoId === item.repoId)
12
12
  const report = (item: Content) => scopes == null || scopes.some(scope => scope.repoId === item.repoId