@preventive/triage 1.0.0-alpha.31 → 1.0.0-alpha.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@preventive/triage",
3
- "version": "1.0.0-alpha.31",
3
+ "version": "1.0.0-alpha.33",
4
4
  "description": "Client & relay server for triaging of automated reports",
5
5
  "license": "MIT OR Apache-2.0",
6
6
  "author": {
@@ -139,6 +139,7 @@
139
139
  "server-managed/report-migration.ts",
140
140
  "server-managed/report-query.ts",
141
141
  "server-managed/report-response.ts",
142
+ "server-managed/report-store.ts",
142
143
  "server-managed/report-sources.ts",
143
144
  "server-managed/revisions.ts",
144
145
  "server-managed/repository-browser.ts",
@@ -266,7 +267,7 @@
266
267
  "@exodus/stasis": "1.0.0-beta.5",
267
268
  "@exodus/stasis-core": "1.0.0-beta.5",
268
269
  "@preventive/report": "1.0.0-alpha.5",
269
- "@preventive/upstream": "1.0.0-alpha.7",
270
+ "@preventive/upstream": "1.0.0-alpha.8",
270
271
  "ws": "^8.20.1"
271
272
  },
272
273
  "peerDependencies": {
@@ -4,7 +4,7 @@ import type { Buffer } from 'node:buffer'
4
4
 
5
5
  const compress = promisify(brotliCompress)
6
6
 
7
- // Quality 4 avoids Brotli's expensive default (11) for uploads and cold caches.
8
- export function encodeBrotli(bytes: Uint8Array): Promise<Buffer> {
9
- return compress(bytes, { params: { [constants.BROTLI_PARAM_QUALITY]: 4 } })
7
+ // Bundles and cold caches use quality 4; stored reports use quality 9.
8
+ export function encodeBrotli(bytes: Uint8Array, quality = 4): Promise<Buffer> {
9
+ return compress(bytes, { params: { [constants.BROTLI_PARAM_QUALITY]: quality } })
10
10
  }
@@ -44,7 +44,7 @@ export interface ManagedConfig {
44
44
  githubAppSlug: string | null
45
45
  githubNewIssueLabels?: string
46
46
  // Max accepted size (bytes) for an uploaded report on the "Manage reports"
47
- // page. Reports are findings dumps (JSON / markdown / CSV), small to a few MB.
47
+ // page, before storage compression (default 25 MiB).
48
48
  maxReportBytes: number
49
49
  // Max accepted size (bytes) for an uploaded bundle (sourcemap / stasis
50
50
  // archive). Bundles run larger than reports, so a higher cap (default 200 MiB).
@@ -135,7 +135,7 @@ export function loadManagedConfig({ combined = false } = {}): ManagedConfig {
135
135
  githubAppPrivateKey: normalizePem(env['GITHUB_APP_PRIVATE_KEY']),
136
136
  githubAppSlug: env['GITHUB_APP_SLUG'] ?? null,
137
137
  githubNewIssueLabels: env['GITHUB_NEW_ISSUE_LABELS'] ?? '',
138
- maxReportBytes: intEnv('MAX_REPORT_BYTES', 10_485_760, 1, 104_857_600),
138
+ maxReportBytes: intEnv('MAX_REPORT_BYTES', 26_214_400, 1, 104_857_600),
139
139
  maxBundleBytes: intEnv('MAX_BUNDLE_BYTES', 209_715_200, 1, MAX_UPLOAD_BYTES),
140
140
  triageHistoryLimit: intEnv('TRIAGE_HISTORY_LIMIT', 0, 0, 1_000_000_000),
141
141
  }
@@ -312,6 +312,7 @@ export interface UserTeam {
312
312
  id: string
313
313
  slug: string
314
314
  name: string
315
+ permissions: TeamUserPermissions
315
316
  cacheKey?: string
316
317
  reports: UserTeamReport[]
317
318
  bundles: UserTeamBundle[]
@@ -1337,6 +1338,7 @@ function teamMethods(stmts: ReturnType<typeof prepareStatements>, db: ManagedSql
1337
1338
  }
1338
1339
  return teams.map((t) => ({
1339
1340
  id: t.id, slug: t.slug, name: t.name,
1341
+ permissions: { dependencies: t.dependencies === 1, security: t.security === 1 },
1340
1342
  cacheKey: createHash('sha256').update(JSON.stringify([
1341
1343
  t.dependencies, t.security, scopes.filter(scope => scope.teamId === t.id),
1342
1344
  ])).digest('base64url'),
@@ -21,10 +21,13 @@ const REAP_RETRY_MS = 60_000
21
21
  type ManagedAppOptions = Partial<Pick<ManagedHttpDeps, 'next' | 'serverInfo' | 'isShuttingDown'>>
22
22
 
23
23
  export async function createManagedApp(config: ManagedConfig, options: ManagedAppOptions = {}) {
24
- return await initializeApp(rollback => assembleManagedApp(config, options, rollback))
24
+ // Load the optional peer only for serverless managed deployments. Embedding
25
+ // hosts can discard listener promises, so retain maintenance inside the app.
26
+ const waitUntil = config.serverless ? (await import('@vercel/functions')).waitUntil : undefined
27
+ return await initializeApp(rollback => assembleManagedApp(config, options, rollback, waitUntil))
25
28
  }
26
29
 
27
- async function assembleManagedApp(config: ManagedConfig, options: ManagedAppOptions, rollback: AsyncDisposableStack) {
30
+ async function assembleManagedApp(config: ManagedConfig, options: ManagedAppOptions, rollback: AsyncDisposableStack, waitUntil?: (promise: Promise<unknown>) => void) {
28
31
  const storage = await openManagedStorage(config)
29
32
  const { db, avatarStore, reportStore, bundleStore, bundleCache, reportSourcesCache } = storage
30
33
  rollback.defer(() => db.close())
@@ -87,10 +90,9 @@ async function assembleManagedApp(config: ManagedConfig, options: ManagedAppOpti
87
90
  return reap(true).catch(err => console.warn('managed-reaper: cleanup failed:', err))
88
91
  }
89
92
  async function handleRequest(req: Parameters<typeof routeRequest>[0], res: Parameters<typeof routeRequest>[1]): Promise<void> {
90
- const pendingMaintenance = automaticReap()
91
- // Send the normal response without waiting for storage housekeeping. The
92
- // Vercel entrypoint registers this whole promise with waitUntil; awaiting
93
- // work after res.end() alone cannot extend a serverless invocation.
93
+ // Register before routing can end the response, including when init()'s
94
+ // listener is mounted in Fastify rather than using api/managed.ts.
95
+ const pendingMaintenance = automaticReap(); waitUntil?.(pendingMaintenance)
94
96
  try { await routeRequest(req, res) }
95
97
  finally { await pendingMaintenance }
96
98
  }
@@ -1,5 +1,5 @@
1
1
  import { randomUUID } from 'node:crypto'
2
- import { lstat, mkdir, open, readdir, rename, rm, rmdir, stat } from 'node:fs/promises'
2
+ import { link, lstat, mkdir, open, readdir, rename, rm, rmdir, stat } from 'node:fs/promises'
3
3
  import { dirname, join } from 'node:path'
4
4
  import { type ObjectPage, type RawObjectStorage, objectPath } from './object-storage.ts'
5
5
 
@@ -100,11 +100,15 @@ export function createDiskObjectStorage(dir: string): RawObjectStorage {
100
100
  await file.writeFile(bytes, signal ? { signal } : {})
101
101
  await file.sync(); await file.close()
102
102
  signal?.throwIfAborted()
103
- if (expected !== undefined) {
103
+ if (expected === null) {
104
+ try { await link(temp, target) }
105
+ catch (err) { if ((err as NodeJS.ErrnoException).code === 'EEXIST') return false; throw err }
106
+ } else if (expected === undefined) await rename(temp, target)
107
+ else {
104
108
  try { if (version(await stat(target)) !== expected) return false }
105
109
  catch (err) { if (missing(err)) return false; throw err }
110
+ await rename(temp, target)
106
111
  }
107
- await rename(temp, target)
108
112
  await syncDirectory(dirname(target))
109
113
  return true
110
114
  } finally { await file.close(); await rm(temp, { force: true }) }
@@ -6,7 +6,7 @@ import { type RawObjectStorage, objectPath } from './object-storage.ts'
6
6
  export async function openVercelObjectStorage(token: string, sdk?: VercelBlobSdk): Promise<RawObjectStorage> {
7
7
  const blobs = sdk ?? await loadVercelBlobSdk()
8
8
  const path = (key: string) => `.managed/${objectPath(key)}`
9
- return {
9
+ const storage: RawObjectStorage = {
10
10
  async head(key, signal) {
11
11
  try {
12
12
  const meta = await blobs.head(path(key), { token, ...(signal ? { abortSignal: signal } : {}) })
@@ -32,11 +32,14 @@ export async function openVercelObjectStorage(token: string, sdk?: VercelBlobSdk
32
32
  },
33
33
  async put(key, bytes, signal, expected, sizeHint) {
34
34
  const multipart = (sizeHint ?? (Buffer.isBuffer(bytes) ? bytes.length : Infinity)) >= 5 * 1024 * 1024
35
- try { await blobs.put(path(key), bytes, { token, access: 'private', addRandomSuffix: false, allowOverwrite: true,
35
+ try { await blobs.put(path(key), bytes, { token, access: 'private', addRandomSuffix: false, allowOverwrite: expected !== null,
36
36
  ...(expected ? { ifMatch: expected } : {}), multipart, contentType: 'application/octet-stream', cacheControlMaxAge: 60, ...(signal ? { abortSignal: signal } : {}) })
37
37
  return true
38
38
  } catch (err) {
39
39
  if (blobs.BlobPreconditionFailedError && err instanceof blobs.BlobPreconditionFailedError) return false
40
+ // The SDK uses a generic error for an occupied pathname. Reconcile a
41
+ // create-only failure against storage; the caller verifies the winner.
42
+ if (expected === null && await storage.head(key, signal)) return false
40
43
  throw err
41
44
  }
42
45
  },
@@ -60,4 +63,5 @@ export async function openVercelObjectStorage(token: string, sdk?: VercelBlobSdk
60
63
  cursor: page.hasMore ? page.cursor! : null }
61
64
  },
62
65
  }
66
+ return storage
63
67
  }
@@ -16,7 +16,10 @@ export interface RawObjectStorage {
16
16
  // Persist an observed disk replacement before its SQL migration checkpoint.
17
17
  sync?(key: string, signal?: AbortSignal): Promise<void>
18
18
  // A plaintext size hint selects multipart uploads without buffering streams.
19
- put(key: string, bytes: Buffer | Readable, signal?: AbortSignal, expected?: string, sizeHint?: number): Promise<boolean>
19
+ // Publish atomically only after consuming the body through clean EOF; a body
20
+ // error must preserve the existing object (migration verifies while streaming).
21
+ // A null expected version creates only if absent, without replacing a winner.
22
+ put(key: string, bytes: Buffer | Readable, signal?: AbortSignal, expected?: string | null, sizeHint?: number): Promise<boolean>
20
23
  // Return false if the original version changed; do not delete its replacement.
21
24
  delete(key: string, version?: string, signal?: AbortSignal): Promise<boolean>
22
25
  list(prefix: string, cursor: string | null, limit: number, signal?: AbortSignal): Promise<ObjectPage>
@@ -25,7 +28,7 @@ export interface ObjectStorage {
25
28
  open(key: string): Promise<OpenedBlob | null>
26
29
  get(key: string): Promise<Buffer | null>
27
30
  exists(key: string): Promise<boolean>
28
- put(key: string, bytes: Buffer): Promise<string | null>
31
+ put(key: string, bytes: Buffer, options?: { replaceReport?: boolean }): Promise<string | null>
29
32
  delete(key: string): Promise<void>
30
33
  deletePrefix(prefix: string): Promise<void>
31
34
  }
@@ -0,0 +1,47 @@
1
+ // Compress before the object-storage encryption boundary. Separate paths let
2
+ // legacy arbitrary report bytes remain readable without format sniffing.
3
+ import { Buffer } from 'node:buffer'
4
+ import { Readable } from 'node:stream'
5
+ import { pipeline } from 'node:stream/promises'
6
+ import { createBrotliDecompress } from 'node:zlib'
7
+ import type { BlobStore, OpenedBlob } from './blob-store.ts'
8
+ import { encodeBrotli } from './brotli.ts'
9
+
10
+ async function readBytes(stored: OpenedBlob): Promise<Buffer> {
11
+ const parts: Buffer[] = []
12
+ try {
13
+ for await (const part of stored.stream) parts.push(Buffer.from(part))
14
+ return Buffer.concat(parts)
15
+ } finally { stored.stream.destroy() }
16
+ }
17
+
18
+ export function createReportStore(legacy: BlobStore, compressed: BlobStore,
19
+ replaceLegacy: (id: string, bytes: Buffer) => Promise<unknown>): BlobStore {
20
+ async function open(id: string): Promise<OpenedBlob | null> {
21
+ // Keep the original available until conversion succeeds. Concurrent reads
22
+ // can finish using an already-open stream while another reader converts it.
23
+ const original = await legacy.open(id)
24
+ if (original) {
25
+ const bytes = await readBytes(original)
26
+ await replaceLegacy(id, await encodeBrotli(bytes, 9))
27
+ return { size: bytes.length, stream: Readable.from([bytes]) }
28
+ }
29
+ const stored = await compressed.open(id)
30
+ if (!stored) return null
31
+ const stream = createBrotliDecompress()
32
+ // pipeline carries storage/authentication errors into the decoded stream
33
+ // and closes the stored source when a reader leaves before EOF.
34
+ void pipeline(stored.stream, stream).catch(() => {})
35
+ return { size: null, stream }
36
+ }
37
+ return {
38
+ put: async (id, bytes) => compressed.put(id, await encodeBrotli(bytes, 9)),
39
+ open,
40
+ async get(id) {
41
+ const stored = await open(id)
42
+ if (!stored) return null
43
+ return readBytes(stored)
44
+ },
45
+ async delete(id) { await Promise.all([compressed.delete(id), legacy.delete(id)]) },
46
+ }
47
+ }
@@ -4,7 +4,7 @@ import { Readable } from 'node:stream'
4
4
  import { type StorageKey, decryptStorageStream, encryptStorageStream, wrapStorageValue } from '../server-common/storage-crypto.ts'
5
5
  import { ENCRYPTED_CACHE_PREFIX, type ObjectStorage, type RawObject, type RawObjectStorage, deleteObjects, objectPath, openObjectVersion } from './object-storage.ts'
6
6
  import { type StorageDb, type StorageEncryptionState, type StorageRow, type StorageRowKind, dataKeyIdentity, unwrapDataKey } from './storage-db.ts'
7
- import { inspectStorageObject, storageOwner, storageRowPath, verifyStoragePayload } from './storage-payload.ts'
7
+ import { inspectStorageObject, storageOwner, storageRowPaths, verifyStoragePayload } from './storage-payload.ts'
8
8
 
9
9
  const STORAGE_WRITE_MS = 180_000
10
10
  export const STORAGE_UPLOAD_TTL_MS = 86_400_000
@@ -78,7 +78,7 @@ async function openEncrypted(raw: RawObjectStorage, db: StorageDb, key: StorageK
78
78
  for (let attempt = 0; attempt < 3; attempt++) {
79
79
  const row = await db.getStorageRow(owner.type, owner.id)
80
80
  if (!row) return null
81
- if (!owner.cache && storageRowPath(owner.type, row) !== identity) return null
81
+ if (!owner.cache && !storageRowPaths(owner.type, row).includes(identity)) return null
82
82
  if (owner.cache && !row.dataKey) return null
83
83
  if (!owner.cache && !row.encrypted) state = (await db.getStorageEncryption())!
84
84
  const stored = await raw.open(owner.cache ? encryptedCachePath(identity) : identity)
@@ -104,13 +104,78 @@ async function openEncrypted(raw: RawObjectStorage, db: StorageDb, key: StorageK
104
104
  }
105
105
  // Only a pending row may read plaintext, and only the original upload.
106
106
  // Verify first, then reopen that exact version before exposing bytes.
107
- await verifyStoragePayload(owner.type, row, inspected.stream)
107
+ await verifyStoragePayload(owner.type, row, inspected.stream, undefined, identity)
108
108
  const current = await openObjectVersion(raw, identity, stored.version)
109
109
  if (current) return current
110
110
  }
111
111
  throw new Error('Storage changed repeatedly during read')
112
112
  }
113
113
 
114
+ async function verifyReportConversion(db: StorageDb, identity: string, bytes: Buffer): Promise<string> {
115
+ const owner = storageOwner(identity)
116
+ if (owner?.type !== 'report' || owner.cache || identity !== `reports/${owner.id}.br`) throw new Error('Invalid report conversion')
117
+ const row = await db.getStorageRow('report', owner.id)
118
+ if (!row) throw new Error('Report deleted during compression')
119
+ // Representation changes must preserve the immutable upload hash.
120
+ await verifyStoragePayload('report', row, Readable.from([bytes]), undefined, identity)
121
+ return owner.id
122
+ }
123
+
124
+ async function finishReportConversion(raw: RawObjectStorage, db: StorageDb, id: string, encrypted: boolean): Promise<void> {
125
+ const row = await db.getStorageRow('report', id)
126
+ if (!row) { await raw.delete(`reports/${id}.br`); throw new Error('Report deleted during compression') }
127
+ // Conversion preserves the original row/key. Keep the original until the
128
+ // compressed PUT has succeeded, and mark encrypted only after removing it.
129
+ await raw.delete(`reports/${id}`)
130
+ if (encrypted) await db.markStorageEncrypted('report', id, row.dataKey!)
131
+ }
132
+
133
+ async function existingReportConversion(raw: RawObjectStorage, identity: string, row: StorageRow, key: StorageKey | null,
134
+ encryptedMode: boolean): Promise<{ version: string; encrypted: boolean } | null> {
135
+ const stored = await raw.open(identity)
136
+ if (!stored) return null
137
+ const inspected = await inspectStorageObject(stored)
138
+ // Activation or a concurrent key allocation can win after the SQL read.
139
+ // Retry with fresh policy/row state before interpreting any ciphertext.
140
+ if (inspected.encrypted && (!encryptedMode || !row.dataKey)) {
141
+ inspected.stream.destroy()
142
+ return { version: stored.version, encrypted: true }
143
+ }
144
+ const source = inspected.encrypted ? await decryptOwned(inspected, key!, 'report', row, identity) : inspected
145
+ await verifyStoragePayload('report', row, source.stream, undefined, identity)
146
+ return { version: stored.version, encrypted: inspected.encrypted }
147
+ }
148
+
149
+ async function publishReportConversion(raw: RawObjectStorage, identity: string, bytes: Buffer, row: StorageRow,
150
+ key: StorageKey | null, expected: string | null, encrypted: boolean): Promise<boolean> {
151
+ if (!encrypted) return raw.put(identity, bytes, AbortSignal.timeout(STORAGE_WRITE_MS), expected)
152
+ const dataKey = unwrapDataKey(key!, 'report', row)
153
+ const source = encryptStorageStream(Readable.from([bytes]), dataKey, identity, bytes.length)
154
+ try { return await raw.put(identity, source, AbortSignal.timeout(STORAGE_WRITE_MS), expected, bytes.length) }
155
+ finally { source.destroy(); dataKey.fill(0) }
156
+ }
157
+
158
+ async function convertStoredReport(raw: RawObjectStorage, db: StorageDb, key: StorageKey | null, identity: string, bytes: Buffer,
159
+ mode: (fresh?: boolean) => Promise<StorageEncryptionState | null>): Promise<null> {
160
+ const id = await verifyReportConversion(db, identity, bytes)
161
+ for (let attempt = 0; attempt < 3; attempt++) {
162
+ const state = await mode(true)
163
+ const row = state ? await db.ensureStorageDataKey('report', id) : await db.getStorageRow('report', id)
164
+ if (!row) throw new Error('Report deleted during compression')
165
+ const current = await existingReportConversion(raw, identity, row, key, Boolean(state))
166
+ if (current?.encrypted && (!state || !row.dataKey)) continue
167
+ // A stale plaintext writer must never replace a concurrent encrypted
168
+ // winner. Retain the original on uncertain writes; the next open resumes.
169
+ if ((!current || state && !current.encrypted) &&
170
+ !await publishReportConversion(raw, identity, bytes, row, key, current?.version ?? null, Boolean(state))) continue
171
+ if (!state && await mode(true)) continue
172
+ await raw.sync?.(identity)
173
+ await finishReportConversion(raw, db, id, Boolean(state))
174
+ return null
175
+ }
176
+ throw new Error('Report changed repeatedly during compression')
177
+ }
178
+
114
179
  // Startup enables encryption before constructing stores; validate the persisted
115
180
  // requirement even when this adapter is opened without a key.
116
181
  async function storagePolicy(db: StorageDb, key: StorageKey | null) {
@@ -156,17 +221,18 @@ export async function createEncryptedObjectStorage(raw: RawObjectStorage, db: St
156
221
  const owner = storageOwner(identity), state = await mode()
157
222
  if (state && owner) {
158
223
  const row = await db.getStorageRow(owner.type, owner.id)
159
- if (!row || (owner.cache ? !row.dataKey : storageRowPath(owner.type, row) !== identity)) return false
224
+ if (!row || (owner.cache ? !row.dataKey : !storageRowPaths(owner.type, row).includes(identity))) return false
160
225
  }
161
226
  // Existence is metadata only; open() authenticates the actual bytes.
162
227
  return await raw.head(state && owner?.cache ? encryptedCachePath(identity) : identity) !== null
163
228
  },
164
- async put(identity, bytes) {
229
+ async put(identity, bytes, { replaceReport = false } = {}) {
165
230
  logicalKey(identity)
231
+ if (replaceReport) return convertStoredReport(raw, db, key, identity, bytes, mode)
166
232
  const state = await mode()
233
+ const owner = storageOwner(identity)
167
234
  if (identity.startsWith('avatars/')) { await raw.put(identity, bytes); return null }
168
235
  if (!state) return putPlaintext(raw, identity, bytes, () => mode(true))
169
- const owner = storageOwner(identity)
170
236
  let dataKey: Buffer, wrapped: string | null = null
171
237
  if (owner?.cache) {
172
238
  const row = await db.ensureStorageDataKey(owner.type, owner.id)
@@ -1,8 +1,9 @@
1
+ import type { Buffer } from 'node:buffer'
1
2
  import { type StorageKey, decryptStorageStream, encryptStorageStream } from '../server-common/storage-crypto.ts'
2
3
  import { ENCRYPTED_CACHE_PREFIX, type RawObjectStorage, deleteObjects, isBlobId, openObjectVersion } from './object-storage.ts'
3
- import { type StorageDb, type StorageEncryptionState, type StorageMigrationRow, unwrapDataKey } from './storage-db.ts'
4
+ import { type StorageDb, type StorageEncryptionState, type StorageMigrationRow, type StorageRow, type StorageRowKind, unwrapDataKey } from './storage-db.ts'
4
5
  import { STORAGE_UPLOAD_TTL_MS } from './storage-encryption.ts'
5
- import { StoragePayloadError, inspectStorageObject, storageOwner, storageRowPath, verifyStoragePayload } from './storage-payload.ts'
6
+ import { StoragePayloadError, inspectStorageObject, storageOwner, storageRowPaths, verifiedStoragePayload, verifyStoragePayload } from './storage-payload.ts'
6
7
 
7
8
  async function replacementFailure(raw: RawObjectStorage, identity: string, version: string, signal: AbortSignal): Promise<string> {
8
9
  const format = (value: string) => value.startsWith('W/') ? 'weak' : value.startsWith('"') && value.endsWith('"') ? 'quoted' : 'unquoted'
@@ -20,56 +21,78 @@ async function replacementFailure(raw: RawObjectStorage, identity: string, versi
20
21
  return `${reason}; metadata=${format(current.version)}; comparison=${comparison}`
21
22
  }
22
23
 
23
- async function migrateRow(raw: RawObjectStorage, db: StorageDb, key: StorageKey, item: StorageMigrationRow, signal: AbortSignal): Promise<string | void> {
24
+ type MigrationPhase = 'prepare-key' | 'open-payload' | 'verify-existing' | 'encrypt-upload' | 'verify-replacement' | 'checkpoint'
25
+ async function migratePayload(raw: RawObjectStorage, db: StorageDb, type: StorageRowKind, row: StorageRow,
26
+ dataKey: Buffer, identity: string, signal: AbortSignal, progress: (phase: MigrationPhase) => void): Promise<boolean | string | null> {
27
+ progress('open-payload')
28
+ const source = await raw.open(identity, signal)
29
+ if (!source) return false
30
+ let inspected = await inspectStorageObject(source)
31
+ if (inspected.encrypted) {
32
+ progress('verify-existing')
33
+ // A previous PUT may have completed before a timeout or lost SQL ack.
34
+ try {
35
+ const decoded = await decryptStorageStream(inspected.stream, dataKey, identity)
36
+ await verifyStoragePayload(type, row, decoded.stream, signal, identity)
37
+ } catch {
38
+ signal.throwIfAborted()
39
+ // Legacy arbitrary bytes can share the magic prefix. A matching
40
+ // original upload hash is required before encrypting them as plaintext.
41
+ const legacy = await openObjectVersion(raw, identity, source.version, signal)
42
+ if (!legacy) return 'Payload disappeared or changed version before plaintext verification'
43
+ inspected = { ...legacy, encrypted: false }
44
+ }
45
+ }
46
+ if (!inspected.encrypted) {
47
+ progress('encrypt-upload')
48
+ signal.throwIfAborted()
49
+ const verified = verifiedStoragePayload(type, row, inspected.stream, signal, identity)
50
+ const encrypted = encryptStorageStream(verified, dataKey, identity, inspected.size)
51
+ let replaced: boolean
52
+ try { replaced = await raw.put(identity, encrypted, signal, source.version, inspected.size ?? undefined) }
53
+ // A provider may wrap a request-body error; keep safe payload diagnostics.
54
+ catch (err) { throw verified.errored ?? err }
55
+ finally { encrypted.destroy(); verified.destroy(); inspected.stream.destroy(); source.stream.destroy() }
56
+ if (!replaced) return await replacementFailure(raw, identity, source.version, signal)
57
+ // Do not mark the row encrypted until the stored bytes authenticate and
58
+ // match the original upload (including Brotli decompression).
59
+ progress('verify-replacement')
60
+ const replacement = await raw.open(identity, signal)
61
+ if (!replacement) {
62
+ if (!await db.getStorageRow(type, row.id)) return null
63
+ throw new StoragePayloadError('Migrated payload unavailable')
64
+ }
65
+ const decoded = await decryptStorageStream(replacement.stream, dataKey, identity)
66
+ await verifyStoragePayload(type, row, decoded.stream, signal, identity)
67
+ }
68
+ signal.throwIfAborted()
69
+ // Persist observed replacements, including both report representations
70
+ // if they coexist, before marking the whole row encrypted.
71
+ progress('checkpoint')
72
+ await raw.sync?.(identity, signal)
73
+ return true
74
+ }
75
+
76
+ async function migrateRow(raw: RawObjectStorage, db: StorageDb, key: StorageKey, item: StorageMigrationRow, signal: AbortSignal,
77
+ progress: (phase: MigrationPhase) => void): Promise<string | void> {
24
78
  if (item.type === 'user') { await db.migrateStorageUserTokens(item.id); return }
79
+ progress('prepare-key')
25
80
  const row = await db.ensureStorageDataKey(item.type, item.id)
26
81
  if (!row || row.encrypted) return
27
- const dataKey = unwrapDataKey(key, item.type, row), identity = storageRowPath(item.type, row)
82
+ const dataKey = unwrapDataKey(key, item.type, row)
28
83
  try {
29
- const source = await raw.open(identity, signal)
30
- if (!source) {
84
+ let found = false
85
+ for (const identity of storageRowPaths(item.type, row)) {
86
+ const result = await migratePayload(raw, db, item.type, row, dataKey, identity, signal, progress)
87
+ if (result === null) return
88
+ if (typeof result === 'string') return result
89
+ found ||= result
90
+ }
91
+ if (!found) {
31
92
  if (!await db.getStorageRow(item.type, item.id)) return
32
93
  throw new StoragePayloadError('Migration payload unavailable')
33
94
  }
34
- let inspected = await inspectStorageObject(source)
35
- if (inspected.encrypted) {
36
- // A previous PUT may have completed before a timeout or lost SQL ack.
37
- try {
38
- const decoded = await decryptStorageStream(inspected.stream, dataKey, identity)
39
- await verifyStoragePayload(item.type, row, decoded.stream, signal)
40
- } catch {
41
- signal.throwIfAborted()
42
- // Legacy arbitrary bytes can share the magic prefix. A matching
43
- // original upload hash is required before encrypting them as plaintext.
44
- const legacy = await openObjectVersion(raw, identity, source.version, signal)
45
- if (!legacy) return 'Payload disappeared or changed version before plaintext verification'
46
- inspected = { ...legacy, encrypted: false }
47
- }
48
- }
49
- if (!inspected.encrypted) {
50
- await verifyStoragePayload(item.type, row, inspected.stream, signal)
51
- signal.throwIfAborted()
52
- const current = await openObjectVersion(raw, identity, source.version, signal)
53
- if (!current) return 'Payload disappeared or changed version before encryption'
54
- const encrypted = encryptStorageStream(current.stream, dataKey, identity, current.size)
55
- let replaced: boolean
56
- try { replaced = await raw.put(identity, encrypted, signal, source.version, current.size ?? undefined) }
57
- finally { encrypted.destroy(); current.stream.destroy() }
58
- if (!replaced) return await replacementFailure(raw, identity, source.version, signal)
59
- // Do not mark the row encrypted until the stored bytes authenticate and
60
- // match the original upload (including decompression for sourcemaps).
61
- const replacement = await raw.open(identity, signal)
62
- if (!replacement) {
63
- if (!await db.getStorageRow(item.type, item.id)) return
64
- throw new StoragePayloadError('Migrated payload unavailable')
65
- }
66
- const decoded = await decryptStorageStream(replacement.stream, dataKey, identity)
67
- await verifyStoragePayload(item.type, row, decoded.stream, signal)
68
- }
69
95
  signal.throwIfAborted()
70
- // Another process may have renamed ciphertext but stopped before syncing
71
- // the directory. Persist the observed rename even on the resume path.
72
- await raw.sync?.(identity, signal)
73
96
  await db.markStorageEncrypted(item.type, item.id, row.dataKey!)
74
97
  } finally { dataKey.fill(0) }
75
98
  }
@@ -108,7 +131,7 @@ async function cleanupLegacy(raw: RawObjectStorage, db: StorageDb, state: Storag
108
131
  const { owner, temporary } = target
109
132
  if (!temporary && !owner.cache) {
110
133
  const row = await db.getStorageRow(owner.type, owner.id)
111
- if (row && storageRowPath(owner.type, row) === object.key) continue
134
+ if (row && storageRowPaths(owner.type, row).includes(object.key)) continue
112
135
  }
113
136
  const stored = await raw.head(object.key, signal)
114
137
  if (!stored) continue
@@ -176,8 +199,17 @@ export async function migrateStorage(raw: RawObjectStorage, db: StorageDb, key:
176
199
  for (const row of rows) {
177
200
  if (Date.now() >= deadline || signal.aborted) break
178
201
  console.info('managed-storage-migration-row-start:', JSON.stringify({ type: row.type, id: row.id }))
202
+ const startedAt = Date.now()
203
+ let phase: MigrationPhase | 'tokens' = 'tokens'
204
+ const logPhase = (aborted = false) => console.info('managed-storage-migration-row-phase:', JSON.stringify({
205
+ type: row.type, id: row.id, phase, elapsedMs: Date.now() - startedAt, ...(aborted ? { aborted: true } : {}),
206
+ }))
207
+ // Log directly on cancellation even if an underlying operation stalls
208
+ // while unwinding. These fixed phases contain no object/provider data.
209
+ const aborted = () => logPhase(true)
210
+ signal.addEventListener('abort', aborted, { once: true })
179
211
  try {
180
- const pending = await migrateRow(raw, db, key, row, signal)
212
+ const pending = await migrateRow(raw, db, key, row, signal, next => { phase = next; logPhase() })
181
213
  if (pending) failures.push(rowFailure(row, pending))
182
214
  }
183
215
  catch (err) {
@@ -192,6 +224,7 @@ export async function migrateStorage(raw: RawObjectStorage, db: StorageDb, key:
192
224
  errors.push(new Error(`Could not migrate ${row.type} ${row.id}`, { cause: err }))
193
225
  } else failures.push(rowFailure(row, failure))
194
226
  }
227
+ finally { signal.removeEventListener('abort', aborted) }
195
228
  if (stopping?.aborted) break
196
229
  await db.advanceStorageMigration(state.cursor, row.position)
197
230
  processed++
@@ -1,6 +1,6 @@
1
1
  import { Buffer } from 'node:buffer'
2
2
  import { createHash } from 'node:crypto'
3
- import { Readable, Writable } from 'node:stream'
3
+ import { PassThrough, Readable, Writable } from 'node:stream'
4
4
  import { pipeline } from 'node:stream/promises'
5
5
  import { createBrotliDecompress } from 'node:zlib'
6
6
  import { STORAGE_MAGIC } from '../server-common/storage-crypto.ts'
@@ -11,13 +11,14 @@ import type { StorageRow, StorageRowKind } from './storage-db.ts'
11
11
  // not fix it. Keep this distinct from database, storage and cancellation errors.
12
12
  export class StoragePayloadError extends Error {}
13
13
 
14
- export function storageRowPath(type: StorageRowKind, row: Pick<StorageRow, 'id' | 'kind'>): string {
15
- return `${type === 'report' ? 'reports' : 'bundles'}/${row.id}${type === 'bundle' && row.kind === 'sourcemap' ? '.map.br' : ''}`
14
+ export function storageRowPaths(type: StorageRowKind, row: Pick<StorageRow, 'id' | 'kind'>): string[] {
15
+ return type === 'report' ? [`reports/${row.id}.br`, `reports/${row.id}`]
16
+ : [`bundles/${row.id}${row.kind === 'sourcemap' ? '.map.br' : ''}`]
16
17
  }
17
18
  export function storageOwner(identity: string): { type: StorageRowKind; id: string; cache: boolean } | null {
18
19
  const match = /^(reports|bundles)\/([^/]+)$/u.exec(identity)
19
20
  if (match) {
20
- const id = match[1] === 'bundles' ? match[2]!.replace(/\.map\.br$/u, '') : match[2]!
21
+ const id = match[2]!.replace(match[1] === 'bundles' ? /\.map\.br$/u : /\.br$/u, '')
21
22
  if (isBlobId(id)) return { type: match[1] === 'reports' ? 'report' : 'bundle', id, cache: false }
22
23
  }
23
24
  const cache = /^cache\/(?:bundles|report-sources)\/([a-f\d-]{36})\//u.exec(identity)
@@ -50,23 +51,49 @@ export async function inspectStorageObject(stored: RawObject): Promise<RawObject
50
51
  return { ...stored, stream, encrypted: length === prefix.length && prefix.equals(STORAGE_MAGIC) }
51
52
  }
52
53
 
53
- // Sourcemap integrity describes the original upload, before Brotli storage.
54
+ // Report hashes and sourcemap integrity describe the original upload, before Brotli storage.
54
55
  // Hash incrementally, including decompression, so migration stays streaming.
55
- export async function verifyStoragePayload(type: StorageRowKind, row: StorageRow, source: Readable, signal?: AbortSignal): Promise<void> {
56
+ export async function verifyStoragePayload(type: StorageRowKind, row: StorageRow, source: Readable, signal?: AbortSignal, identity?: string): Promise<void> {
56
57
  const hash = createHash(type === 'report' ? 'sha256' : 'sha512')
57
58
  const sink = new Writable({ write(chunk, _encoding, next) { hash.update(chunk); next() } })
58
59
  const options = signal ? { signal } : {}
59
60
  try {
60
- if (type === 'bundle' && row.kind === 'sourcemap') {
61
+ if (type === 'bundle' && row.kind === 'sourcemap' || type === 'report' && identity?.endsWith('.br')) {
61
62
  await pipeline(source, createBrotliDecompress(), sink, options)
62
63
  } else await pipeline(source, sink, options)
63
64
  } catch (err) {
64
65
  const code = (err as NodeJS.ErrnoException).code ?? ''
65
66
  if (['Z_DATA_ERROR', 'Z_BUF_ERROR'].includes(code) || code.startsWith('ERR__ERROR_FORMAT_')) {
66
- throw new StoragePayloadError('Stored sourcemap cannot be decompressed', { cause: err })
67
+ throw new StoragePayloadError(`Stored ${type === 'report' ? 'report' : 'sourcemap'} cannot be decompressed`, { cause: err })
67
68
  }
68
69
  throw err
69
70
  }
70
71
  const actual = type === 'report' ? hash.digest('base64url') : `sha512-${hash.digest('base64')}`
71
72
  if (actual !== row.hash) throw new StoragePayloadError('Stored payload does not match its upload hash')
72
73
  }
74
+
75
+ // Forward the stored representation with backpressure, but withhold EOF until
76
+ // its upload hash passes. Encryption cannot emit its final frame, and an atomic
77
+ // PUT cannot commit, until verification (including Brotli decompression) ends.
78
+ export function verifiedStoragePayload(type: StorageRowKind, row: StorageRow, source: Readable, signal?: AbortSignal, identity?: string): Readable {
79
+ const input = new PassThrough(), output = new PassThrough()
80
+ const verification = verifyStoragePayload(type, row, input, signal, identity)
81
+ // Propagate asynchronous decompression errors even if the source is stalled.
82
+ void verification.catch(err => { source.destroy(err); output.destroy(err) })
83
+ const forwarding = pipeline(source, async function* (chunks) {
84
+ try {
85
+ for await (const chunk of chunks) {
86
+ await new Promise<void>((resolve, reject) => { input.write(chunk, err => err ? reject(err) : resolve()) })
87
+ yield chunk
88
+ }
89
+ input.end()
90
+ await verification
91
+ } finally {
92
+ input.destroy()
93
+ await verification.catch(() => {})
94
+ }
95
+ }, output, signal ? { signal } : {})
96
+ // pipeline forwards errors to output and tears down input on early cancellation.
97
+ void forwarding.catch(() => {})
98
+ return output
99
+ }
@@ -5,6 +5,7 @@ import type { AvatarStore } from './avatar-store.ts'
5
5
  import type { BlobStore } from './blob-store.ts'
6
6
  import type { BundleCacheStorage } from './bundle-cache.ts'
7
7
  import { createBundleStore } from './bundle-store.ts'
8
+ import { createReportStore } from './report-store.ts'
8
9
  import { CacheMissError, type CacheStorage, validateCacheKey } from './cache-storage.ts'
9
10
  import { type ObjectStorage, isBlobId } from './object-storage.ts'
10
11
 
@@ -71,7 +72,8 @@ export function createManagedStores(objects: ObjectStorage, avatarSidecar: boole
71
72
  delete: value => bundles.delete(id(value)),
72
73
  }
73
74
  return {
74
- reportStore: blobStore(objects, 'reports'),
75
+ reportStore: createReportStore(blobStore(objects, 'reports'), blobStore(objects, 'reports', '.br'),
76
+ (value, bytes) => objects.put(`reports/${id(value)}.br`, bytes, { replaceReport: true })),
75
77
  bundleStore: createBundleStore(blobStore(objects, 'bundles'), blobStore(objects, 'bundles', '.map.br')),
76
78
  uploadStore: blobStore(objects, 'uploads'),
77
79
  avatarStore: avatarStore(objects, avatarSidecar),
@@ -149,7 +149,7 @@ export function workspaceShareMethods(db: ManagedSql): WorkspaceShareStore {
149
149
  repositories: await q.repositories.all(team.id) as WorkspaceShareSnapshot['repositories'],
150
150
  reports: rows.map(row => ({ id: row.id, filename: row.filename, byteSize: row.byteSize, sha256: row.sha256,
151
151
  repo: { github: row.github, directory: row.directory }, permissions })),
152
- team: { ...team,
152
+ team: { ...team, permissions,
153
153
  reports: rows.map(row => ({ id: row.id, slug: row.slug, filename: row.filename, analyzer: row.analyzer, repoFullName: row.github, repoDirectory: row.directory,
154
154
  cacheKey: JSON.stringify([row.sha256, row.github, row.directory, row.filename, permissions]) })),
155
155
  bundles: bundleRows.map(row => ({ id: row.id, slug: row.slug, integrity: row.integrity,