@preventive/triage 1.0.0-alpha.31 → 1.0.0-alpha.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/client-managed-import.js +18 -18
- package/out/client-managed.js +9 -9
- package/out/view.css +1 -1
- package/out/view.js +176 -168
- package/package.json +3 -2
- package/server-managed/brotli.ts +3 -3
- package/server-managed/config.ts +2 -2
- package/server-managed/db-methods.ts +2 -0
- package/server-managed/index.ts +8 -6
- package/server-managed/object-storage-disk.ts +7 -3
- package/server-managed/object-storage-vercel.ts +6 -2
- package/server-managed/object-storage.ts +5 -2
- package/server-managed/report-store.ts +47 -0
- package/server-managed/storage-encryption.ts +72 -6
- package/server-managed/storage-maintenance.ts +79 -46
- package/server-managed/storage-payload.ts +35 -8
- package/server-managed/storage-stores.ts +3 -1
- package/server-managed/workspace-shares.ts +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@preventive/triage",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.33",
|
|
4
4
|
"description": "Client & relay server for triaging of automated reports",
|
|
5
5
|
"license": "MIT OR Apache-2.0",
|
|
6
6
|
"author": {
|
|
@@ -139,6 +139,7 @@
|
|
|
139
139
|
"server-managed/report-migration.ts",
|
|
140
140
|
"server-managed/report-query.ts",
|
|
141
141
|
"server-managed/report-response.ts",
|
|
142
|
+
"server-managed/report-store.ts",
|
|
142
143
|
"server-managed/report-sources.ts",
|
|
143
144
|
"server-managed/revisions.ts",
|
|
144
145
|
"server-managed/repository-browser.ts",
|
|
@@ -266,7 +267,7 @@
|
|
|
266
267
|
"@exodus/stasis": "1.0.0-beta.5",
|
|
267
268
|
"@exodus/stasis-core": "1.0.0-beta.5",
|
|
268
269
|
"@preventive/report": "1.0.0-alpha.5",
|
|
269
|
-
"@preventive/upstream": "1.0.0-alpha.
|
|
270
|
+
"@preventive/upstream": "1.0.0-alpha.8",
|
|
270
271
|
"ws": "^8.20.1"
|
|
271
272
|
},
|
|
272
273
|
"peerDependencies": {
|
package/server-managed/brotli.ts
CHANGED
|
@@ -4,7 +4,7 @@ import type { Buffer } from 'node:buffer'
|
|
|
4
4
|
|
|
5
5
|
const compress = promisify(brotliCompress)
|
|
6
6
|
|
|
7
|
-
//
|
|
8
|
-
export function encodeBrotli(bytes: Uint8Array): Promise<Buffer> {
|
|
9
|
-
return compress(bytes, { params: { [constants.BROTLI_PARAM_QUALITY]:
|
|
7
|
+
// Bundles and cold caches use quality 4; stored reports use quality 9.
|
|
8
|
+
export function encodeBrotli(bytes: Uint8Array, quality = 4): Promise<Buffer> {
|
|
9
|
+
return compress(bytes, { params: { [constants.BROTLI_PARAM_QUALITY]: quality } })
|
|
10
10
|
}
|
package/server-managed/config.ts
CHANGED
|
@@ -44,7 +44,7 @@ export interface ManagedConfig {
|
|
|
44
44
|
githubAppSlug: string | null
|
|
45
45
|
githubNewIssueLabels?: string
|
|
46
46
|
// Max accepted size (bytes) for an uploaded report on the "Manage reports"
|
|
47
|
-
// page
|
|
47
|
+
// page, before storage compression (default 25 MiB).
|
|
48
48
|
maxReportBytes: number
|
|
49
49
|
// Max accepted size (bytes) for an uploaded bundle (sourcemap / stasis
|
|
50
50
|
// archive). Bundles run larger than reports, so a higher cap (default 200 MiB).
|
|
@@ -135,7 +135,7 @@ export function loadManagedConfig({ combined = false } = {}): ManagedConfig {
|
|
|
135
135
|
githubAppPrivateKey: normalizePem(env['GITHUB_APP_PRIVATE_KEY']),
|
|
136
136
|
githubAppSlug: env['GITHUB_APP_SLUG'] ?? null,
|
|
137
137
|
githubNewIssueLabels: env['GITHUB_NEW_ISSUE_LABELS'] ?? '',
|
|
138
|
-
maxReportBytes: intEnv('MAX_REPORT_BYTES',
|
|
138
|
+
maxReportBytes: intEnv('MAX_REPORT_BYTES', 26_214_400, 1, 104_857_600),
|
|
139
139
|
maxBundleBytes: intEnv('MAX_BUNDLE_BYTES', 209_715_200, 1, MAX_UPLOAD_BYTES),
|
|
140
140
|
triageHistoryLimit: intEnv('TRIAGE_HISTORY_LIMIT', 0, 0, 1_000_000_000),
|
|
141
141
|
}
|
|
@@ -312,6 +312,7 @@ export interface UserTeam {
|
|
|
312
312
|
id: string
|
|
313
313
|
slug: string
|
|
314
314
|
name: string
|
|
315
|
+
permissions: TeamUserPermissions
|
|
315
316
|
cacheKey?: string
|
|
316
317
|
reports: UserTeamReport[]
|
|
317
318
|
bundles: UserTeamBundle[]
|
|
@@ -1337,6 +1338,7 @@ function teamMethods(stmts: ReturnType<typeof prepareStatements>, db: ManagedSql
|
|
|
1337
1338
|
}
|
|
1338
1339
|
return teams.map((t) => ({
|
|
1339
1340
|
id: t.id, slug: t.slug, name: t.name,
|
|
1341
|
+
permissions: { dependencies: t.dependencies === 1, security: t.security === 1 },
|
|
1340
1342
|
cacheKey: createHash('sha256').update(JSON.stringify([
|
|
1341
1343
|
t.dependencies, t.security, scopes.filter(scope => scope.teamId === t.id),
|
|
1342
1344
|
])).digest('base64url'),
|
package/server-managed/index.ts
CHANGED
|
@@ -21,10 +21,13 @@ const REAP_RETRY_MS = 60_000
|
|
|
21
21
|
type ManagedAppOptions = Partial<Pick<ManagedHttpDeps, 'next' | 'serverInfo' | 'isShuttingDown'>>
|
|
22
22
|
|
|
23
23
|
export async function createManagedApp(config: ManagedConfig, options: ManagedAppOptions = {}) {
|
|
24
|
-
|
|
24
|
+
// Load the optional peer only for serverless managed deployments. Embedding
|
|
25
|
+
// hosts can discard listener promises, so retain maintenance inside the app.
|
|
26
|
+
const waitUntil = config.serverless ? (await import('@vercel/functions')).waitUntil : undefined
|
|
27
|
+
return await initializeApp(rollback => assembleManagedApp(config, options, rollback, waitUntil))
|
|
25
28
|
}
|
|
26
29
|
|
|
27
|
-
async function assembleManagedApp(config: ManagedConfig, options: ManagedAppOptions, rollback: AsyncDisposableStack) {
|
|
30
|
+
async function assembleManagedApp(config: ManagedConfig, options: ManagedAppOptions, rollback: AsyncDisposableStack, waitUntil?: (promise: Promise<unknown>) => void) {
|
|
28
31
|
const storage = await openManagedStorage(config)
|
|
29
32
|
const { db, avatarStore, reportStore, bundleStore, bundleCache, reportSourcesCache } = storage
|
|
30
33
|
rollback.defer(() => db.close())
|
|
@@ -87,10 +90,9 @@ async function assembleManagedApp(config: ManagedConfig, options: ManagedAppOpti
|
|
|
87
90
|
return reap(true).catch(err => console.warn('managed-reaper: cleanup failed:', err))
|
|
88
91
|
}
|
|
89
92
|
async function handleRequest(req: Parameters<typeof routeRequest>[0], res: Parameters<typeof routeRequest>[1]): Promise<void> {
|
|
90
|
-
|
|
91
|
-
//
|
|
92
|
-
|
|
93
|
-
// work after res.end() alone cannot extend a serverless invocation.
|
|
93
|
+
// Register before routing can end the response, including when init()'s
|
|
94
|
+
// listener is mounted in Fastify rather than using api/managed.ts.
|
|
95
|
+
const pendingMaintenance = automaticReap(); waitUntil?.(pendingMaintenance)
|
|
94
96
|
try { await routeRequest(req, res) }
|
|
95
97
|
finally { await pendingMaintenance }
|
|
96
98
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto'
|
|
2
|
-
import { lstat, mkdir, open, readdir, rename, rm, rmdir, stat } from 'node:fs/promises'
|
|
2
|
+
import { link, lstat, mkdir, open, readdir, rename, rm, rmdir, stat } from 'node:fs/promises'
|
|
3
3
|
import { dirname, join } from 'node:path'
|
|
4
4
|
import { type ObjectPage, type RawObjectStorage, objectPath } from './object-storage.ts'
|
|
5
5
|
|
|
@@ -100,11 +100,15 @@ export function createDiskObjectStorage(dir: string): RawObjectStorage {
|
|
|
100
100
|
await file.writeFile(bytes, signal ? { signal } : {})
|
|
101
101
|
await file.sync(); await file.close()
|
|
102
102
|
signal?.throwIfAborted()
|
|
103
|
-
if (expected
|
|
103
|
+
if (expected === null) {
|
|
104
|
+
try { await link(temp, target) }
|
|
105
|
+
catch (err) { if ((err as NodeJS.ErrnoException).code === 'EEXIST') return false; throw err }
|
|
106
|
+
} else if (expected === undefined) await rename(temp, target)
|
|
107
|
+
else {
|
|
104
108
|
try { if (version(await stat(target)) !== expected) return false }
|
|
105
109
|
catch (err) { if (missing(err)) return false; throw err }
|
|
110
|
+
await rename(temp, target)
|
|
106
111
|
}
|
|
107
|
-
await rename(temp, target)
|
|
108
112
|
await syncDirectory(dirname(target))
|
|
109
113
|
return true
|
|
110
114
|
} finally { await file.close(); await rm(temp, { force: true }) }
|
|
@@ -6,7 +6,7 @@ import { type RawObjectStorage, objectPath } from './object-storage.ts'
|
|
|
6
6
|
export async function openVercelObjectStorage(token: string, sdk?: VercelBlobSdk): Promise<RawObjectStorage> {
|
|
7
7
|
const blobs = sdk ?? await loadVercelBlobSdk()
|
|
8
8
|
const path = (key: string) => `.managed/${objectPath(key)}`
|
|
9
|
-
|
|
9
|
+
const storage: RawObjectStorage = {
|
|
10
10
|
async head(key, signal) {
|
|
11
11
|
try {
|
|
12
12
|
const meta = await blobs.head(path(key), { token, ...(signal ? { abortSignal: signal } : {}) })
|
|
@@ -32,11 +32,14 @@ export async function openVercelObjectStorage(token: string, sdk?: VercelBlobSdk
|
|
|
32
32
|
},
|
|
33
33
|
async put(key, bytes, signal, expected, sizeHint) {
|
|
34
34
|
const multipart = (sizeHint ?? (Buffer.isBuffer(bytes) ? bytes.length : Infinity)) >= 5 * 1024 * 1024
|
|
35
|
-
try { await blobs.put(path(key), bytes, { token, access: 'private', addRandomSuffix: false, allowOverwrite:
|
|
35
|
+
try { await blobs.put(path(key), bytes, { token, access: 'private', addRandomSuffix: false, allowOverwrite: expected !== null,
|
|
36
36
|
...(expected ? { ifMatch: expected } : {}), multipart, contentType: 'application/octet-stream', cacheControlMaxAge: 60, ...(signal ? { abortSignal: signal } : {}) })
|
|
37
37
|
return true
|
|
38
38
|
} catch (err) {
|
|
39
39
|
if (blobs.BlobPreconditionFailedError && err instanceof blobs.BlobPreconditionFailedError) return false
|
|
40
|
+
// The SDK uses a generic error for an occupied pathname. Reconcile a
|
|
41
|
+
// create-only failure against storage; the caller verifies the winner.
|
|
42
|
+
if (expected === null && await storage.head(key, signal)) return false
|
|
40
43
|
throw err
|
|
41
44
|
}
|
|
42
45
|
},
|
|
@@ -60,4 +63,5 @@ export async function openVercelObjectStorage(token: string, sdk?: VercelBlobSdk
|
|
|
60
63
|
cursor: page.hasMore ? page.cursor! : null }
|
|
61
64
|
},
|
|
62
65
|
}
|
|
66
|
+
return storage
|
|
63
67
|
}
|
|
@@ -16,7 +16,10 @@ export interface RawObjectStorage {
|
|
|
16
16
|
// Persist an observed disk replacement before its SQL migration checkpoint.
|
|
17
17
|
sync?(key: string, signal?: AbortSignal): Promise<void>
|
|
18
18
|
// A plaintext size hint selects multipart uploads without buffering streams.
|
|
19
|
-
|
|
19
|
+
// Publish atomically only after consuming the body through clean EOF; a body
|
|
20
|
+
// error must preserve the existing object (migration verifies while streaming).
|
|
21
|
+
// A null expected version creates only if absent, without replacing a winner.
|
|
22
|
+
put(key: string, bytes: Buffer | Readable, signal?: AbortSignal, expected?: string | null, sizeHint?: number): Promise<boolean>
|
|
20
23
|
// Return false if the original version changed; do not delete its replacement.
|
|
21
24
|
delete(key: string, version?: string, signal?: AbortSignal): Promise<boolean>
|
|
22
25
|
list(prefix: string, cursor: string | null, limit: number, signal?: AbortSignal): Promise<ObjectPage>
|
|
@@ -25,7 +28,7 @@ export interface ObjectStorage {
|
|
|
25
28
|
open(key: string): Promise<OpenedBlob | null>
|
|
26
29
|
get(key: string): Promise<Buffer | null>
|
|
27
30
|
exists(key: string): Promise<boolean>
|
|
28
|
-
put(key: string, bytes: Buffer): Promise<string | null>
|
|
31
|
+
put(key: string, bytes: Buffer, options?: { replaceReport?: boolean }): Promise<string | null>
|
|
29
32
|
delete(key: string): Promise<void>
|
|
30
33
|
deletePrefix(prefix: string): Promise<void>
|
|
31
34
|
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// Compress before the object-storage encryption boundary. Separate paths let
|
|
2
|
+
// legacy arbitrary report bytes remain readable without format sniffing.
|
|
3
|
+
import { Buffer } from 'node:buffer'
|
|
4
|
+
import { Readable } from 'node:stream'
|
|
5
|
+
import { pipeline } from 'node:stream/promises'
|
|
6
|
+
import { createBrotliDecompress } from 'node:zlib'
|
|
7
|
+
import type { BlobStore, OpenedBlob } from './blob-store.ts'
|
|
8
|
+
import { encodeBrotli } from './brotli.ts'
|
|
9
|
+
|
|
10
|
+
async function readBytes(stored: OpenedBlob): Promise<Buffer> {
|
|
11
|
+
const parts: Buffer[] = []
|
|
12
|
+
try {
|
|
13
|
+
for await (const part of stored.stream) parts.push(Buffer.from(part))
|
|
14
|
+
return Buffer.concat(parts)
|
|
15
|
+
} finally { stored.stream.destroy() }
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function createReportStore(legacy: BlobStore, compressed: BlobStore,
|
|
19
|
+
replaceLegacy: (id: string, bytes: Buffer) => Promise<unknown>): BlobStore {
|
|
20
|
+
async function open(id: string): Promise<OpenedBlob | null> {
|
|
21
|
+
// Keep the original available until conversion succeeds. Concurrent reads
|
|
22
|
+
// can finish using an already-open stream while another reader converts it.
|
|
23
|
+
const original = await legacy.open(id)
|
|
24
|
+
if (original) {
|
|
25
|
+
const bytes = await readBytes(original)
|
|
26
|
+
await replaceLegacy(id, await encodeBrotli(bytes, 9))
|
|
27
|
+
return { size: bytes.length, stream: Readable.from([bytes]) }
|
|
28
|
+
}
|
|
29
|
+
const stored = await compressed.open(id)
|
|
30
|
+
if (!stored) return null
|
|
31
|
+
const stream = createBrotliDecompress()
|
|
32
|
+
// pipeline carries storage/authentication errors into the decoded stream
|
|
33
|
+
// and closes the stored source when a reader leaves before EOF.
|
|
34
|
+
void pipeline(stored.stream, stream).catch(() => {})
|
|
35
|
+
return { size: null, stream }
|
|
36
|
+
}
|
|
37
|
+
return {
|
|
38
|
+
put: async (id, bytes) => compressed.put(id, await encodeBrotli(bytes, 9)),
|
|
39
|
+
open,
|
|
40
|
+
async get(id) {
|
|
41
|
+
const stored = await open(id)
|
|
42
|
+
if (!stored) return null
|
|
43
|
+
return readBytes(stored)
|
|
44
|
+
},
|
|
45
|
+
async delete(id) { await Promise.all([compressed.delete(id), legacy.delete(id)]) },
|
|
46
|
+
}
|
|
47
|
+
}
|
|
@@ -4,7 +4,7 @@ import { Readable } from 'node:stream'
|
|
|
4
4
|
import { type StorageKey, decryptStorageStream, encryptStorageStream, wrapStorageValue } from '../server-common/storage-crypto.ts'
|
|
5
5
|
import { ENCRYPTED_CACHE_PREFIX, type ObjectStorage, type RawObject, type RawObjectStorage, deleteObjects, objectPath, openObjectVersion } from './object-storage.ts'
|
|
6
6
|
import { type StorageDb, type StorageEncryptionState, type StorageRow, type StorageRowKind, dataKeyIdentity, unwrapDataKey } from './storage-db.ts'
|
|
7
|
-
import { inspectStorageObject, storageOwner,
|
|
7
|
+
import { inspectStorageObject, storageOwner, storageRowPaths, verifyStoragePayload } from './storage-payload.ts'
|
|
8
8
|
|
|
9
9
|
const STORAGE_WRITE_MS = 180_000
|
|
10
10
|
export const STORAGE_UPLOAD_TTL_MS = 86_400_000
|
|
@@ -78,7 +78,7 @@ async function openEncrypted(raw: RawObjectStorage, db: StorageDb, key: StorageK
|
|
|
78
78
|
for (let attempt = 0; attempt < 3; attempt++) {
|
|
79
79
|
const row = await db.getStorageRow(owner.type, owner.id)
|
|
80
80
|
if (!row) return null
|
|
81
|
-
if (!owner.cache &&
|
|
81
|
+
if (!owner.cache && !storageRowPaths(owner.type, row).includes(identity)) return null
|
|
82
82
|
if (owner.cache && !row.dataKey) return null
|
|
83
83
|
if (!owner.cache && !row.encrypted) state = (await db.getStorageEncryption())!
|
|
84
84
|
const stored = await raw.open(owner.cache ? encryptedCachePath(identity) : identity)
|
|
@@ -104,13 +104,78 @@ async function openEncrypted(raw: RawObjectStorage, db: StorageDb, key: StorageK
|
|
|
104
104
|
}
|
|
105
105
|
// Only a pending row may read plaintext, and only the original upload.
|
|
106
106
|
// Verify first, then reopen that exact version before exposing bytes.
|
|
107
|
-
await verifyStoragePayload(owner.type, row, inspected.stream)
|
|
107
|
+
await verifyStoragePayload(owner.type, row, inspected.stream, undefined, identity)
|
|
108
108
|
const current = await openObjectVersion(raw, identity, stored.version)
|
|
109
109
|
if (current) return current
|
|
110
110
|
}
|
|
111
111
|
throw new Error('Storage changed repeatedly during read')
|
|
112
112
|
}
|
|
113
113
|
|
|
114
|
+
async function verifyReportConversion(db: StorageDb, identity: string, bytes: Buffer): Promise<string> {
|
|
115
|
+
const owner = storageOwner(identity)
|
|
116
|
+
if (owner?.type !== 'report' || owner.cache || identity !== `reports/${owner.id}.br`) throw new Error('Invalid report conversion')
|
|
117
|
+
const row = await db.getStorageRow('report', owner.id)
|
|
118
|
+
if (!row) throw new Error('Report deleted during compression')
|
|
119
|
+
// Representation changes must preserve the immutable upload hash.
|
|
120
|
+
await verifyStoragePayload('report', row, Readable.from([bytes]), undefined, identity)
|
|
121
|
+
return owner.id
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
async function finishReportConversion(raw: RawObjectStorage, db: StorageDb, id: string, encrypted: boolean): Promise<void> {
|
|
125
|
+
const row = await db.getStorageRow('report', id)
|
|
126
|
+
if (!row) { await raw.delete(`reports/${id}.br`); throw new Error('Report deleted during compression') }
|
|
127
|
+
// Conversion preserves the original row/key. Keep the original until the
|
|
128
|
+
// compressed PUT has succeeded, and mark encrypted only after removing it.
|
|
129
|
+
await raw.delete(`reports/${id}`)
|
|
130
|
+
if (encrypted) await db.markStorageEncrypted('report', id, row.dataKey!)
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
async function existingReportConversion(raw: RawObjectStorage, identity: string, row: StorageRow, key: StorageKey | null,
|
|
134
|
+
encryptedMode: boolean): Promise<{ version: string; encrypted: boolean } | null> {
|
|
135
|
+
const stored = await raw.open(identity)
|
|
136
|
+
if (!stored) return null
|
|
137
|
+
const inspected = await inspectStorageObject(stored)
|
|
138
|
+
// Activation or a concurrent key allocation can win after the SQL read.
|
|
139
|
+
// Retry with fresh policy/row state before interpreting any ciphertext.
|
|
140
|
+
if (inspected.encrypted && (!encryptedMode || !row.dataKey)) {
|
|
141
|
+
inspected.stream.destroy()
|
|
142
|
+
return { version: stored.version, encrypted: true }
|
|
143
|
+
}
|
|
144
|
+
const source = inspected.encrypted ? await decryptOwned(inspected, key!, 'report', row, identity) : inspected
|
|
145
|
+
await verifyStoragePayload('report', row, source.stream, undefined, identity)
|
|
146
|
+
return { version: stored.version, encrypted: inspected.encrypted }
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async function publishReportConversion(raw: RawObjectStorage, identity: string, bytes: Buffer, row: StorageRow,
|
|
150
|
+
key: StorageKey | null, expected: string | null, encrypted: boolean): Promise<boolean> {
|
|
151
|
+
if (!encrypted) return raw.put(identity, bytes, AbortSignal.timeout(STORAGE_WRITE_MS), expected)
|
|
152
|
+
const dataKey = unwrapDataKey(key!, 'report', row)
|
|
153
|
+
const source = encryptStorageStream(Readable.from([bytes]), dataKey, identity, bytes.length)
|
|
154
|
+
try { return await raw.put(identity, source, AbortSignal.timeout(STORAGE_WRITE_MS), expected, bytes.length) }
|
|
155
|
+
finally { source.destroy(); dataKey.fill(0) }
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
async function convertStoredReport(raw: RawObjectStorage, db: StorageDb, key: StorageKey | null, identity: string, bytes: Buffer,
|
|
159
|
+
mode: (fresh?: boolean) => Promise<StorageEncryptionState | null>): Promise<null> {
|
|
160
|
+
const id = await verifyReportConversion(db, identity, bytes)
|
|
161
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
162
|
+
const state = await mode(true)
|
|
163
|
+
const row = state ? await db.ensureStorageDataKey('report', id) : await db.getStorageRow('report', id)
|
|
164
|
+
if (!row) throw new Error('Report deleted during compression')
|
|
165
|
+
const current = await existingReportConversion(raw, identity, row, key, Boolean(state))
|
|
166
|
+
if (current?.encrypted && (!state || !row.dataKey)) continue
|
|
167
|
+
// A stale plaintext writer must never replace a concurrent encrypted
|
|
168
|
+
// winner. Retain the original on uncertain writes; the next open resumes.
|
|
169
|
+
if ((!current || state && !current.encrypted) &&
|
|
170
|
+
!await publishReportConversion(raw, identity, bytes, row, key, current?.version ?? null, Boolean(state))) continue
|
|
171
|
+
if (!state && await mode(true)) continue
|
|
172
|
+
await raw.sync?.(identity)
|
|
173
|
+
await finishReportConversion(raw, db, id, Boolean(state))
|
|
174
|
+
return null
|
|
175
|
+
}
|
|
176
|
+
throw new Error('Report changed repeatedly during compression')
|
|
177
|
+
}
|
|
178
|
+
|
|
114
179
|
// Startup enables encryption before constructing stores; validate the persisted
|
|
115
180
|
// requirement even when this adapter is opened without a key.
|
|
116
181
|
async function storagePolicy(db: StorageDb, key: StorageKey | null) {
|
|
@@ -156,17 +221,18 @@ export async function createEncryptedObjectStorage(raw: RawObjectStorage, db: St
|
|
|
156
221
|
const owner = storageOwner(identity), state = await mode()
|
|
157
222
|
if (state && owner) {
|
|
158
223
|
const row = await db.getStorageRow(owner.type, owner.id)
|
|
159
|
-
if (!row || (owner.cache ? !row.dataKey :
|
|
224
|
+
if (!row || (owner.cache ? !row.dataKey : !storageRowPaths(owner.type, row).includes(identity))) return false
|
|
160
225
|
}
|
|
161
226
|
// Existence is metadata only; open() authenticates the actual bytes.
|
|
162
227
|
return await raw.head(state && owner?.cache ? encryptedCachePath(identity) : identity) !== null
|
|
163
228
|
},
|
|
164
|
-
async put(identity, bytes) {
|
|
229
|
+
async put(identity, bytes, { replaceReport = false } = {}) {
|
|
165
230
|
logicalKey(identity)
|
|
231
|
+
if (replaceReport) return convertStoredReport(raw, db, key, identity, bytes, mode)
|
|
166
232
|
const state = await mode()
|
|
233
|
+
const owner = storageOwner(identity)
|
|
167
234
|
if (identity.startsWith('avatars/')) { await raw.put(identity, bytes); return null }
|
|
168
235
|
if (!state) return putPlaintext(raw, identity, bytes, () => mode(true))
|
|
169
|
-
const owner = storageOwner(identity)
|
|
170
236
|
let dataKey: Buffer, wrapped: string | null = null
|
|
171
237
|
if (owner?.cache) {
|
|
172
238
|
const row = await db.ensureStorageDataKey(owner.type, owner.id)
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
+
import type { Buffer } from 'node:buffer'
|
|
1
2
|
import { type StorageKey, decryptStorageStream, encryptStorageStream } from '../server-common/storage-crypto.ts'
|
|
2
3
|
import { ENCRYPTED_CACHE_PREFIX, type RawObjectStorage, deleteObjects, isBlobId, openObjectVersion } from './object-storage.ts'
|
|
3
|
-
import { type StorageDb, type StorageEncryptionState, type StorageMigrationRow, unwrapDataKey } from './storage-db.ts'
|
|
4
|
+
import { type StorageDb, type StorageEncryptionState, type StorageMigrationRow, type StorageRow, type StorageRowKind, unwrapDataKey } from './storage-db.ts'
|
|
4
5
|
import { STORAGE_UPLOAD_TTL_MS } from './storage-encryption.ts'
|
|
5
|
-
import { StoragePayloadError, inspectStorageObject, storageOwner,
|
|
6
|
+
import { StoragePayloadError, inspectStorageObject, storageOwner, storageRowPaths, verifiedStoragePayload, verifyStoragePayload } from './storage-payload.ts'
|
|
6
7
|
|
|
7
8
|
async function replacementFailure(raw: RawObjectStorage, identity: string, version: string, signal: AbortSignal): Promise<string> {
|
|
8
9
|
const format = (value: string) => value.startsWith('W/') ? 'weak' : value.startsWith('"') && value.endsWith('"') ? 'quoted' : 'unquoted'
|
|
@@ -20,56 +21,78 @@ async function replacementFailure(raw: RawObjectStorage, identity: string, versi
|
|
|
20
21
|
return `${reason}; metadata=${format(current.version)}; comparison=${comparison}`
|
|
21
22
|
}
|
|
22
23
|
|
|
23
|
-
|
|
24
|
+
type MigrationPhase = 'prepare-key' | 'open-payload' | 'verify-existing' | 'encrypt-upload' | 'verify-replacement' | 'checkpoint'
|
|
25
|
+
async function migratePayload(raw: RawObjectStorage, db: StorageDb, type: StorageRowKind, row: StorageRow,
|
|
26
|
+
dataKey: Buffer, identity: string, signal: AbortSignal, progress: (phase: MigrationPhase) => void): Promise<boolean | string | null> {
|
|
27
|
+
progress('open-payload')
|
|
28
|
+
const source = await raw.open(identity, signal)
|
|
29
|
+
if (!source) return false
|
|
30
|
+
let inspected = await inspectStorageObject(source)
|
|
31
|
+
if (inspected.encrypted) {
|
|
32
|
+
progress('verify-existing')
|
|
33
|
+
// A previous PUT may have completed before a timeout or lost SQL ack.
|
|
34
|
+
try {
|
|
35
|
+
const decoded = await decryptStorageStream(inspected.stream, dataKey, identity)
|
|
36
|
+
await verifyStoragePayload(type, row, decoded.stream, signal, identity)
|
|
37
|
+
} catch {
|
|
38
|
+
signal.throwIfAborted()
|
|
39
|
+
// Legacy arbitrary bytes can share the magic prefix. A matching
|
|
40
|
+
// original upload hash is required before encrypting them as plaintext.
|
|
41
|
+
const legacy = await openObjectVersion(raw, identity, source.version, signal)
|
|
42
|
+
if (!legacy) return 'Payload disappeared or changed version before plaintext verification'
|
|
43
|
+
inspected = { ...legacy, encrypted: false }
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
if (!inspected.encrypted) {
|
|
47
|
+
progress('encrypt-upload')
|
|
48
|
+
signal.throwIfAborted()
|
|
49
|
+
const verified = verifiedStoragePayload(type, row, inspected.stream, signal, identity)
|
|
50
|
+
const encrypted = encryptStorageStream(verified, dataKey, identity, inspected.size)
|
|
51
|
+
let replaced: boolean
|
|
52
|
+
try { replaced = await raw.put(identity, encrypted, signal, source.version, inspected.size ?? undefined) }
|
|
53
|
+
// A provider may wrap a request-body error; keep safe payload diagnostics.
|
|
54
|
+
catch (err) { throw verified.errored ?? err }
|
|
55
|
+
finally { encrypted.destroy(); verified.destroy(); inspected.stream.destroy(); source.stream.destroy() }
|
|
56
|
+
if (!replaced) return await replacementFailure(raw, identity, source.version, signal)
|
|
57
|
+
// Do not mark the row encrypted until the stored bytes authenticate and
|
|
58
|
+
// match the original upload (including Brotli decompression).
|
|
59
|
+
progress('verify-replacement')
|
|
60
|
+
const replacement = await raw.open(identity, signal)
|
|
61
|
+
if (!replacement) {
|
|
62
|
+
if (!await db.getStorageRow(type, row.id)) return null
|
|
63
|
+
throw new StoragePayloadError('Migrated payload unavailable')
|
|
64
|
+
}
|
|
65
|
+
const decoded = await decryptStorageStream(replacement.stream, dataKey, identity)
|
|
66
|
+
await verifyStoragePayload(type, row, decoded.stream, signal, identity)
|
|
67
|
+
}
|
|
68
|
+
signal.throwIfAborted()
|
|
69
|
+
// Persist observed replacements, including both report representations
|
|
70
|
+
// if they coexist, before marking the whole row encrypted.
|
|
71
|
+
progress('checkpoint')
|
|
72
|
+
await raw.sync?.(identity, signal)
|
|
73
|
+
return true
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
async function migrateRow(raw: RawObjectStorage, db: StorageDb, key: StorageKey, item: StorageMigrationRow, signal: AbortSignal,
|
|
77
|
+
progress: (phase: MigrationPhase) => void): Promise<string | void> {
|
|
24
78
|
if (item.type === 'user') { await db.migrateStorageUserTokens(item.id); return }
|
|
79
|
+
progress('prepare-key')
|
|
25
80
|
const row = await db.ensureStorageDataKey(item.type, item.id)
|
|
26
81
|
if (!row || row.encrypted) return
|
|
27
|
-
const dataKey = unwrapDataKey(key, item.type, row)
|
|
82
|
+
const dataKey = unwrapDataKey(key, item.type, row)
|
|
28
83
|
try {
|
|
29
|
-
|
|
30
|
-
|
|
84
|
+
let found = false
|
|
85
|
+
for (const identity of storageRowPaths(item.type, row)) {
|
|
86
|
+
const result = await migratePayload(raw, db, item.type, row, dataKey, identity, signal, progress)
|
|
87
|
+
if (result === null) return
|
|
88
|
+
if (typeof result === 'string') return result
|
|
89
|
+
found ||= result
|
|
90
|
+
}
|
|
91
|
+
if (!found) {
|
|
31
92
|
if (!await db.getStorageRow(item.type, item.id)) return
|
|
32
93
|
throw new StoragePayloadError('Migration payload unavailable')
|
|
33
94
|
}
|
|
34
|
-
let inspected = await inspectStorageObject(source)
|
|
35
|
-
if (inspected.encrypted) {
|
|
36
|
-
// A previous PUT may have completed before a timeout or lost SQL ack.
|
|
37
|
-
try {
|
|
38
|
-
const decoded = await decryptStorageStream(inspected.stream, dataKey, identity)
|
|
39
|
-
await verifyStoragePayload(item.type, row, decoded.stream, signal)
|
|
40
|
-
} catch {
|
|
41
|
-
signal.throwIfAborted()
|
|
42
|
-
// Legacy arbitrary bytes can share the magic prefix. A matching
|
|
43
|
-
// original upload hash is required before encrypting them as plaintext.
|
|
44
|
-
const legacy = await openObjectVersion(raw, identity, source.version, signal)
|
|
45
|
-
if (!legacy) return 'Payload disappeared or changed version before plaintext verification'
|
|
46
|
-
inspected = { ...legacy, encrypted: false }
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
if (!inspected.encrypted) {
|
|
50
|
-
await verifyStoragePayload(item.type, row, inspected.stream, signal)
|
|
51
|
-
signal.throwIfAborted()
|
|
52
|
-
const current = await openObjectVersion(raw, identity, source.version, signal)
|
|
53
|
-
if (!current) return 'Payload disappeared or changed version before encryption'
|
|
54
|
-
const encrypted = encryptStorageStream(current.stream, dataKey, identity, current.size)
|
|
55
|
-
let replaced: boolean
|
|
56
|
-
try { replaced = await raw.put(identity, encrypted, signal, source.version, current.size ?? undefined) }
|
|
57
|
-
finally { encrypted.destroy(); current.stream.destroy() }
|
|
58
|
-
if (!replaced) return await replacementFailure(raw, identity, source.version, signal)
|
|
59
|
-
// Do not mark the row encrypted until the stored bytes authenticate and
|
|
60
|
-
// match the original upload (including decompression for sourcemaps).
|
|
61
|
-
const replacement = await raw.open(identity, signal)
|
|
62
|
-
if (!replacement) {
|
|
63
|
-
if (!await db.getStorageRow(item.type, item.id)) return
|
|
64
|
-
throw new StoragePayloadError('Migrated payload unavailable')
|
|
65
|
-
}
|
|
66
|
-
const decoded = await decryptStorageStream(replacement.stream, dataKey, identity)
|
|
67
|
-
await verifyStoragePayload(item.type, row, decoded.stream, signal)
|
|
68
|
-
}
|
|
69
95
|
signal.throwIfAborted()
|
|
70
|
-
// Another process may have renamed ciphertext but stopped before syncing
|
|
71
|
-
// the directory. Persist the observed rename even on the resume path.
|
|
72
|
-
await raw.sync?.(identity, signal)
|
|
73
96
|
await db.markStorageEncrypted(item.type, item.id, row.dataKey!)
|
|
74
97
|
} finally { dataKey.fill(0) }
|
|
75
98
|
}
|
|
@@ -108,7 +131,7 @@ async function cleanupLegacy(raw: RawObjectStorage, db: StorageDb, state: Storag
|
|
|
108
131
|
const { owner, temporary } = target
|
|
109
132
|
if (!temporary && !owner.cache) {
|
|
110
133
|
const row = await db.getStorageRow(owner.type, owner.id)
|
|
111
|
-
if (row &&
|
|
134
|
+
if (row && storageRowPaths(owner.type, row).includes(object.key)) continue
|
|
112
135
|
}
|
|
113
136
|
const stored = await raw.head(object.key, signal)
|
|
114
137
|
if (!stored) continue
|
|
@@ -176,8 +199,17 @@ export async function migrateStorage(raw: RawObjectStorage, db: StorageDb, key:
|
|
|
176
199
|
for (const row of rows) {
|
|
177
200
|
if (Date.now() >= deadline || signal.aborted) break
|
|
178
201
|
console.info('managed-storage-migration-row-start:', JSON.stringify({ type: row.type, id: row.id }))
|
|
202
|
+
const startedAt = Date.now()
|
|
203
|
+
let phase: MigrationPhase | 'tokens' = 'tokens'
|
|
204
|
+
const logPhase = (aborted = false) => console.info('managed-storage-migration-row-phase:', JSON.stringify({
|
|
205
|
+
type: row.type, id: row.id, phase, elapsedMs: Date.now() - startedAt, ...(aborted ? { aborted: true } : {}),
|
|
206
|
+
}))
|
|
207
|
+
// Log directly on cancellation even if an underlying operation stalls
|
|
208
|
+
// while unwinding. These fixed phases contain no object/provider data.
|
|
209
|
+
const aborted = () => logPhase(true)
|
|
210
|
+
signal.addEventListener('abort', aborted, { once: true })
|
|
179
211
|
try {
|
|
180
|
-
const pending = await migrateRow(raw, db, key, row, signal)
|
|
212
|
+
const pending = await migrateRow(raw, db, key, row, signal, next => { phase = next; logPhase() })
|
|
181
213
|
if (pending) failures.push(rowFailure(row, pending))
|
|
182
214
|
}
|
|
183
215
|
catch (err) {
|
|
@@ -192,6 +224,7 @@ export async function migrateStorage(raw: RawObjectStorage, db: StorageDb, key:
|
|
|
192
224
|
errors.push(new Error(`Could not migrate ${row.type} ${row.id}`, { cause: err }))
|
|
193
225
|
} else failures.push(rowFailure(row, failure))
|
|
194
226
|
}
|
|
227
|
+
finally { signal.removeEventListener('abort', aborted) }
|
|
195
228
|
if (stopping?.aborted) break
|
|
196
229
|
await db.advanceStorageMigration(state.cursor, row.position)
|
|
197
230
|
processed++
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer'
|
|
2
2
|
import { createHash } from 'node:crypto'
|
|
3
|
-
import { Readable, Writable } from 'node:stream'
|
|
3
|
+
import { PassThrough, Readable, Writable } from 'node:stream'
|
|
4
4
|
import { pipeline } from 'node:stream/promises'
|
|
5
5
|
import { createBrotliDecompress } from 'node:zlib'
|
|
6
6
|
import { STORAGE_MAGIC } from '../server-common/storage-crypto.ts'
|
|
@@ -11,13 +11,14 @@ import type { StorageRow, StorageRowKind } from './storage-db.ts'
|
|
|
11
11
|
// not fix it. Keep this distinct from database, storage and cancellation errors.
|
|
12
12
|
export class StoragePayloadError extends Error {}
|
|
13
13
|
|
|
14
|
-
export function
|
|
15
|
-
return
|
|
14
|
+
export function storageRowPaths(type: StorageRowKind, row: Pick<StorageRow, 'id' | 'kind'>): string[] {
|
|
15
|
+
return type === 'report' ? [`reports/${row.id}.br`, `reports/${row.id}`]
|
|
16
|
+
: [`bundles/${row.id}${row.kind === 'sourcemap' ? '.map.br' : ''}`]
|
|
16
17
|
}
|
|
17
18
|
export function storageOwner(identity: string): { type: StorageRowKind; id: string; cache: boolean } | null {
|
|
18
19
|
const match = /^(reports|bundles)\/([^/]+)$/u.exec(identity)
|
|
19
20
|
if (match) {
|
|
20
|
-
const id = match[1] === 'bundles' ?
|
|
21
|
+
const id = match[2]!.replace(match[1] === 'bundles' ? /\.map\.br$/u : /\.br$/u, '')
|
|
21
22
|
if (isBlobId(id)) return { type: match[1] === 'reports' ? 'report' : 'bundle', id, cache: false }
|
|
22
23
|
}
|
|
23
24
|
const cache = /^cache\/(?:bundles|report-sources)\/([a-f\d-]{36})\//u.exec(identity)
|
|
@@ -50,23 +51,49 @@ export async function inspectStorageObject(stored: RawObject): Promise<RawObject
|
|
|
50
51
|
return { ...stored, stream, encrypted: length === prefix.length && prefix.equals(STORAGE_MAGIC) }
|
|
51
52
|
}
|
|
52
53
|
|
|
53
|
-
//
|
|
54
|
+
// Report hashes and sourcemap integrity describe the original upload, before Brotli storage.
|
|
54
55
|
// Hash incrementally, including decompression, so migration stays streaming.
|
|
55
|
-
export async function verifyStoragePayload(type: StorageRowKind, row: StorageRow, source: Readable, signal?: AbortSignal): Promise<void> {
|
|
56
|
+
export async function verifyStoragePayload(type: StorageRowKind, row: StorageRow, source: Readable, signal?: AbortSignal, identity?: string): Promise<void> {
|
|
56
57
|
const hash = createHash(type === 'report' ? 'sha256' : 'sha512')
|
|
57
58
|
const sink = new Writable({ write(chunk, _encoding, next) { hash.update(chunk); next() } })
|
|
58
59
|
const options = signal ? { signal } : {}
|
|
59
60
|
try {
|
|
60
|
-
if (type === 'bundle' && row.kind === 'sourcemap') {
|
|
61
|
+
if (type === 'bundle' && row.kind === 'sourcemap' || type === 'report' && identity?.endsWith('.br')) {
|
|
61
62
|
await pipeline(source, createBrotliDecompress(), sink, options)
|
|
62
63
|
} else await pipeline(source, sink, options)
|
|
63
64
|
} catch (err) {
|
|
64
65
|
const code = (err as NodeJS.ErrnoException).code ?? ''
|
|
65
66
|
if (['Z_DATA_ERROR', 'Z_BUF_ERROR'].includes(code) || code.startsWith('ERR__ERROR_FORMAT_')) {
|
|
66
|
-
throw new StoragePayloadError(
|
|
67
|
+
throw new StoragePayloadError(`Stored ${type === 'report' ? 'report' : 'sourcemap'} cannot be decompressed`, { cause: err })
|
|
67
68
|
}
|
|
68
69
|
throw err
|
|
69
70
|
}
|
|
70
71
|
const actual = type === 'report' ? hash.digest('base64url') : `sha512-${hash.digest('base64')}`
|
|
71
72
|
if (actual !== row.hash) throw new StoragePayloadError('Stored payload does not match its upload hash')
|
|
72
73
|
}
|
|
74
|
+
|
|
75
|
+
// Forward the stored representation with backpressure, but withhold EOF until
|
|
76
|
+
// its upload hash passes. Encryption cannot emit its final frame, and an atomic
|
|
77
|
+
// PUT cannot commit, until verification (including Brotli decompression) ends.
|
|
78
|
+
export function verifiedStoragePayload(type: StorageRowKind, row: StorageRow, source: Readable, signal?: AbortSignal, identity?: string): Readable {
|
|
79
|
+
const input = new PassThrough(), output = new PassThrough()
|
|
80
|
+
const verification = verifyStoragePayload(type, row, input, signal, identity)
|
|
81
|
+
// Propagate asynchronous decompression errors even if the source is stalled.
|
|
82
|
+
void verification.catch(err => { source.destroy(err); output.destroy(err) })
|
|
83
|
+
const forwarding = pipeline(source, async function* (chunks) {
|
|
84
|
+
try {
|
|
85
|
+
for await (const chunk of chunks) {
|
|
86
|
+
await new Promise<void>((resolve, reject) => { input.write(chunk, err => err ? reject(err) : resolve()) })
|
|
87
|
+
yield chunk
|
|
88
|
+
}
|
|
89
|
+
input.end()
|
|
90
|
+
await verification
|
|
91
|
+
} finally {
|
|
92
|
+
input.destroy()
|
|
93
|
+
await verification.catch(() => {})
|
|
94
|
+
}
|
|
95
|
+
}, output, signal ? { signal } : {})
|
|
96
|
+
// pipeline forwards errors to output and tears down input on early cancellation.
|
|
97
|
+
void forwarding.catch(() => {})
|
|
98
|
+
return output
|
|
99
|
+
}
|
|
@@ -5,6 +5,7 @@ import type { AvatarStore } from './avatar-store.ts'
|
|
|
5
5
|
import type { BlobStore } from './blob-store.ts'
|
|
6
6
|
import type { BundleCacheStorage } from './bundle-cache.ts'
|
|
7
7
|
import { createBundleStore } from './bundle-store.ts'
|
|
8
|
+
import { createReportStore } from './report-store.ts'
|
|
8
9
|
import { CacheMissError, type CacheStorage, validateCacheKey } from './cache-storage.ts'
|
|
9
10
|
import { type ObjectStorage, isBlobId } from './object-storage.ts'
|
|
10
11
|
|
|
@@ -71,7 +72,8 @@ export function createManagedStores(objects: ObjectStorage, avatarSidecar: boole
|
|
|
71
72
|
delete: value => bundles.delete(id(value)),
|
|
72
73
|
}
|
|
73
74
|
return {
|
|
74
|
-
reportStore: blobStore(objects, 'reports'),
|
|
75
|
+
reportStore: createReportStore(blobStore(objects, 'reports'), blobStore(objects, 'reports', '.br'),
|
|
76
|
+
(value, bytes) => objects.put(`reports/${id(value)}.br`, bytes, { replaceReport: true })),
|
|
75
77
|
bundleStore: createBundleStore(blobStore(objects, 'bundles'), blobStore(objects, 'bundles', '.map.br')),
|
|
76
78
|
uploadStore: blobStore(objects, 'uploads'),
|
|
77
79
|
avatarStore: avatarStore(objects, avatarSidecar),
|
|
@@ -149,7 +149,7 @@ export function workspaceShareMethods(db: ManagedSql): WorkspaceShareStore {
|
|
|
149
149
|
repositories: await q.repositories.all(team.id) as WorkspaceShareSnapshot['repositories'],
|
|
150
150
|
reports: rows.map(row => ({ id: row.id, filename: row.filename, byteSize: row.byteSize, sha256: row.sha256,
|
|
151
151
|
repo: { github: row.github, directory: row.directory }, permissions })),
|
|
152
|
-
team: { ...team,
|
|
152
|
+
team: { ...team, permissions,
|
|
153
153
|
reports: rows.map(row => ({ id: row.id, slug: row.slug, filename: row.filename, analyzer: row.analyzer, repoFullName: row.github, repoDirectory: row.directory,
|
|
154
154
|
cacheKey: JSON.stringify([row.sha256, row.github, row.directory, row.filename, permissions]) })),
|
|
155
155
|
bundles: bundleRows.map(row => ({ id: row.id, slug: row.slug, integrity: row.integrity,
|