@preventive/triage 1.0.0-alpha.25 → 1.0.0-alpha.27

package/server-managed/storage-stores.ts

@@ -0,0 +1,81 @@
+// Logical stores share one encryption boundary. Compression, identifiers and
+// public response bytes stay the same with either physical backend.
+import { Buffer } from 'node:buffer'
+import type { AvatarStore } from './avatar-store.ts'
+import type { BlobStore } from './blob-store.ts'
+import type { BundleCacheStorage } from './bundle-cache.ts'
+import { createBundleStore } from './bundle-store.ts'
+import { type CacheStorage, validateCacheKey } from './cache-storage.ts'
+import { type ObjectStorage, isBlobId } from './object-storage.ts'
+
+function id(value: string): string {
+  if (!isBlobId(value)) throw new Error('Invalid managed blob id')
+  return value
+}
+
+function blobStore(objects: ObjectStorage, name: string, suffix = ''): BlobStore {
+  const path = (value: string) => `${name}/${id(value)}${suffix}`
+  return {
+    put: (value, bytes) => objects.put(path(value), bytes),
+    get: value => objects.get(path(value)),
+    open: value => objects.open(path(value)),
+    delete: value => objects.delete(path(value)),
+  }
+}
+
+function cacheStore(objects: ObjectStorage, name: string): CacheStorage {
+  const path = (key: string) => `cache/${name}/${validateCacheKey(key)}`
+  return {
+    exists: key => objects.exists(path(key)),
+    put: async (key, bytes) => { await objects.put(path(key), bytes) },
+    async open(key) {
+      const value = await objects.open(path(key))
+      if (!value) throw new Error('Managed cache unavailable')
+      return value
+    },
+    delete: prefix => objects.deletePrefix(`${path(prefix)}/`),
+  }
+}
+
+function avatarStore(objects: ObjectStorage, sidecar: boolean): AvatarStore {
+  return {
+    async put(value, contentType, bytes) {
+      const path = `avatars/${id(value)}`, type = contentType.split(';', 1)[0]!.trim()
+      if (!/^image\/[a-z0-9.+-]+$/iu.test(type)) throw new Error('Invalid avatar type')
+      if (sidecar) {
+        await objects.put(path, bytes)
+        await objects.put(`${path}.type`, Buffer.from(type))
+      } else await objects.put(path, Buffer.concat([Buffer.from(`${type}\n`), bytes]))
+    },
+    async get(value) {
+      const path = `avatars/${id(value)}`
+      const bytes = await objects.get(path)
+      if (!bytes) return null
+      if (sidecar) {
+        const type = await objects.get(`${path}.type`)
+        return type ? { bytes, contentType: type.toString().trim() || 'application/octet-stream' } : null
+      }
+      const split = bytes.indexOf(10)
+      if (split < 0 || split > 100) throw new Error('Invalid cached avatar')
+      return { contentType: bytes.subarray(0, split).toString(), bytes: bytes.subarray(split + 1) }
+    },
+  }
+}
+
+export function createManagedStores(objects: ObjectStorage, avatarSidecar: boolean) {
+  const bundles = cacheStore(objects, 'bundles')
+  const cacheStorage: BundleCacheStorage = {
+    exists: (value, file) => bundles.exists(`${id(value)}/${file}`),
+    put: (value, file, bytes) => bundles.put(`${id(value)}/${file}`, bytes),
+    open: (value, file) => bundles.open(`${id(value)}/${file}`),
+    delete: value => bundles.delete(id(value)),
+  }
+  return {
+    reportStore: blobStore(objects, 'reports'),
+    bundleStore: createBundleStore(blobStore(objects, 'bundles'), blobStore(objects, 'bundles', '.map.br')),
+    uploadStore: blobStore(objects, 'uploads'),
+    avatarStore: avatarStore(objects, avatarSidecar),
+    cacheStorage,
+    reportSourcesStorage: cacheStore(objects, 'report-sources'),
+  }
+}

package/server-managed/storage.ts

@@ -1,34 +1,56 @@
 // Backend selection is shared by standalone, combined and function entrypoints.
-import { dirname, join } from 'node:path'
-import { createDiskAvatarStore } from './avatar-store.ts'
-import { createDiskBlobStore } from './blob-store.ts'
-import { createDiskBundleStore } from './bundle-store.ts'
-import { createBundleCache, createDiskBundleCache } from './bundle-cache.ts'
-import { createDiskCacheStorage } from './cache-storage.ts'
+import { dirname } from 'node:path'
+import { parseStorageKey } from '../server-common/storage-crypto.ts'
+import { createBundleCache } from './bundle-cache.ts'
 import { createReportSourcesCache } from './report-sources.ts'
-import { openManagedVercelStorage } from './blob-vercel.ts'
 import { openNeonManagedDb } from './db-neon.ts'
 import type { ManagedConfig } from './config.ts'
+import { createEncryptedObjectStorage } from './storage-encryption.ts'
+import { createManagedStores } from './storage-stores.ts'
+import { createDiskObjectStorage } from './object-storage-disk.ts'
+import { openVercelObjectStorage } from './object-storage-vercel.ts'
+import { migrateStorage, reapStorageUploads } from './storage-maintenance.ts'
+
+// Database-only callers (notably status) must not activate byte encryption.
+export async function openManagedStorageDb(config: ManagedConfig) {
+  const options = { triageHistoryLimit: config.triageHistoryLimit, storageEncryptionKey: parseStorageKey(config.storageEncryptionKey) }
+  return config.neonUrl ? openNeonManagedDb(config.neonUrl, options)
+    : (await import('./db.ts')).openSqliteManagedDb(config.dbPath, options)
+}
 
 export async function openManagedStorage(config: ManagedConfig) {
-  const options = { triageHistoryLimit: config.triageHistoryLimit }
-  if (config.neonUrl) {
-    if (!config.blobToken) throw new Error('Managed Neon mode requires BLOB_READ_WRITE_TOKEN')
-    const storage = await openManagedVercelStorage(config.blobToken)
-    const db = await openNeonManagedDb(config.neonUrl, options)
-    return { ...storage, db, bundleCache: createBundleCache(storage.cacheStorage, db, storage.bundleStore),
+  const key = parseStorageKey(config.storageEncryptionKey)
+  if (config.storageEncryptionMigrate && !key) throw new Error('MANAGED_STORAGE_ENCRYPTION_MIGRATE requires MANAGED_STORAGE_ENCRYPTION_KEY')
+  if (config.neonUrl && !config.blobToken) throw new Error('Managed Neon mode requires BLOB_READ_WRITE_TOKEN')
+  if (config.serverless && !config.neonUrl) throw new Error('Serverless managed storage requires Neon')
+  const db = await openManagedStorageDb(config)
+  try {
+    const raw = config.neonUrl ? await openVercelObjectStorage(config.blobToken!) : createDiskObjectStorage(dirname(config.dbPath))
+    // Enable before exposing stores or token writers. Concurrent first starts
+    // serialize activation in the database and validate the winning key.
+    if (key && !await db.getStorageEncryption()) {
+      if (config.vercelPreview) throw new Error('Storage encryption must be enabled outside a Vercel preview deployment')
+      await db.enableStorageEncryption()
+    }
+    const objects = await createEncryptedObjectStorage(raw, db, key)
+    const storage = createManagedStores(objects, !config.neonUrl)
+    return { ...storage, db, uploadStore: config.neonUrl ? storage.uploadStore : undefined,
+      bundleCache: createBundleCache(storage.cacheStorage, db, storage.bundleStore),
       reportSourcesCache: createReportSourcesCache(storage.reportSourcesStorage, db, storage.reportStore, storage.bundleStore),
+      async reapStorage() {
+        if (!config.storageEncryptionMigrate) return
+        const state = await db.getStorageEncryption()
+        if (state?.complete && state.cleanupComplete) return
+        const result = await migrateStorage(raw, db, key!, { maxMs: config.storageEncryptionMigrateMaxMs })
+        console.info('managed-storage-migration:', JSON.stringify({
+          complete: result.complete === 1, cleanupComplete: result.cleanupComplete === 1,
+          migrated: result.migrated, cursor: result.cursor, retryAt: result.retryAt,
+        }))
+      },
+      ...(config.neonUrl ? { reapUploads: () => reapStorageUploads(raw, db) } : {}),
     }
-  }
-  if (config.serverless) throw new Error('Serverless managed storage requires Neon')
-  const { openSqliteManagedDb } = await import('./db.ts')
-  const db = openSqliteManagedDb(config.dbPath, options)
-  const dir = dirname(config.dbPath)
-  const reportStore = createDiskBlobStore(join(dir, 'reports'))
-  const bundleStore = createDiskBundleStore(join(dir, 'bundles'))
-  return { db, reportStore, bundleStore,
-    reportSourcesCache: createReportSourcesCache(createDiskCacheStorage(join(dir, 'cache', 'report-sources')), db, reportStore, bundleStore),
-    avatarStore: createDiskAvatarStore(join(dir, 'avatars')),
-    bundleCache: createDiskBundleCache(join(dir, 'cache', 'bundles'), db, bundleStore),
+  } catch (err) {
+    await db.close()
+    throw err
   }
 }

package/server-managed/uploads.ts

@@ -7,6 +7,8 @@ import type { IncomingMessage } from 'node:http'
 import type { BlobStore } from './blob-store.ts'
 
 export const UPLOAD_CHUNK_BYTES = 3 * 1024 * 1024
+export const MAX_UPLOAD_BYTES = 1024 * 1024 * 1024
+const MAX_UPLOAD_PARTS = Math.ceil(MAX_UPLOAD_BYTES / UPLOAD_CHUNK_BYTES)
 const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u
 export type UploadKind = 'reports' | 'bundles'
 
@@ -16,24 +18,40 @@ function partId(session: string, kind: UploadKind, id: string, index: number): s
 }
 
 export function validUploadPart(id: string, index: number, maxBytes: number): boolean {
-  return UUID.test(id) && Number.isSafeInteger(index) && index >= 0 && index < Math.ceil(maxBytes / UPLOAD_CHUNK_BYTES)
+  return UUID.test(id) && Number.isSafeInteger(index) && index >= 0 && index < Math.ceil(Math.min(maxBytes, MAX_UPLOAD_BYTES) / UPLOAD_CHUNK_BYTES)
 }
 
-export function putUploadPart(store: BlobStore, session: string, kind: UploadKind, id: string, index: number, bytes: Buffer): Promise<void> {
-  return store.put(partId(session, kind, id, index), bytes)
+export function validUpload(id: string, count: number): boolean {
+  return UUID.test(id) && Number.isSafeInteger(count) && count > 0
+}
+
+// Limit cleanup to indices that any configured server could have accepted,
+// even if the request advertises an enormous count or the limit was lowered.
+export async function deleteUpload(store: BlobStore, session: string, kind: UploadKind, id: string, count: number): Promise<void> {
+  if (!validUpload(id, count)) throw new Error('bad-upload')
+  const end = Math.min(count, MAX_UPLOAD_PARTS), errors = []
+  for (let start = 0; start < end; start += 8) {
+    const results = await Promise.allSettled(Array.from({ length: Math.min(8, end - start) },
+      (_, index) => store.delete(partId(session, kind, id, start + index))))
+    for (const result of results) if (result.status === 'rejected') errors.push(result.reason)
+  }
+  if (errors.length > 0) throw new AggregateError(errors, 'Upload cleanup failed')
+}
+
+export async function putUploadPart(store: BlobStore, session: string, kind: UploadKind, id: string, index: number, bytes: Buffer): Promise<void> {
+  await store.put(partId(session, kind, id, index), bytes)
 }
 
 export async function readUpload(store: BlobStore, req: IncomingMessage, session: string, kind: UploadKind, maxBytes: number): Promise<Buffer> {
   const id = req.headers['x-upload-id']
   const count = Number(req.headers['x-upload-parts']), size = Number(req.headers['x-upload-size'])
-  if (typeof id !== 'string' || !UUID.test(id) || !Number.isSafeInteger(size) || size <= 0
-    || !Number.isSafeInteger(count) || count !== Math.ceil(size / UPLOAD_CHUNK_BYTES)) throw new Error('bad-upload')
-  if (size > maxBytes) throw new Error('too-large')
-  const ids = Array.from({ length: count }, (_, index) => partId(session, kind, id, index))
-  const parts: Buffer[] = []
+  if (typeof id !== 'string' || !validUpload(id, count)) throw new Error('bad-upload')
   try {
-    for (const [index, key] of ids.entries()) {
-      const bytes = await store.get(key)
+    if (!Number.isSafeInteger(size) || size <= 0 || count !== Math.ceil(size / UPLOAD_CHUNK_BYTES)) throw new Error('bad-upload')
+    if (size > maxBytes || size > MAX_UPLOAD_BYTES) throw new Error('too-large')
+    const parts: Buffer[] = []
+    for (let index = 0; index < count; index++) {
+      const bytes = await store.get(partId(session, kind, id, index))
       const expected = Math.min(UPLOAD_CHUNK_BYTES, size - index * UPLOAD_CHUNK_BYTES)
       if (!bytes || bytes.length !== expected) throw new Error('bad-upload')
       parts.push(bytes)
@@ -41,7 +59,7 @@ export async function readUpload(store: BlobStore, req: IncomingMessage, session
     return Buffer.concat(parts, size)
   } finally {
     // These bytes can no longer be finalized. Failed/abandoned chunk uploads
-    // are also covered by the daily staging sweep.
-    for (const key of ids) await store.delete(key).catch(err => console.warn('managed: upload cleanup failed:', err))
+    // are also covered by automatic staging maintenance.
+    await deleteUpload(store, session, kind, id, count).catch(err => console.warn('managed: upload cleanup failed:', err))
   }
 }

package/server-managed/blob-vercel.ts

@@ -1,125 +0,0 @@
-import { Buffer } from 'node:buffer'
-import { Readable } from 'node:stream'
-import { type VercelBlobSdk, isNotFound, loadVercelBlobSdk } from '../server-common/vercel-blob.ts'
-import type { AvatarStore } from './avatar-store.ts'
-import type { BlobStore } from './blob-store.ts'
-import { type CacheStorage, validateCacheKey } from './cache-storage.ts'
-import type { BundleCacheStorage } from './bundle-cache.ts'
-import { createBundleStore } from './bundle-store.ts'
-
-const ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u
-function validate(id: string): string {
-  if (!ID.test(id)) throw new Error('Invalid managed blob id')
-  return id
-}
-
-type ListedBlob = Awaited<ReturnType<VercelBlobSdk['list']>>['blobs'][number]
-
-async function deletePrefix(blobs: VercelBlobSdk, token: string, prefix: string, remove: (path: string) => Promise<void>, matches = (_blob: ListedBlob) => true): Promise<void> {
-  const cursors = new Set<string>(), paths = new Set<string>()
-  let cursor: string | undefined
-  // Finish listing before deleting so pagination cannot skip objects as the
-  // listed set shrinks. Callers supply a trailing slash to isolate a namespace.
-  do {
-    const page = await blobs.list({ token, prefix, ...(cursor ? { cursor } : {}) })
-    for (const blob of page.blobs) if (blob.pathname.startsWith(prefix) && matches(blob)) paths.add(blob.pathname)
-    if (!page.hasMore) break
-    if (!page.cursor || cursors.has(page.cursor)) throw new Error('Invalid blob pagination')
-    cursor = page.cursor
-    cursors.add(cursor)
-  } while (cursor)
-  for (const path of paths) await remove(path)
-}
-
-function createBlobAvatarStore(avatarBlobs: BlobStore): AvatarStore {
-  return {
-    put(id, contentType, bytes) {
-      const type = contentType.split(';', 1)[0]!.trim()
-      if (!/^image\/[a-z0-9.+-]+$/iu.test(type)) throw new Error('Invalid avatar type')
-      return avatarBlobs.put(id, Buffer.concat([Buffer.from(`${type}\n`), bytes]))
-    },
-    async get(id) {
-      const bytes = await avatarBlobs.get(id)
-      if (!bytes) return null
-      const split = bytes.indexOf(10)
-      if (split < 0 || split > 100) throw new Error('Invalid cached avatar')
-      return { contentType: bytes.subarray(0, split).toString(), bytes: bytes.subarray(split + 1) }
-    },
-  }
-}
-
-export async function openManagedVercelStorage(token: string, sdk?: VercelBlobSdk) {
-  const blobs = sdk ?? await loadVercelBlobSdk()
-  const options = { token, access: 'private' as const, useCache: false }
-  async function put(path: string, bytes: Buffer) {
-    await blobs.put(path, bytes, { token, access: 'private', addRandomSuffix: false,
-      allowOverwrite: true, contentType: 'application/octet-stream', cacheControlMaxAge: 60 })
-  }
-  async function open(path: string) {
-    try {
-      const result = await blobs.get(path, options)
-      if (!result) return null
-      if (result.statusCode !== 200 || !result.stream) throw new Error('Unexpected blob response')
-      // Private SDK GETs may report zero for a nonempty body. Stream without
-      // Content-Length in that case so HTTP clients do not truncate the body.
-      const size = result.blob.size === 0 ? null : result.blob.size
-      return { size, stream: Readable.fromWeb(result.stream as Parameters<typeof Readable.fromWeb>[0]) }
-    } catch (err) { if (isNotFound(err, blobs)) return null; throw err }
-  }
-  async function get(path: string) {
-    const result = await open(path)
-    if (!result) return null
-    const parts: Buffer[] = []
-    for await (const chunk of result.stream) parts.push(Buffer.from(chunk))
-    return Buffer.concat(parts)
-  }
-  async function remove(path: string) {
-    try { await blobs.del(path, { token }) } catch (err) { if (!isNotFound(err, blobs)) throw err }
-  }
-  function store(name: string, suffix = ''): BlobStore {
-    const path = (id: string) => `.managed/${name}/${validate(id)}${suffix}`
-    return {
-      async put(id, bytes) { await put(path(id), bytes) },
-      async get(id) { return await get(path(id)) },
-      open: id => open(path(id)),
-      async delete(id) { await remove(path(id)) },
-    }
-  }
-  const avatarStore = createBlobAvatarStore(store('avatars'))
-  const cachePath = (id: string, file: string) => `.managed/cache/bundles/${validate(id)}/${file}`
-  const cacheStorage: BundleCacheStorage = {
-    async exists(id, file) {
-      try { await blobs.head(cachePath(id, file), { token }); return true }
-      catch (err) { if (isNotFound(err, blobs)) return false; throw err }
-    },
-    put: (id, file, bytes) => put(cachePath(id, file), bytes),
-    async open(id, file) {
-      const result = await open(cachePath(id, file))
-      if (!result) throw new Error('Bundle cache unavailable')
-      return result
-    },
-    delete: id => deletePrefix(blobs, token, cachePath(id, ''), remove),
-  }
-  // Staging uploads are never published. An interrupted browser leaves only
-  // parts here; the authenticated cron removes them after a full day.
-  function reapUploads(now = Date.now()) {
-    return deletePrefix(blobs, token, '.managed/uploads/', remove,
-      blob => new Date(blob.uploadedAt ?? now).getTime() < now - 86_400_000)
-  }
-  const sourcesPath = (key: string) => `.managed/cache/report-sources/${validateCacheKey(key)}`
-  const reportSourcesStorage: CacheStorage = {
-    async exists(key) {
-      try { await blobs.head(sourcesPath(key), { token }); return true }
-      catch (err) { if (isNotFound(err, blobs)) return false; throw err }
-    },
-    put: (key, bytes) => put(sourcesPath(key), bytes),
-    async open(key) {
-      const result = await open(sourcesPath(key))
-      if (!result) throw new Error('Report sources cache unavailable')
-      return result
-    },
-    delete: prefix => deletePrefix(blobs, token, `${sourcesPath(prefix)}/`, remove),
-  }
-  const bundleStore = createBundleStore(store('bundles'), store('bundles', '.map.br'))
-  return { reportStore: store('reports'), bundleStore, uploadStore: store('uploads'), avatarStore, cacheStorage, reportSourcesStorage, reapUploads }
-}