@preventive/triage 1.0.0-alpha.25 → 1.0.0-alpha.27

package/server-managed/storage-db.ts

@@ -0,0 +1,168 @@
+// Keys belong to the existing upload rows. Only installation state and bounded
+// migration progress need a separate table; payload paths never move.
+import { Buffer } from 'node:buffer'
+import { randomBytes } from 'node:crypto'
+import { type StorageKey, unwrapStorageValue, wrapStorageValue } from '../server-common/storage-crypto.ts'
+import type { ManagedSql } from './sql.ts'
+
+export const STORAGE_SCHEMA = `
+CREATE TABLE IF NOT EXISTS managed_storage_encryption (
+  id INTEGER PRIMARY KEY CHECK (id = 1), test_value TEXT NOT NULL,
+  enabled_at INTEGER NOT NULL, cursor TEXT, complete INTEGER NOT NULL DEFAULT 0,
+  migrated INTEGER NOT NULL DEFAULT 0, cleanup_cursor TEXT,
+  cleanup_found INTEGER NOT NULL DEFAULT 0, cleanup_complete INTEGER NOT NULL DEFAULT 0
+) STRICT;
+`
+const SENTINEL = Buffer.from('deepview.storage.required.v1')
+export type StorageRowKind = 'report' | 'bundle'
+export interface StorageEncryptionState {
+  enabledAt: number; cursor: string | null; complete: number; migrated: number
+  cleanupCursor: string | null; cleanupFound: number; cleanupComplete: number
+}
+export interface StorageRow {
+  id: string; dataKey: string | null; encrypted: number; hash: string; kind: string | null
+}
+export interface StorageMigrationRow { position: string; id: string; type: StorageRowKind | 'user' }
+export interface StorageDb {
+  enableStorageEncryption(): Promise<StorageEncryptionState>
+  getStorageEncryption(): Promise<StorageEncryptionState | null>
+  getStorageRow(type: StorageRowKind, id: string): Promise<StorageRow | null>
+  ensureStorageDataKey(type: StorageRowKind, id: string): Promise<StorageRow | null>
+  markStorageEncrypted(type: StorageRowKind, id: string, dataKey: string): Promise<void>
+  listStorageMigrationRows(after: string | null, limit: number): Promise<StorageMigrationRow[]>
+  advanceStorageMigration(expected: string | null, next: string | null): Promise<void>
+  advanceStorageCleanup(expected: string | null, next: string | null, removed: number): Promise<void>
+  migrateStorageUserTokens(id: string): Promise<void>
+}
+
+function table(type: StorageRowKind): string {
+  if (type !== 'report' && type !== 'bundle') throw new Error('Invalid storage row type')
+  return `managed_${type}`
+}
+export const dataKeyIdentity = (type: StorageRowKind, id: string): string => `${table(type)}:${id}`
+export function unwrapDataKey(key: StorageKey, type: StorageRowKind, row: Pick<StorageRow, 'id' | 'dataKey'>): Buffer {
+  if (!row.dataKey) throw new Error('Missing storage data key')
+  const value = unwrapStorageValue(key, dataKeyIdentity(type, row.id), row.dataKey)
+  if (value.length !== 32) { value.fill(0); throw new Error('Invalid storage data key') }
+  return value
+}
+
+export async function storageState(db: ManagedSql, key: StorageKey | null): Promise<StorageEncryptionState | null> {
+  const row = await db.prepare(`SELECT test_value AS testValue, enabled_at AS enabledAt, cursor, complete, migrated,
+    cleanup_cursor AS cleanupCursor, cleanup_found AS cleanupFound, cleanup_complete AS cleanupComplete
+    FROM managed_storage_encryption WHERE id = 1`).get() as (StorageEncryptionState & { testValue: string }) | undefined
+  if (!row) return null
+  if (!key) throw new Error('Managed storage requires its configured encryption key')
+  const plain = unwrapStorageValue(key, 'managed_storage_encryption:1', row.testValue)
+  try { if (!plain.equals(SENTINEL)) throw new Error('Invalid storage encryption test value') }
+  finally { plain.fill(0) }
+  const { testValue: _testValue, ...state } = row
+  return state
+}
+
+// Called inside the insert transaction, fencing an old/plain upload that raced
+// activation. The wrapped key never enters public report/bundle DTOs.
+export async function validateStorageUpload(db: ManagedSql, key: StorageKey | null, type: StorageRowKind,
+  id: string, dataKey: string | null | undefined): Promise<void> {
+  const state = await storageState(db, key)
+  if (state) unwrapDataKey(key!, type, { id, dataKey: dataKey ?? null }).fill(0)
+  else if (dataKey) throw new Error('Storage encryption is not enabled')
+}
+
+export interface StoredTokens { access: string | null; refresh: string | null; exp: number | null; encrypted: number }
+const tokenIdentity = (id: string, field: string) => `managed_user.gh_${field}_token:${id}`
+export function encodeStorageTokens(key: StorageKey, id: string, access: string | null, refresh: string | null) {
+  const encode = (field: string, value: string | null) => value === null ? null : wrapStorageValue(key, tokenIdentity(id, field), Buffer.from(value))
+  return { access: encode('access', access), refresh: encode('refresh', refresh) }
+}
+export function decodeStorageTokens(key: StorageKey, id: string, row: StoredTokens) {
+  const decode = (field: string, value: string | null) => {
+    if (value === null) return null
+    const plain = unwrapStorageValue(key, tokenIdentity(id, field), value)
+    try { return plain.toString('utf8') } finally { plain.fill(0) }
+  }
+  return { access: decode('access', row.access), refresh: decode('refresh', row.refresh) }
+}
+
+const PENDING = `SELECT 'bundle:' || id AS position, id, 'bundle' AS type FROM managed_bundle WHERE storage_encrypted = 0
+  UNION ALL SELECT 'report:' || id AS position, id, 'report' AS type FROM managed_report WHERE storage_encrypted = 0
+  UNION ALL SELECT 'user:' || id AS position, id, 'user' AS type FROM managed_user
+    WHERE gh_tokens_encrypted = 0 AND (gh_access_token IS NOT NULL OR gh_refresh_token IS NOT NULL)`
+
+export function storageMethods(db: ManagedSql, key: StorageKey | null): StorageDb {
+  async function requireEnabled() {
+    const state = await storageState(db, key)
+    if (!state) throw new Error('Storage encryption is not enabled; configure MANAGED_STORAGE_ENCRYPTION_KEY')
+    return state
+  }
+  async function getRow(type: StorageRowKind, id: string): Promise<StorageRow | null> {
+    const row = await db.prepare(`SELECT id, data_key AS dataKey, storage_encrypted AS encrypted,
+      ${type === 'report' ? 'sha256 AS hash, NULL AS kind' : 'integrity AS hash, kind'} FROM ${table(type)} WHERE id = ?`).get(id) as StorageRow | undefined ?? null
+    if (row && row.encrypted !== 0 && row.encrypted !== 1) throw new Error('Invalid storage encryption state')
+    if (row?.encrypted && !row.dataKey) throw new Error('Encrypted storage row is missing its data key')
+    return row
+  }
+  return {
+    async enableStorageEncryption() {
+      if (!key) throw new Error('Set MANAGED_STORAGE_ENCRYPTION_KEY before enabling storage encryption')
+      // Called during startup in a writer transaction. Concurrent instances
+      // must reuse the marker and validate its key before serving requests.
+      const existing = await storageState(db, key)
+      if (existing) return existing
+      await db.prepare('INSERT INTO managed_storage_encryption (id, test_value, enabled_at) VALUES (1, ?, ?)')
+        .run(wrapStorageValue(key, 'managed_storage_encryption:1', SENTINEL), Date.now())
+      return (await storageState(db, key))!
+    },
+    getStorageEncryption: () => storageState(db, key),
+    getStorageRow: getRow,
+    async ensureStorageDataKey(type, id) {
+      await requireEnabled()
+      const current = await getRow(type, id)
+      if (!current || current.dataKey) return current
+      const bytes = randomBytes(32)
+      let wrapped: string
+      try { wrapped = wrapStorageValue(key!, dataKeyIdentity(type, id), bytes) } finally { bytes.fill(0) }
+      await db.prepare(`UPDATE ${table(type)} SET data_key = ? WHERE id = ? AND data_key IS NULL`).run(wrapped, id)
+      // Return the persisted winner, never a worker's uncommitted candidate.
+      return getRow(type, id)
+    },
+    async markStorageEncrypted(type, id, wrapped) {
+      await requireEnabled()
+      const result = await db.prepare(`UPDATE ${table(type)} SET storage_encrypted = 1
+        WHERE id = ? AND data_key = ? AND storage_encrypted = 0`).run(id, wrapped)
+      if (Number(result.changes)) await db.prepare('UPDATE managed_storage_encryption SET migrated = migrated + 1 WHERE id = 1').run()
+    },
+    async listStorageMigrationRows(after, limit) {
+      await requireEnabled()
+      return await db.prepare(`SELECT position, id, type FROM (${PENDING}) AS pending
+        WHERE position > ? ORDER BY position LIMIT ?`).all(after ?? '', limit) as StorageMigrationRow[]
+    },
+    async advanceStorageMigration(expected, next) {
+      await requireEnabled()
+      if (expected !== null && next !== null && next <= expected) return
+      // End-of-pass completion checks every pending row, including failed rows
+      // before the cursor. A failure cannot be mistaken for a finished pass.
+      await db.prepare(`UPDATE managed_storage_encryption SET cursor = ?, complete =
+        CASE WHEN ? IS NULL AND NOT EXISTS (SELECT 1 FROM (${PENDING}) AS pending) THEN 1 ELSE 0 END
+        WHERE id = 1 AND (cursor = ? OR (cursor IS NULL AND ? IS NULL))`).run(next, next, expected, expected)
+    },
+    async advanceStorageCleanup(expected, next, removed) {
+      const state = await requireEnabled()
+      const found = state.cleanupFound + removed
+      await db.prepare(`UPDATE managed_storage_encryption SET cleanup_cursor = ?, cleanup_found = ?, cleanup_complete = ?
+        WHERE id = 1 AND (cleanup_cursor = ? OR (cleanup_cursor IS NULL AND ? IS NULL))`).run(next, next === null ? 0 : found, next === null && found === 0 ? 1 : 0, expected, expected)
+    },
+    async migrateStorageUserTokens(id) {
+      await requireEnabled()
+      const row = await db.prepare(`SELECT gh_access_token AS access, gh_refresh_token AS refresh, gh_token_expires_at AS exp,
+        gh_tokens_encrypted AS encrypted FROM managed_user WHERE id = ?`).get(id) as StoredTokens | undefined
+      if (!row || row.encrypted || (row.access === null && row.refresh === null)) return
+      const value = encodeStorageTokens(key!, id, row.access, row.refresh)
+      // This operation holds the writer transaction; token refresh cannot be
+      // overwritten by a migration that read an earlier token value.
+      await db.prepare('UPDATE managed_user SET gh_access_token = ?, gh_refresh_token = ?, gh_tokens_encrypted = 1 WHERE id = ?')
+        .run(value.access, value.refresh, id)
+      await db.prepare('UPDATE managed_storage_encryption SET migrated = migrated + 1 WHERE id = 1').run()
+    },
+  }
+}

package/server-managed/storage-encryption.ts

@@ -0,0 +1,211 @@
+import { Buffer } from 'node:buffer'
+import { randomBytes } from 'node:crypto'
+import { Readable } from 'node:stream'
+import { type StorageKey, decryptStorageStream, encryptStorageStream, wrapStorageValue } from '../server-common/storage-crypto.ts'
+import { ENCRYPTED_CACHE_PREFIX, type ObjectStorage, type RawObject, type RawObjectStorage, deleteObjects, objectPath, openObjectVersion } from './object-storage.ts'
+import { type StorageDb, type StorageEncryptionState, type StorageRow, type StorageRowKind, dataKeyIdentity, unwrapDataKey } from './storage-db.ts'
+import { inspectStorageObject, storageOwner, storageRowPath, verifyStoragePayload } from './storage-payload.ts'
+
+const STORAGE_WRITE_MS = 180_000
+export const STORAGE_UPLOAD_TTL_MS = 86_400_000
+export const STORAGE_DISABLED_TTL_MS = 5_000
+
+async function readBytes(stored: Awaited<ReturnType<ObjectStorage['open']>>): Promise<Buffer | null> {
+  if (!stored) return null
+  const parts: Buffer[] = []
+  try { for await (const part of stored.stream) parts.push(Buffer.from(part)); return Buffer.concat(parts) }
+  finally { stored.stream.destroy() }
+}
+function logicalKey(identity: string) {
+  objectPath(identity)
+  if (identity.startsWith(ENCRYPTED_CACHE_PREFIX)) throw new Error('Encrypted cache paths are internal')
+}
+const encryptedCachePath = (identity: string) => `${ENCRYPTED_CACHE_PREFIX}${identity.slice('cache/'.length)}`
+const deletionPaths = (identity: string) => identity.startsWith('cache/') ? [identity, encryptedCachePath(identity)] : [identity]
+
+// A PUT can commit and still throw. Remove only this attempted plaintext,
+// comparing bytes rather than magic: arbitrary uploads can share the header.
+// A concurrent migration's ciphertext must survive this reconciliation.
+async function discardPlaintextWrite(raw: RawObjectStorage, identity: string, bytes: Buffer): Promise<void> {
+  // Leave time to report the original failure inside a serverless invocation.
+  const signal = AbortSignal.timeout(30_000)
+  const current = await raw.open(identity, signal)
+  if (!current) return
+  try {
+    let offset = 0
+    for await (const part of current.stream) {
+      const chunk = Buffer.from(part)
+      if (!chunk.equals(bytes.subarray(offset, offset + chunk.length))) return
+      offset += chunk.length
+    }
+    if (offset === bytes.length) await raw.delete(identity, current.version, signal)
+  } finally { current.stream.destroy() }
+}
+
+async function putPlaintext(raw: RawObjectStorage, identity: string, bytes: Buffer, mode: () => Promise<StorageEncryptionState | null>): Promise<null> {
+  try {
+    await raw.put(identity, bytes, AbortSignal.timeout(STORAGE_WRITE_MS))
+    if (!await mode()) return null
+    throw new Error('Storage encryption was enabled during upload; retry the upload')
+  } catch (err) {
+    try { await discardPlaintextWrite(raw, identity, bytes) }
+    catch (cleanup) { throw new AggregateError([err, cleanup], 'Storage write and plaintext cleanup failed', { cause: cleanup }) }
+    throw err
+  }
+}
+
+async function decryptOwned(stored: RawObject, key: StorageKey, type: StorageRowKind, row: StorageRow, identity: string) {
+  let dataKey: Buffer | undefined
+  try {
+    dataKey = unwrapDataKey(key, type, row)
+    return await decryptStorageStream(stored.stream, dataKey, identity)
+  } catch (err) { stored.stream.destroy(); throw err }
+  finally { dataKey?.fill(0) }
+}
+
+async function openEncrypted(raw: RawObjectStorage, db: StorageDb, key: StorageKey, identity: string, state: StorageEncryptionState): ReturnType<ObjectStorage['open']> {
+  const owner = storageOwner(identity)
+  if (!owner) {
+    if (!identity.startsWith('uploads/')) throw new Error('Unknown managed storage owner')
+    const stored = await raw.open(identity)
+    if (!stored) return null
+    const inspected = await inspectStorageObject(stored)
+    if (inspected.encrypted) return decryptStorageStream(inspected.stream, key.bytes, identity)
+    if (stored.modifiedAt <= state.enabledAt && Date.now() < state.enabledAt + STORAGE_UPLOAD_TTL_MS) return inspected
+    inspected.stream.destroy()
+    throw new Error('Unexpected plaintext upload part')
+  }
+  for (let attempt = 0; attempt < 3; attempt++) {
+    const row = await db.getStorageRow(owner.type, owner.id)
+    if (!row) return null
+    if (!owner.cache && storageRowPath(owner.type, row) !== identity) return null
+    if (owner.cache && !row.dataKey) return null
+    if (!owner.cache && !row.encrypted) state = (await db.getStorageEncryption())!
+    const stored = await raw.open(owner.cache ? encryptedCachePath(identity) : identity)
+    if (!stored) return null
+    // Completed uploads and all new caches are always encrypted. Only legacy
+    // pending rows need header inspection and hash-verified plaintext fallback.
+    if (row.encrypted || owner.cache || state.complete) {
+      return decryptOwned(stored, key, owner.type, row, identity)
+    }
+    let inspected = await inspectStorageObject(stored)
+    if (inspected.encrypted && !row.dataKey && (await db.getStorageRow(owner.type, owner.id))?.dataKey) {
+      inspected.stream.destroy(); continue
+    }
+    if (inspected.encrypted && row.dataKey) {
+      try { return await decryptOwned(inspected, key, owner.type, row, identity) }
+      catch {
+        // An arbitrary legacy upload can begin with our magic bytes. Only
+        // its original SQL hash can authorize that plaintext interpretation.
+        const legacy = await openObjectVersion(raw, identity, stored.version)
+        if (!legacy) continue
+        inspected = { ...legacy, encrypted: false }
+      }
+    }
+    // Only a pending row may read plaintext, and only the original upload.
+    // Verify first, then reopen that exact version before exposing bytes.
+    await verifyStoragePayload(owner.type, row, inspected.stream)
+    const current = await openObjectVersion(raw, identity, stored.version)
+    if (current) return current
+  }
+  throw new Error('Storage changed repeatedly during read')
+}
+
+// Startup enables encryption before constructing stores; validate the persisted
+// requirement even when this adapter is opened without a key.
+async function storagePolicy(db: StorageDb, key: StorageKey | null) {
+  let enabled = await db.getStorageEncryption()
+  let checkedAt = Date.now()
+  return async (fresh = false) => {
+    // Enabled is irreversible. Keyless instances cache "off" briefly, while
+    // writes and detected ciphertext always recheck the activation fence.
+    if (!enabled && (fresh || key || Date.now() - checkedAt >= STORAGE_DISABLED_TTL_MS)) {
+      const state = await db.getStorageEncryption()
+      if (state) enabled = state // A delayed disabled read cannot undo "on".
+      checkedAt = Date.now()
+    }
+    if (enabled && !key) throw new Error('Managed storage requires its configured encryption key')
+    return enabled
+  }
+}
+
+export async function createEncryptedObjectStorage(raw: RawObjectStorage, db: StorageDb, key: StorageKey | null): Promise<ObjectStorage> {
+  const mode = await storagePolicy(db, key)
+  async function open(identity: string): ReturnType<ObjectStorage['open']> {
+    logicalKey(identity)
+    const state = await mode()
+    if (identity.startsWith('avatars/')) return raw.open(identity)
+    if (state) return openEncrypted(raw, db, key!, identity, state)
+    const stored = await raw.open(identity)
+    if (!stored) return null
+    const inspected = await inspectStorageObject(stored)
+    // Activation/migration can win after the disabled-state read. Never send
+    // that ciphertext to a caller expecting the original plaintext bytes.
+    if (inspected.encrypted) {
+      let current: StorageEncryptionState | null
+      try { current = await mode(true) } catch (err) { inspected.stream.destroy(); throw err }
+      if (current) { inspected.stream.destroy(); return openEncrypted(raw, db, key!, identity, current) }
+    }
+    return inspected
+  }
+  return {
+    open,
+    get: async identity => readBytes(await open(identity)),
+    async exists(identity) {
+      logicalKey(identity)
+      const owner = storageOwner(identity), state = await mode()
+      if (state && owner) {
+        const row = await db.getStorageRow(owner.type, owner.id)
+        if (!row || (owner.cache ? !row.dataKey : storageRowPath(owner.type, row) !== identity)) return false
+      }
+      // Existence is metadata only; open() authenticates the actual bytes.
+      return await raw.head(state && owner?.cache ? encryptedCachePath(identity) : identity) !== null
+    },
+    async put(identity, bytes) {
+      logicalKey(identity)
+      const state = await mode()
+      if (identity.startsWith('avatars/')) { await raw.put(identity, bytes); return null }
+      if (!state) return putPlaintext(raw, identity, bytes, () => mode(true))
+      const owner = storageOwner(identity)
+      let dataKey: Buffer, wrapped: string | null = null
+      if (owner?.cache) {
+        const row = await db.ensureStorageDataKey(owner.type, owner.id)
+        if (!row) throw new Error('Cache owner unavailable')
+        dataKey = unwrapDataKey(key!, owner.type, row)
+      } else if (owner) {
+        if (await db.getStorageRow(owner.type, owner.id)) throw new Error('Stored payloads are immutable')
+        dataKey = randomBytes(32)
+        wrapped = wrapStorageValue(key!, dataKeyIdentity(owner.type, owner.id), dataKey)
+      } else {
+        if (!identity.startsWith('uploads/')) throw new Error('Unknown managed storage owner')
+        dataKey = Buffer.from(key!.bytes)
+      }
+      const target = owner?.cache ? encryptedCachePath(identity) : identity
+      const encrypted = encryptStorageStream(Readable.from([bytes]), dataKey, identity, bytes.length)
+      try {
+        await raw.put(target, encrypted, AbortSignal.timeout(STORAGE_WRITE_MS), undefined, bytes.length)
+        // A late cache builder must not leave a readable cache for a deleted
+        // bundle. The missing SQL key also makes failed cleanup unreadable.
+        if (owner?.cache && !await db.getStorageRow(owner.type, owner.id)) await raw.delete(target)
+        return wrapped
+      } finally { encrypted.destroy(); dataKey.fill(0) }
+    },
+    async delete(identity) {
+      logicalKey(identity)
+      await mode()
+      // Always remove both generations, including caches predating activation
+      // when migration is disabled or another instance has just enabled it.
+      for (const target of deletionPaths(identity)) await raw.delete(target)
+    },
+    async deletePrefix(prefix) {
+      logicalKey(`${prefix}validate`)
+      if (!prefix.endsWith('/')) throw new Error('Invalid storage prefix')
+      await mode()
+      const signal = AbortSignal.timeout(STORAGE_WRITE_MS)
+      for (const target of deletionPaths(prefix)) {
+        await deleteObjects(raw, target, signal)
+        await raw.prune?.(target, signal)
+      }
+    },
+  }
+}

package/server-managed/storage-maintenance.ts

@@ -0,0 +1,182 @@
+import { type StorageKey, decryptStorageStream, encryptStorageStream } from '../server-common/storage-crypto.ts'
+import { ENCRYPTED_CACHE_PREFIX, type RawObjectStorage, deleteObjects, isBlobId, openObjectVersion } from './object-storage.ts'
+import { type StorageDb, type StorageEncryptionState, type StorageMigrationRow, unwrapDataKey } from './storage-db.ts'
+import { STORAGE_UPLOAD_TTL_MS } from './storage-encryption.ts'
+import { inspectStorageObject, storageOwner, storageRowPath, verifyStoragePayload } from './storage-payload.ts'
+
+async function migrateRow(raw: RawObjectStorage, db: StorageDb, key: StorageKey, item: StorageMigrationRow, signal: AbortSignal) {
+  if (item.type === 'user') { await db.migrateStorageUserTokens(item.id); return }
+  const row = await db.ensureStorageDataKey(item.type, item.id)
+  if (!row || row.encrypted) return
+  const dataKey = unwrapDataKey(key, item.type, row), identity = storageRowPath(item.type, row)
+  try {
+    const source = await raw.open(identity, signal)
+    if (!source) {
+      if (!await db.getStorageRow(item.type, item.id)) return
+      throw new Error('Migration payload unavailable')
+    }
+    let inspected = await inspectStorageObject(source)
+    if (inspected.encrypted) {
+      // A previous PUT may have completed before a timeout or lost SQL ack.
+      try {
+        const decoded = await decryptStorageStream(inspected.stream, dataKey, identity)
+        await verifyStoragePayload(item.type, row, decoded.stream, signal)
+      } catch {
+        signal.throwIfAborted()
+        // Legacy arbitrary bytes can share the magic prefix. A matching
+        // original upload hash is required before encrypting them as plaintext.
+        const legacy = await openObjectVersion(raw, identity, source.version, signal)
+        if (!legacy) return
+        inspected = { ...legacy, encrypted: false }
+      }
+    }
+    if (!inspected.encrypted) {
+      await verifyStoragePayload(item.type, row, inspected.stream, signal)
+      signal.throwIfAborted()
+      const current = await openObjectVersion(raw, identity, source.version, signal)
+      if (!current) return
+      const encrypted = encryptStorageStream(current.stream, dataKey, identity, current.size)
+      let replaced: boolean
+      try { replaced = await raw.put(identity, encrypted, signal, source.version, current.size ?? undefined) }
+      finally { encrypted.destroy(); current.stream.destroy() }
+      if (!replaced) return
+      // Do not mark the row encrypted until the stored bytes authenticate and
+      // match the original upload (including decompression for sourcemaps).
+      const replacement = await raw.open(identity, signal)
+      if (!replacement) {
+        if (!await db.getStorageRow(item.type, item.id)) return
+        throw new Error('Migrated payload unavailable')
+      }
+      const decoded = await decryptStorageStream(replacement.stream, dataKey, identity)
+      await verifyStoragePayload(item.type, row, decoded.stream, signal)
+    }
+    signal.throwIfAborted()
+    // Another process may have renamed ciphertext but stopped before syncing
+    // the directory. Persist the observed rename even on the resume path.
+    await raw.sync?.(identity, signal)
+    await db.markStorageEncrypted(item.type, item.id, row.dataKey!)
+  } finally { dataKey.fill(0) }
+}
+
+const CLEANUP_PREFIXES = ['cache/', 'reports/', 'bundles/', ENCRYPTED_CACHE_PREFIX]
+function cleanupPosition(state: StorageEncryptionState): { prefix: number; cursor: string | null } {
+  const value = state.cleanupCursor === null ? { prefix: 0, cursor: null } : JSON.parse(state.cleanupCursor)
+  if (!Number.isInteger(value.prefix) || value.prefix < 0 || value.prefix >= CLEANUP_PREFIXES.length
+    || (value.cursor !== null && typeof value.cursor !== 'string')) throw new Error('Invalid legacy cleanup cursor')
+  return value
+}
+
+// The database directory may be shared. Recognize managed paths, including
+// disk atomic-write temps, before treating any listed file as ours to remove.
+function cleanupTarget(identity: string) {
+  const temp = /^(.+)\.([^.]+)\.tmp$/u.exec(identity)
+  const temporary = !!temp && isBlobId(temp[2]!)
+  const path = temporary ? temp![1]! : identity
+  const encryptedCache = path.startsWith(ENCRYPTED_CACHE_PREFIX)
+  if (encryptedCache && !temporary) return null
+  const owner = storageOwner(encryptedCache ? `cache/${path.slice(ENCRYPTED_CACHE_PREFIX.length)}` : path)
+  return owner ? { owner, temporary } : null
+}
+
+// Referenced-data migration never depends on Blob listings. A separate bounded
+// inventory removes legacy caches and unreferenced plaintext; ciphertext
+// orphans do not hold a recoverable data key and need no manifest-based GC.
+async function cleanupLegacy(raw: RawObjectStorage, db: StorageDb, state: StorageEncryptionState, limit: number, signal: AbortSignal) {
+  const position = cleanupPosition(state), prefix = CLEANUP_PREFIXES[position.prefix]!
+  const page = await raw.list(prefix, position.cursor, Math.min(limit, 16), signal)
+  let found = 0, retryAt: number | null = null
+  for (const object of page.objects) {
+    signal.throwIfAborted()
+    const target = cleanupTarget(object.key)
+    if (!target) continue
+    const { owner, temporary } = target
+    if (!temporary && !owner.cache) {
+      const row = await db.getStorageRow(owner.type, owner.id)
+      if (row && storageRowPath(owner.type, row) === object.key) continue
+    }
+    const stored = await raw.head(object.key, signal)
+    if (!stored) continue
+    // A killed disk writer can leave plaintext or ciphertext in its temp file.
+    // Recheck its current age and give live writes a full staging grace window.
+    if (temporary && stored.modifiedAt > Date.now() - STORAGE_UPLOAD_TTL_MS) {
+      found++
+      retryAt = Math.min(retryAt ?? Infinity, stored.modifiedAt + STORAGE_UPLOAD_TTL_MS)
+      continue
+    }
+    // Old pre-activation orphans are legacy even when their arbitrary bytes
+    // share the magic prefix. Keep a full staging grace window around enable:
+    // Blob Last-Modified has only second precision, and a new encrypted upload
+    // may still be waiting for its SQL insert.
+    let legacy = temporary || owner.cache || stored.modifiedAt < state.enabledAt - STORAGE_UPLOAD_TTL_MS
+    if (!legacy) {
+      const source = await raw.open(object.key, signal)
+      if (!source) continue
+      try {
+        if (source.version !== stored.version) throw new Error('Legacy object changed during cleanup')
+        const inspected = await inspectStorageObject(source)
+        legacy = !inspected.encrypted
+        inspected.stream.destroy()
+      } finally { source.stream.destroy() }
+    }
+    if (legacy) {
+      found++
+      if (!await raw.delete(object.key, stored.version, signal)) throw new Error('Legacy object changed during cleanup')
+    }
+  }
+  const next = page.cursor === null ? position.prefix + 1 < CLEANUP_PREFIXES.length ? { prefix: position.prefix + 1, cursor: null } : null
+    : { ...position, cursor: page.cursor }
+  // A full pass without removals closes offset-pagination gaps left by deletes.
+  await db.advanceStorageCleanup(state.cleanupCursor, next === null ? null : JSON.stringify(next), found)
+  return { processed: page.objects.length || 1, retryAt }
+}
+
+interface StorageMigrationResult extends StorageEncryptionState { retryAt: number | null }
+export async function migrateStorage(raw: RawObjectStorage, db: StorageDb, key: StorageKey,
+  { maxObjects = 64, maxMs = 150_000 } = {}): Promise<StorageMigrationResult> {
+  if (!Number.isSafeInteger(maxObjects) || maxObjects < 1 || !Number.isSafeInteger(maxMs) || maxMs < 1) throw new Error('Invalid migration budget')
+  const deadline = Date.now() + maxMs, signal = AbortSignal.timeout(maxMs)
+  const errors: unknown[] = []
+  let processed = 0, retryAt: number | null = null, state = await db.getStorageEncryption()
+  if (!state) throw new Error('Storage encryption is not enabled; configure MANAGED_STORAGE_ENCRYPTION_KEY')
+  // At most one SQL pass per invocation. Checkpoint each row, including a
+  // failure: other rows progress, and the failed row is retried next pass.
+  while (!state.complete && processed < maxObjects && Date.now() < deadline) {
+    const rows = await db.listStorageMigrationRows(state.cursor, Math.min(16, maxObjects - processed))
+    if (rows.length === 0) { await db.advanceStorageMigration(state.cursor, null); break }
+    for (const row of rows) {
+      if (Date.now() >= deadline) break
+      try { await migrateRow(raw, db, key, row, signal) }
+      catch (err) {
+        // A row started late gets a full budget next time. Skip an oversized
+        // first row until the next pass so later rows and tokens can advance.
+        if (signal.aborted && processed > 0) break
+        const message = signal.aborted
+          ? `Could not migrate ${row.type} ${row.id} within MANAGED_STORAGE_ENCRYPTION_MIGRATE_MAX_MS=${maxMs}; increase the budget`
+          : `Could not migrate ${row.type} ${row.id}`
+        errors.push(new Error(message, { cause: err }))
+      }
+      await db.advanceStorageMigration(state.cursor, row.position)
+      processed++
+      state = (await db.getStorageEncryption())!
+    }
+  }
+  state = (await db.getStorageEncryption())!
+  while (!state.cleanupComplete && processed < maxObjects && Date.now() < deadline) {
+    try {
+      const cleanup = await cleanupLegacy(raw, db, state, maxObjects - processed, signal)
+      processed += cleanup.processed
+      retryAt = cleanup.retryAt
+    }
+    catch (err) { if (!signal.aborted) errors.push(err); break }
+    state = (await db.getStorageEncryption())!
+    if (retryAt !== null) break // Wait for staging grace rather than spinning over live temp files.
+  }
+  if (errors.length > 0) throw new AggregateError(errors, 'Storage migration has pending failures', { cause: errors[0] })
+  return { ...state, retryAt }
+}
+
+export async function reapStorageUploads(raw: RawObjectStorage, db: StorageDb, now = Date.now()): Promise<number> {
+  const signal = AbortSignal.timeout(150_000)
+  await db.getStorageEncryption()
+  return deleteObjects(raw, 'uploads/', signal, now - STORAGE_UPLOAD_TTL_MS)
+}

package/server-managed/storage-payload.ts

@@ -0,0 +1,60 @@
+import { Buffer } from 'node:buffer'
+import { createHash } from 'node:crypto'
+import { Readable, Writable } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
+import { createBrotliDecompress } from 'node:zlib'
+import { STORAGE_MAGIC } from '../server-common/storage-crypto.ts'
+import { type RawObject, isBlobId } from './object-storage.ts'
+import type { StorageRow, StorageRowKind } from './storage-db.ts'
+
+export function storageRowPath(type: StorageRowKind, row: Pick<StorageRow, 'id' | 'kind'>): string {
+  return `${type === 'report' ? 'reports' : 'bundles'}/${row.id}${type === 'bundle' && row.kind === 'sourcemap' ? '.map.br' : ''}`
+}
+export function storageOwner(identity: string): { type: StorageRowKind; id: string; cache: boolean } | null {
+  const match = /^(reports|bundles)\/([^/]+)$/u.exec(identity)
+  if (match) {
+    const id = match[1] === 'bundles' ? match[2]!.replace(/\.map\.br$/u, '') : match[2]!
+    if (isBlobId(id)) return { type: match[1] === 'reports' ? 'report' : 'bundle', id, cache: false }
+  }
+  const cache = /^cache\/(?:bundles|report-sources)\/([a-f\d-]{36})\//u.exec(identity)
+  return cache && isBlobId(cache[1]!) ? { type: 'bundle', id: cache[1]!, cache: true } : null
+}
+
+// Peek without buffering the object or consuming bytes from its next reader.
+export async function inspectStorageObject(stored: RawObject): Promise<RawObject & { encrypted: boolean }> {
+  const original = stored.stream
+  const iterator = original[Symbol.asyncIterator]()
+  const first: Buffer[] = [], prefix = Buffer.alloc(STORAGE_MAGIC.length)
+  let length = 0
+  try {
+    while (length < prefix.length) {
+      const next = await iterator.next()
+      if (next.done) break
+      const bytes = Buffer.from(next.value)
+      first.push(bytes)
+      length += bytes.copy(prefix, length, 0, prefix.length - length)
+    }
+  } catch (err) { original.destroy(); throw err }
+  async function* joined() {
+    try {
+      yield* first
+      for (;;) { const next = await iterator.next(); if (next.done) return; yield next.value }
+    } finally { await iterator.return?.(); original.destroy() }
+  }
+  const stream = Readable.from(joined(), { objectMode: false })
+  stream.once('close', () => original.destroy())
+  return { ...stored, stream, encrypted: length === prefix.length && prefix.equals(STORAGE_MAGIC) }
+}
+
+// Sourcemap integrity describes the original upload, before Brotli storage.
+// Hash incrementally, including decompression, so migration stays streaming.
+export async function verifyStoragePayload(type: StorageRowKind, row: StorageRow, source: Readable, signal?: AbortSignal): Promise<void> {
+  const hash = createHash(type === 'report' ? 'sha256' : 'sha512')
+  const sink = new Writable({ write(chunk, _encoding, next) { hash.update(chunk); next() } })
+  const options = signal ? { signal } : {}
+  if (type === 'bundle' && row.kind === 'sourcemap') {
+    await pipeline(source, createBrotliDecompress(), sink, options)
+  } else await pipeline(source, sink, options)
+  const actual = type === 'report' ? hash.digest('base64url') : `sha512-${hash.digest('base64')}`
+  if (actual !== row.hash) throw new Error('Stored payload does not match its upload hash')
+}