@preventive/triage 1.0.0-alpha.4 → 1.0.0-alpha.40

package/server-common/storage-crypto.ts

@@ -0,0 +1,182 @@
+// Storage envelope v1: header, then 64 KiB plaintext chunks with full AEAD tags.
+// The nonce layout follows age's STREAM construction (counter || final flag).
+// Every write derives a new key, including rewrites of the same logical object.
+import { Buffer } from 'node:buffer'
+import { createCipheriv, createDecipheriv, createHash, hkdfSync, randomBytes } from 'node:crypto'
+import { Readable } from 'node:stream'
+
+export const STORAGE_MAGIC = Buffer.from('DeepView.storage')
+const VERSION = 1
+export const STORAGE_CHUNK_BYTES = 65_536
+export const STORAGE_HEADER_BYTES = 57
+const TAG_BYTES = 16
+const UNKNOWN_SIZE = 0xffff_ffff_ffff_ffffn
+
+export interface StorageKey { bytes: Buffer; id: string }
+
+export function parseStorageKey(value: string | null | undefined): StorageKey | null {
+  if (value == null || value === '') return null
+  const bytes = Buffer.from(value, 'base64')
+  if (bytes.length !== 32 || bytes.toString('base64') !== value) {
+    throw new Error('MANAGED_STORAGE_ENCRYPTION_KEY must be canonical base64 encoding of 32 random bytes')
+  }
+  const id = createHash('sha256').update('deepview.storage.key.v1\0').update(bytes).digest('hex').slice(0, 32)
+  return { bytes, id }
+}
+
+// Re-chunk without accumulating a file, regardless of the input's read sizes.
+async function* chunks(source: AsyncIterable<Uint8Array>, size: number): AsyncGenerator<Buffer> {
+  let length = 0, pending: Buffer[] = []
+  for await (const value of source) {
+    const input = Buffer.from(value.buffer, value.byteOffset, value.byteLength)
+    for (let offset = 0; offset < input.length;) {
+      const n = Math.min(size - length, input.length - offset)
+      pending.push(input.subarray(offset, offset + n)); length += n; offset += n
+      if (length === size) { yield Buffer.concat(pending, length); pending = []; length = 0 }
+    }
+  }
+  if (length) yield Buffer.concat(pending, length)
+}
+
+async function* frames(source: AsyncIterable<Uint8Array>, size: number) {
+  const iterator = chunks(source, size)[Symbol.asyncIterator]()
+  try {
+    let current = await iterator.next()
+    if (current.done) { yield { bytes: Buffer.alloc(0), last: true }; return }
+    while (!current.done) {
+      const next = await iterator.next()
+      yield { bytes: current.value, last: next.done === true }
+      current = next
+    }
+  } finally { await iterator.return(undefined) }
+}
+
+function nonce(index: number, last: boolean): Buffer {
+  if (!Number.isSafeInteger(index) || index < 0 || index > 0xffff_ffff) throw new Error('Storage object has too many chunks')
+  const value = Buffer.alloc(12)
+  value.writeUInt32BE(index, 7)
+  value[11] = last ? 1 : 0
+  return value
+}
+
+function objectKey(key: Uint8Array, header: Buffer, identity: string): Buffer {
+  return Buffer.from(hkdfSync('sha256', key, header.subarray(17, 49),
+    Buffer.from(JSON.stringify(['deepview.storage.v1', identity])), 32))
+}
+
+function makeHeader(size: number | null): Buffer {
+  if (size !== null && (!Number.isSafeInteger(size) || size < 0)) throw new Error('Invalid storage size')
+  const header = Buffer.alloc(STORAGE_HEADER_BYTES)
+  STORAGE_MAGIC.copy(header); header[16] = VERSION
+  randomBytes(32).copy(header, 17)
+  header.writeBigUInt64BE(size === null ? UNKNOWN_SIZE : BigInt(size), 49)
+  return header
+}
+
+export function encryptStorageStream(source: Readable, key: Uint8Array, identity: string, size: number | null): Readable {
+  async function* encrypt() {
+    const header = makeHeader(size)
+    const derived = objectKey(key, header, identity)
+    let index = 0, length = 0
+    try {
+      yield header
+      for await (const { bytes, last } of frames(source, STORAGE_CHUNK_BYTES)) {
+        length += bytes.length
+        if (last && size !== null && length !== size) throw new Error('Storage size changed while encrypting')
+        const cipher = createCipheriv('chacha20-poly1305', derived, nonce(index++, last), { authTagLength: TAG_BYTES })
+        cipher.setAAD(header)
+        yield Buffer.concat([cipher.update(bytes), cipher.final(), cipher.getAuthTag()])
+      }
+    } finally { derived.fill(0); source.destroy() }
+  }
+  const stream = Readable.from(encrypt(), { objectMode: false })
+  stream.once('close', () => source.destroy())
+  return stream
+}
+
+// Read just the envelope and authenticate the first frame before returning.
+// In particular, never trust a corrupted size or release update() output
+// before final() authenticates the complete chunk.
+export async function decryptStorageStream(source: Readable, key: Uint8Array, identity: string) {
+  const iterator = source[Symbol.asyncIterator]()
+  let done = false
+  let peek: Buffer = Buffer.alloc(0), remainder: Buffer = Buffer.alloc(0)
+  async function read(size: number): Promise<Buffer> {
+    const parts: Buffer[] = peek.length > 0 ? [peek] : []
+    let length = peek.length
+    peek = Buffer.alloc(0)
+    while (length < size) {
+      if (remainder.length === 0 && !done) {
+        const next = await iterator.next(); done = next.done === true
+        remainder = done ? Buffer.alloc(0) : next.value
+      }
+      if (remainder.length === 0 && done) break
+      const n = Math.min(size - length, remainder.length)
+      parts.push(remainder.subarray(0, n)); remainder = remainder.subarray(n); length += n
+    }
+    return Buffer.concat(parts, length)
+  }
+  let derived: Buffer | undefined
+  try {
+    const header = await read(STORAGE_HEADER_BYTES)
+    if (header.length !== STORAGE_HEADER_BYTES || !header.subarray(0, 16).equals(STORAGE_MAGIC) || header[16] !== VERSION) {
+      throw new Error('Invalid encrypted storage envelope')
+    }
+    const rawSize = header.readBigUInt64BE(49)
+    if (rawSize !== UNKNOWN_SIZE && rawSize > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error('Invalid encrypted storage size')
+    const size = rawSize === UNKNOWN_SIZE ? null : Number(rawSize)
+    derived = objectKey(key, header, identity)
+    const derivedKey = derived
+    let index = 0, length = 0
+    const nextFrame = async () => {
+      const bytes = await read(STORAGE_CHUNK_BYTES + TAG_BYTES)
+      // Peek beyond the frame: EOF authenticates the final flag, including
+      // exact chunk boundaries. Keep the byte for the next read.
+      peek = await read(1)
+      const last = peek.length === 0
+      if (bytes.length < TAG_BYTES || (index > 0 && bytes.length === TAG_BYTES)) throw new Error('Truncated encrypted storage object')
+      const decipher = createDecipheriv('chacha20-poly1305', derivedKey, nonce(index++, last), { authTagLength: TAG_BYTES })
+      decipher.setAAD(header); decipher.setAuthTag(bytes.subarray(-TAG_BYTES))
+      const plain = Buffer.concat([decipher.update(bytes.subarray(0, -TAG_BYTES)), decipher.final()])
+      length += plain.length
+      if (size !== null && (length > size || (last && length !== size))) throw new Error('Encrypted storage size mismatch')
+      return { plain, last }
+    }
+    const first = await nextFrame()
+    const decrypt = async function* () {
+      try {
+        yield first.plain
+        let last = first.last
+        while (!last) { const frame = await nextFrame(); last = frame.last; yield frame.plain }
+      } finally { derivedKey.fill(0); await iterator.return?.(); source.destroy() }
+    }
+    const stream = Readable.from(decrypt(), { objectMode: false })
+    stream.once('close', () => { derivedKey.fill(0); source.destroy() })
+    return { size, stream }
+  } catch (err) { derived?.fill(0); source.destroy(); throw err }
+}
+
+// Small SQL values (data keys, OAuth tokens and the installation sentinel).
+// Fresh salt derives a one-use wrapping key, so the nonce can be all zeroes.
+// The version and master-key ID are authenticated along with the row/field.
+export function wrapStorageValue(key: StorageKey, identity: string, value: Uint8Array): string {
+  const header = Buffer.concat([Buffer.from([1]), Buffer.from(key.id, 'hex'), randomBytes(32)])
+  const derived = Buffer.from(hkdfSync('sha256', key.bytes, header.subarray(17), 'deepview.wrap.v1', 32))
+  try {
+    const cipher = createCipheriv('chacha20-poly1305', derived, Buffer.alloc(12), { authTagLength: TAG_BYTES })
+    cipher.setAAD(Buffer.concat([header, Buffer.from(identity)]))
+    return Buffer.concat([header, cipher.update(value), cipher.final(), cipher.getAuthTag()]).toString('base64')
+  } finally { derived.fill(0) }
+}
+
+export function unwrapStorageValue(key: StorageKey, identity: string, value: string): Buffer {
+  const bytes = Buffer.from(value, 'base64'), header = bytes.subarray(0, 49)
+  if (bytes.length < 65 || bytes.toString('base64') !== value || header[0] !== 1
+    || header.subarray(1, 17).toString('hex') !== key.id) throw new Error('Invalid wrapped storage value or encryption key')
+  const derived = Buffer.from(hkdfSync('sha256', key.bytes, header.subarray(17), 'deepview.wrap.v1', 32))
+  try {
+    const decipher = createDecipheriv('chacha20-poly1305', derived, Buffer.alloc(12), { authTagLength: TAG_BYTES })
+    decipher.setAAD(Buffer.concat([header, Buffer.from(identity)])); decipher.setAuthTag(bytes.subarray(-TAG_BYTES))
+    return Buffer.concat([decipher.update(bytes.subarray(49, -TAG_BYTES)), decipher.final()])
+  } finally { derived.fill(0) }
+}

package/server-common/storage-log.ts

@@ -0,0 +1,38 @@
+import { dirname, resolve } from 'node:path'
+
+type DatabaseConfig = { dbPath: string; neonUrl?: string | null }
+
+function databaseLocation({ dbPath, neonUrl }: DatabaseConfig): string {
+  if (!neonUrl) return `SQLite ${dbPath === ':memory:' ? dbPath : resolve(dbPath)}`
+  // Report the destination, never URL credentials or query-string secrets.
+  try {
+    const url = new URL(neonUrl)
+    return `Neon Postgres ${url.host}${url.pathname}`
+  } catch { return 'Neon Postgres' }
+}
+
+export function e2eStorageLines(config: DatabaseConfig & { objstoreDir: string }): string[] {
+  return [
+    `  E2E database: ${databaseLocation(config)}`,
+    `  E2E objects: ${config.neonUrl ? 'Vercel Blob (private), {workspaceTag}/' : resolve(config.objstoreDir)}`,
+    `  E2E staging: ${config.neonUrl ? 'Vercel Blob (private), {workspaceTag}/.staging/' : resolve(config.objstoreDir, '{workspaceTag}', '.staging')}`,
+  ]
+}
+
+export function managedStorageLines(config: DatabaseConfig & { storageEncryptionKey?: string | null }): string[] {
+  const dir = dirname(config.dbPath)
+  const location = (path: string) => config.neonUrl ? `Vercel Blob (private), .managed/${path}/` : resolve(dir, path)
+  return [
+    `  Managed database: ${databaseLocation(config)}`,
+    ...(config.storageEncryptionKey ? [
+      '  Managed storage encryption: ChaCha20-Poly1305',
+      `  Managed encrypted caches: ${location('cache-encrypted-v1')}`,
+    ] : []),
+    `  Managed reports: ${location('reports')}`,
+    `  Managed bundles: ${location('bundles')}`,
+    `  Managed avatars: ${location('avatars')}`,
+    `  Managed bundle cache: ${location('cache/bundles')}`,
+    `  Managed report sources cache: ${location('cache/report-sources')}`,
+    ...(config.neonUrl ? [`  Managed upload parts: ${location('uploads')}`] : []),
+  ]
+}

package/server-common/vercel-blob.ts

@@ -0,0 +1,103 @@
+// Shared optional Vercel Blob SDK boundary for both server modes.
+import type { Readable } from 'node:stream'
+import type { Buffer } from 'node:buffer'
+
+// Minimal structural shape of the bits of `@vercel/blob` we use.
+// Shared so the optional peer dep doesn't have to type-resolve
+// for SQLite-only deployments. Real type details live in the
+// installed package; the fields/parameters we touch here are stable
+// per the SDK v2 public surface.
+//
+// `put` accepts the SDK's full `PutBody` union (string | Readable |
+// Buffer | Blob | ArrayBuffer | ReadableStream | File). We only ever
+// pass a Node `PassThrough` (a Readable), but the wider type lets
+// callers reuse this signature for future buffer/blob bodies without
+// type gymnastics. `copy` accepts `allowOverwrite` — REQUIRED for
+// version bumps since the live pathname is reused on re-upload.
+
+type VercelBlobBody = Readable | Buffer | string | Blob | ArrayBuffer | ReadableStream<Uint8Array>
+export type VercelBlobSdk = {
+  BlobNotFoundError: new () => Error
+  BlobPreconditionFailedError?: new () => Error
+  put: (
+    pathname: string,
+    body: VercelBlobBody,
+    options: {
+      access: 'private' | 'public'
+      addRandomSuffix?: boolean
+      allowOverwrite?: boolean
+      contentType?: string
+      token?: string
+      ifMatch?: string
+      multipart?: boolean
+      abortSignal?: AbortSignal
+      cacheControlMaxAge?: number
+    },
+  ) => Promise<{ url: string; pathname: string }>
+  head: (
+    pathname: string,
+    options?: { token?: string; abortSignal?: AbortSignal },
+  ) => Promise<{ size: number; pathname: string; url: string; etag?: string; uploadedAt?: Date | string | number }>
+  get: (
+    pathname: string,
+    options: {
+      access: 'private' | 'public'
+      token?: string
+      useCache?: boolean
+      headers?: Record<string, string>
+      abortSignal?: AbortSignal
+    },
+  ) => Promise<{
+    statusCode: 200 | 304
+    stream: ReadableStream<Uint8Array> | null
+    blob: { size: number | null; etag?: string; uploadedAt?: Date | string | number }
+  } | null>
+  copy: (
+    fromPathname: string,
+    toPathname: string,
+    options: {
+      access: 'private' | 'public'
+      addRandomSuffix?: boolean
+      allowOverwrite?: boolean
+      token?: string
+      contentType?: string
+      cacheControlMaxAge?: number
+    },
+  ) => Promise<{ url: string; pathname: string }>
+  del: (
+    urlOrPathname: string | string[],
+    options?: { token?: string; abortSignal?: AbortSignal; ifMatch?: string },
+  ) => Promise<void>
+  list: (options: {
+    prefix?: string
+    cursor?: string
+    limit?: number
+    mode?: 'expanded' | 'folded'
+    token?: string
+    abortSignal?: AbortSignal
+  }) => Promise<{
+    // `uploadedAt` (a Date per the SDK v2 surface) is the blob's
+    // creation time — used by the reaper's GC grace window. Optional
+    // in the type because the `folded` listing path ignores it (only
+    // `listLiveBlobs` reads it, and it uses the default expanded mode).
+    blobs: Array<{ pathname: string; size: number; uploadedAt?: Date | string | number }>
+    folders?: string[]
+    cursor?: string
+    hasMore: boolean
+  }>
+}
+
+// Recognise "blob is gone" errors uniformly across read/write/delete
+// paths so callers can treat them as success (delete) or not-found (read).
+// SDK error subclasses retain `.name === 'Error'`; identify the missing-blob
+// class from the same lazily loaded SDK instance that performs the request.
+// Store/configuration failures must propagate, never masquerade as cache misses.
+export function isNotFound(err: unknown, sdk: Pick<VercelBlobSdk, 'BlobNotFoundError'>): boolean {
+  return err instanceof sdk.BlobNotFoundError
+}
+
+export async function loadVercelBlobSdk(): Promise<VercelBlobSdk> {
+  // @ts-ignore optional peer dep: '@vercel/blob'
+  const mod = (await import('@vercel/blob')) as VercelBlobSdk
+  return mod
+}