@preventive/triage 1.0.0-alpha.5 → 1.0.0-alpha.50

package/server-managed/npm-risk.ts

@@ -0,0 +1,285 @@
+// What calls for a look at an npm version, as risk tells it (risk/), for the
+// managed viewer: risk is asked on the server alone, its records kept where
+// each belongs by its type (risk/npm/registry.js, assess.js):
+//
+// - `risk/npm/assessments`, what of an assessment holds whatever the time:
+//   a public package's in the public npm cache, as its pretty-printed files
+//   are (storage-payload.ts PUBLIC_CACHE), by a hash of its key; a private
+//   one's in this instance's memory alone.
+// - `risk/npm/versions`, what a version's document says, which npm never
+//   changes once published: a public package's in the database's upstream
+//   cache table, shared by every instance; a private one's in memory.
+// - `risk/npm/downloads`, a package's downloads a day, which risk tells as
+//   kept for a week: a public package's in the database too, so that another
+//   instance, or this one restarted, asks npm's downloads API, which answers
+//   some 40 lookups a minute, none of them again that week, each removed a
+//   day past it (npm-kept.ts pruneKept); a private one's in memory.
+// - `risk/npm/packages`, `risk/npm/old-versions`, `risk/npm/public` and
+//   `risk/npm/advisories`, which risk asks again after an hour: a public
+//   package's in memory and in the database too, read from memory first, so
+//   that a cold instance, as a serverless one starts, reads no whole document
+//   of npm's again that one before it read within the hour; by a hash of
+//   their type and key, as a dependency's spec runs to any length, each
+//   removed a day on, and one past 256 KiB as JSON (a package's deprecated
+//   versions, its messages and all, run long) in memory alone. A private
+//   one's in memory.
+// - `npm/tarballs`: none asked, as tarballs are read as the viewer's loads
+//   read them, kept where they keep them (npm-loads.ts loadNpmTarball), and
+//   an assessment kept needs none.
+//
+// What memory keeps is held to 2,000 records and 64 MiB as JSON, the first
+// kept going first.
+//
+// A cache failure is a miss, never a failed assessment. Packages' documents
+// are read as every other npm document here is, within their bounds and
+// shared budget (npm-packages.ts readNpmPackument), and tarballs as the
+// viewer's loads read them, bounded as they arrive and among the loads made
+// at once. Assessments made at once are bounded as those loads are, each
+// unpacking up to two tarballs: one past them is refused as busy.
+import { Buffer } from 'node:buffer'
+import { createHash } from 'node:crypto'
+import { buffer } from 'node:stream/consumers'
+import type { CacheStore } from '@preventive/upstream/npm.js'
+import { DEFAULT_MINIMUM_RELEASE_AGE, type NpmAssessment, assessNpm, npmReadyDownload, npmReadyDownloads } from '../risk/index.js'
+import { CacheMissError, type CacheStorage } from './cache-storage.ts'
+import { npmVersionStore } from './npm-kept.ts'
+import { loadNpmTarball } from './npm-loads.ts'
+import { NpmPackageError, type NpmVersionDocument, npmTokenReads, readNpmPackument, readNpmVersion } from './npm-packages.ts'
+import { npmPackageScope } from '../common/managed/npm-packages.js'
+import { pruneKept } from './npm-kept.ts'
+import type { UpstreamCacheStore } from './upstream-cache.ts'
+
+const MEMORY_ENTRIES = 2000
+// As npm-loads.ts MAX_ACTIVE_LOADS.
+const MAX_ACTIVE_ASSESSMENTS = 4
+// Of them, those no reader waits on (`spare`), such as a scan's: the rest are
+// left for readers.
+export const MAX_SPARE_ASSESSMENTS = 2
+let assessing = 0
+// What memory holds at most, by its records' length as JSON: the first kept
+// go first past it, and a record past an eighth of it isn't kept at all.
+const MEMORY_LENGTH = 64 * 1024 * 1024
+let memoryLength = 0
+const memory = new Map<string, { value: unknown; length: number }>()
+function forget(key: string) {
+  const kept = memory.get(key)
+  if (!kept) return
+  memory.delete(key)
+  memoryLength -= kept.length
+}
+function remember(key: string, value: unknown) {
+  forget(key)
+  const length = JSON.stringify(value)?.length ?? 0
+  if (length > MEMORY_LENGTH / 8) return
+  memory.set(key, { value, length })
+  memoryLength += length
+  while (memory.size > MEMORY_ENTRIES || memoryLength > MEMORY_LENGTH) forget(memory.keys().next().value!)
+}
+
+const digest = (type: string, key: string) => createHash('sha256').update(`${type}\0${key}`).digest('hex')
+const hashed = (type: string, key: string) => `risk/${digest(type, key)}`
+// What of a public package is kept in the database by its key, and for how
+// long its downloads are: risk's week, and a day more.
+const IN_DATABASE = new Set(['risk/npm/versions', 'risk/npm/downloads'])
+const DOWNLOADS_KEPT_MS = 8 * 24 * 60 * 60_000
+// What of a public package risk asks again after an hour, kept in memory and
+// in the database (HOURLY_PREFIX), each a day, none past HOURLY_LENGTH as
+// JSON there.
+const HOURLY = new Set(['risk/npm/packages', 'risk/npm/old-versions', 'risk/npm/public', 'risk/npm/advisories'])
+const HOURLY_PREFIX = 'risk/hourly/'
+const HOURLY_KEPT_MS = 24 * 60 * 60_000
+const HOURLY_LENGTH = 256 * 1024
+const databaseKey = (type: string, key: string) => HOURLY.has(type) ? `${HOURLY_PREFIX}${digest(type, key)}` : `${type}/${key}`
+
+export interface NpmRiskStores {
+  db: UpstreamCacheStore
+  npmCache?: CacheStorage | undefined
+  debug?: boolean | undefined
+}
+
+// A cache that reads and writes many of a type at once too, as risk's
+// readying of downloads asks (npmReadyDownloads), and reads records of any
+// types at once, as risk reads a package's record and its versions'
+// (registry.js npmPackageFacts): one read and one write of the database each,
+// not one a key, as a request's queries take turns on its connection
+// (db-neon.ts).
+export interface NpmRiskCache extends CacheStore {
+  readEach(entries: [type: string, key: string][]): Promise<unknown[]>
+  readMany(type: string, keys: string[]): Promise<Map<string, unknown>>
+  writeMany(type: string, entries: [key: string, value: unknown][]): Promise<void>
+}
+
+// Risk's cache for one package: `isPublic` whether anyone can read it, so
+// that what is kept of it can be shared.
+export function npmRiskCache({ db, npmCache, debug }: NpmRiskStores, isPublic: boolean): NpmRiskCache {
+  const warn = (what: string, error: unknown) => { if (debug) console.warn(`managed: risk cache ${what} failed:`, error) }
+  const memoryKey = (type: string, key: string) => `${isPublic ? '' : 'private\0'}${type}\0${key}`
+  // Whether a record of `type` is read of the database: unless it is one
+  // kept in memory too (HOURLY) and there.
+  const ofDatabase = (type: string, key: string) => isPublic && (IN_DATABASE.has(type) || HOURLY.has(type) && !memory.has(memoryKey(type, key)))
+  // A record as the database keeps it, read: an hourly one kept in memory
+  // too, for the next to ask.
+  const readOf = (type: string, key: string, row: string | null | undefined) => {
+    const value: unknown = row == null ? null : JSON.parse(row)
+    if (value !== null && HOURLY.has(type)) remember(memoryKey(type, key), value)
+    return value
+  }
+  const cache: NpmRiskCache = {
+    async readEach(entries) {
+      const asked = new Set(entries.flatMap(([type, key], i) => ofDatabase(type, key) ? [i] : []))
+      let rows = new Map<string, string>()
+      if (asked.size > 0) {
+        try { rows = await db.getUpstreamCacheEntries([...asked].map(i => databaseKey(...entries[i]!))) } catch (error) { warn('read', error) }
+      }
+      return Promise.all(entries.map(([type, key], i) => {
+        if (!asked.has(i)) return cache.read(type, key)
+        try { return readOf(type, key, rows.get(databaseKey(type, key))) } catch (error) { warn('read', error); return null }
+      }))
+    },
+    async readMany(type, keys) {
+      const values = await cache.readEach(keys.map(key => [type, key]))
+      return new Map(keys.map((key, i) => [key, values[i]]))
+    },
+    async writeMany(type, entries) {
+      if (!isPublic || !IN_DATABASE.has(type)) { for (const [key, value] of entries) await cache.write(type, key, value); return }
+      const at = Date.now()
+      try { await db.setUpstreamCacheEntries(entries.map(([key, value]) => [`${type}/${key}`, JSON.stringify(value)]), at) } catch (error) { warn('write', error); return }
+      if (type === 'risk/npm/downloads') await pruneKept(db, `${type}/`, DOWNLOADS_KEPT_MS, at, error => { warn('prune', error) })
+    },
+    async read(type, key) {
+      if (type === 'npm/tarballs') return null
+      try {
+        if (isPublic && type === 'risk/npm/assessments' && npmCache) return JSON.parse((await buffer((await npmCache.open(hashed(type, key))).stream)).toString('utf8'))
+        if (ofDatabase(type, key)) return readOf(type, key, await db.getUpstreamCacheEntry(databaseKey(type, key)))
+      } catch (error) { if (!(error instanceof CacheMissError)) warn('read', error); return null }
+      return memory.get(memoryKey(type, key))?.value ?? null
+    },
+    async write(type, key, value) {
+      if (type === 'npm/tarballs') return
+      if (isPublic && HOURLY.has(type)) remember(memoryKey(type, key), value)
+      try {
+        if (isPublic && type === 'risk/npm/assessments' && npmCache) { await npmCache.put(hashed(type, key), Buffer.from(JSON.stringify(value))); return }
+        if (isPublic && IN_DATABASE.has(type)) {
+          const at = Date.now()
+          await db.setUpstreamCacheEntry(databaseKey(type, key), JSON.stringify(value), at)
+          if (type === 'risk/npm/downloads') await pruneKept(db, `${type}/`, DOWNLOADS_KEPT_MS, at, error => { warn('prune', error) })
+          return
+        }
+        if (isPublic && HOURLY.has(type)) {
+          const json = JSON.stringify(value)
+          if (json.length > HOURLY_LENGTH) return
+          const at = Date.now()
+          await db.setUpstreamCacheEntry(databaseKey(type, key), json, at)
+          await pruneKept(db, HOURLY_PREFIX, HOURLY_KEPT_MS, at, error => { warn('prune', error) })
+          return
+        }
+      } catch (error) { warn('write', error); return }
+      remember(memoryKey(type, key), value)
+    },
+  }
+  return cache
+}
+
+// A version's tarball as the viewer's loads read it (npm-loads.ts), its bytes
+// held to the integrity its reader's document names: given up on, and the
+// load released, once `signal` aborts, the load going on for its other
+// readers, so that a reader gone holds no assessment's place.
+async function tarballOf(doc: NpmVersionDocument | undefined, signal: AbortSignal): Promise<Uint8Array> {
+  if (!doc) throw new NpmPackageError(502, 'upstream-invalid')
+  signal.throwIfAborted()
+  const load = loadNpmTarball(doc)
+  let onAbort = () => {}
+  const aborted = new Promise<never>((_, reject) => { onAbort = () => reject(signal.reason); signal.addEventListener('abort', onAbort, { once: true }) })
+  try { return await Promise.race([load.result, aborted]) } finally {
+    signal.removeEventListener('abort', onAbort)
+    load.release()
+  }
+}
+
+// The version `doc` names assessed against `old`'s, where given, each as the
+// reader's readNpmVersion read it, or against the one risk picks of its
+// versions (`'pick'`, assessNpm `oldVersion: true`), its document read as the
+// open version's is where its tarball is wanted: trusted however new and however few
+// download it where a team lists its scope (`teamScoped`), its own packages;
+// else held to risk's minimum release age and flagged as unpopular. A new
+// dependency on a package of a scope a team lists (`ownScope`, asked once a
+// scope) is told no tier nor age, as an own package. Its
+// documents are read with npm's token where the reader may have it
+// (`privileged`, as readNpmVersion), its dependencies' without; what is kept
+// of it is shared only where neither version is private, and OSV is asked of
+// its advisories only where it isn't, a private package's name never sent to
+// OSV, as it never is to Socket (http.ts). A `spare` one, no
+// reader waiting on it, is made only within MAX_SPARE_ASSESSMENTS, and tells
+// its dependencies without asking npm of their packages (assessNpm
+// `dependencies`), nor its advisories, which a scan asks of a lockfile's
+// packages at once (repository-risk.ts).
+export function npmRisk(doc: NpmVersionDocument, old: NpmVersionDocument | 'pick' | null,
+  { teamScoped, ownScope = () => Promise.resolve(false), fromEmpty = false, privileged, signal, spare = false, ...stores }: NpmRiskStores & { teamScoped: boolean;
+    ownScope?: (scope: string) => Promise<boolean>; fromEmpty?: boolean; privileged: boolean; signal: AbortSignal; spare?: boolean }): Promise<NpmAssessment> {
+  const scopes = new Map<string, Promise<boolean>>()
+  const owned = (name: string) => {
+    const scope = npmPackageScope(name)
+    if (scope === null) return false
+    if (!scopes.has(scope)) scopes.set(scope, ownScope(scope))
+    return scopes.get(scope)!
+  }
+  const given = old === 'pick' ? null : old
+  // The one picked, read as the viewer reads a version, kept a day where public.
+  const documentOf = async (version: string) => [doc, given].find(read => read?.version === version)
+    ?? await readNpmVersion(doc.name, version, privileged, signal, npmVersionStore(stores.db, stores.debug)) ?? undefined
+  return bounded(spare, () => assessNpm(doc.name, doc.version, { oldVersion: old === 'pick' ? true : old?.version ?? null, fromEmpty,
+    minimumReleaseAge: teamScoped ? 0 : DEFAULT_MINIMUM_RELEASE_AGE,
+    popularity: !teamScoped, cache: npmRiskCache(stores, !doc.private && !given?.private), signal, dependencies: !spare, advisories: !spare, osv: !doc.private, owned,
+    packument: (name, { authorized }) => readNpmPackument(name, privileged && authorized, signal),
+    tarball: async (_name, version) => tarballOf(await documentOf(version), signal) }))
+}
+
+// An assessment made within MAX_ACTIVE_ASSESSMENTS, a `spare` one within
+// MAX_SPARE_ASSESSMENTS, all across the server; one past them refused as
+// busy (`npm-busy`).
+async function bounded(spare: boolean, assess: () => Promise<NpmAssessment>): Promise<NpmAssessment> {
+  if (assessing >= (spare ? MAX_SPARE_ASSESSMENTS : MAX_ACTIVE_ASSESSMENTS)) throw new NpmPackageError(429, 'npm-busy')
+  assessing++
+  try { return await assess() } finally { assessing-- }
+}
+
+// A version npm has no document of, as risk tells one npm removed, as it
+// removes every one it finds malware in (assessNpm): its package taken down
+// by npm, as a scan's (`spare`), and its advisories of malware, npm's and,
+// where anyone can read its package, OSV's, which alone has some of those
+// npm removed (nx's), a private package's name never sent to OSV. Its
+// package's document read anonymously, else with npm's token alone where
+// that reads more for this reader (`privileged`, npmTokenReads), and kept as
+// a private package's is; no
+// tarball read, nothing of the version being left. Spare, within
+// MAX_SPARE_ASSESSMENTS as a scan's others are. An NpmRiskError where
+// nothing tells of it.
+export function npmRiskRemoved(name: string, version: string,
+  { teamScoped, privileged, signal, ...stores }: NpmRiskStores & { teamScoped: boolean; privileged: boolean; signal: AbortSignal }): Promise<NpmAssessment> {
+  return bounded(true, async () => {
+    const open = await readNpmPackument(name, false, signal)
+    return assessNpm(name, version, { minimumReleaseAge: teamScoped ? 0 : DEFAULT_MINIMUM_RELEASE_AGE, popularity: false, downloads: false, dependencies: false,
+      osv: open !== null, cache: npmRiskCache(stores, false), signal,
+      packument: (read, { authorized }) => read === name
+        ? open || !npmTokenReads(name, privileged && authorized) ? Promise.resolve(open) : readNpmPackument(name, true, signal, { anonymous: false })
+        : readNpmPackument(read, privileged && authorized, signal),
+      tarball: () => Promise.reject(new NpmPackageError(502, 'upstream-invalid')) })
+  })
+}
+
+// What a scan readies before assessing its packages (repository-risk.ts):
+// their downloads, as risk asks them, kept where a public package's are
+// (npmRiskCache), so that their assessments wait on npm's downloads API for
+// none. At once, those kept and unscoped ones, in bulk, all public, as only a
+// scoped package can be private; each other one on its own
+// (npmReadyScanDownload), kept only where npm counts some, which it does of
+// public packages alone: a private one's none, as of one it has no such
+// package of, isn't kept where public ones are, for its assessment to ask
+// its own where private ones are.
+export const npmReadyScanDownloads = (names: string[], ready: (name: string) => void, stores: NpmRiskStores, signal: AbortSignal) =>
+  npmReadyDownloads(names, ready, { cache: npmRiskCache(stores, true), signal })
+export function npmReadyScanDownload(name: string, stores: NpmRiskStores, signal: AbortSignal) {
+  const cache = npmRiskCache(stores, true)
+  const counted = (value: unknown) => !(value && typeof value === 'object' && 'none' in value)
+  return npmReadyDownload(name, { cache: { read: cache.read, write: (type, key, value) => counted(value) ? cache.write(type, key, value) : Promise.resolve() }, signal })
+}

package/server-managed/object-storage-disk.ts

@@ -0,0 +1,148 @@
+import { randomUUID } from 'node:crypto'
+import { link, lstat, mkdir, open, readdir, rename, rm, rmdir, stat } from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+import { type ObjectPage, type RawObjectStorage, objectPath } from './object-storage.ts'
+
+function missing(err: unknown): boolean { return (err as NodeJS.ErrnoException).code === 'ENOENT' }
+function version(info: { dev: number; ino: number; size: number; mtimeMs: number; ctimeMs: number }): string {
+  return JSON.stringify([info.dev, info.ino, info.size, info.mtimeMs, info.ctimeMs])
+}
+
+async function syncDirectory(dir: string): Promise<void> {
+  try {
+    const directory = await open(dir, 'r')
+    try { await directory.sync() } finally { await directory.close() }
+  } catch (err) {
+    // Windows and some mounted filesystems cannot open/fsync directories.
+    // Preserve real I/O failures; file fsync remains mandatory on every write.
+    if (!['EINVAL', 'ENOTSUP', 'EOPNOTSUPP', 'ENOSYS', 'EISDIR', 'EBADF', 'EPERM'].includes((err as NodeJS.ErrnoException).code ?? '')) throw err
+  }
+}
+
+// Keyset pagination survives concurrent file creation/deletion. A final sweep
+// catches objects inserted before the cursor during migration.
+async function listFiles(dir: string, prefix: string, cursor: string | null, limit: number, signal?: AbortSignal): Promise<ObjectPage> {
+  const objects: ObjectPage['objects'] = []
+  async function visit(key: string): Promise<void> {
+    signal?.throwIfAborted()
+    let entries
+    try {
+      if (!(await lstat(join(dir, key))).isDirectory()) {
+        console.warn('managed-storage: skipping unsupported directory', JSON.stringify(key))
+        return
+      }
+      entries = await readdir(join(dir, key), { withFileTypes: true })
+    }
+    catch (err) { if (missing(err)) return; throw err }
+    const order = (entry: typeof entries[number]) => `${entry.name}${entry.isDirectory() ? '/' : ''}`
+    entries.sort((a, b) => order(a) < order(b) ? -1 : order(a) > order(b) ? 1 : 0)
+    for (const entry of entries) {
+      signal?.throwIfAborted()
+      const child = `${key}${entry.name}`
+      if (entry.isSymbolicLink() || (!entry.isDirectory() && !entry.isFile())) {
+        console.warn('managed-storage: skipping unsupported entry', JSON.stringify(child))
+        continue
+      }
+      try { objectPath(child) }
+      catch { console.warn('managed-storage: skipping invalid entry', JSON.stringify(child)); continue }
+      if (entry.isDirectory()) {
+        const folder = `${child}/`
+        if (cursor && folder < cursor && !cursor.startsWith(folder)) continue
+        await visit(folder)
+        if (objects.length > limit) return
+      }
+      else if (entry.isFile() && (!cursor || child > cursor)) {
+        try { objects.push({ key: objectPath(child), modifiedAt: (await stat(join(dir, child))).mtimeMs }) }
+        catch (err) { if (!missing(err)) throw err }
+        if (objects.length > limit) return
+      }
+    }
+    // Another migration worker can observe an unlink before its writer has
+    // synced the directory. Persist those observed removals before an empty
+    // inventory can mark the shared migration complete.
+    try { await syncDirectory(join(dir, key)) } catch (err) { if (!missing(err)) throw err }
+  }
+  if (!prefix.endsWith('/') || !Number.isInteger(limit) || limit < 1) throw new Error('Invalid storage page')
+  objectPath(`${prefix}validate`)
+  await visit(prefix)
+  return { objects: objects.slice(0, limit), cursor: objects.length > limit ? objects[limit - 1]!.key : null }
+}
+
+export function createDiskObjectStorage(dir: string): RawObjectStorage {
+  const path = (key: string) => join(dir, objectPath(key))
+  return {
+    async head(key, signal) {
+      signal?.throwIfAborted()
+      try {
+        const info = await stat(path(key))
+        return { size: info.size, version: version(info), modifiedAt: info.mtimeMs }
+      } catch (err) { if (missing(err)) return null; throw err }
+    },
+    async open(key, signal) {
+      signal?.throwIfAborted()
+      let file
+      try { file = await open(path(key), 'r') }
+      catch (err) { if (missing(err)) return null; throw err }
+      try {
+        const info = await file.stat()
+        return { size: info.size, version: version(info), modifiedAt: info.mtimeMs,
+          stream: file.createReadStream(signal ? { signal } : {}) }
+      } catch (err) { await file.close(); throw err }
+    },
+    async put(key, bytes, signal, expected) {
+      const target = path(key)
+      // Temp-file + rename makes replacement atomic to readers. Concurrent
+      // migrations use the same persisted data key and immutable plaintext.
+      const temp = `${target}.${randomUUID()}.tmp`
+      await mkdir(dirname(target), { recursive: true })
+      const file = await open(temp, 'wx', 0o600)
+      try {
+        await file.writeFile(bytes, signal ? { signal } : {})
+        await file.sync(); await file.close()
+        signal?.throwIfAborted()
+        if (expected === null) {
+          try { await link(temp, target) }
+          catch (err) { if ((err as NodeJS.ErrnoException).code === 'EEXIST') return false; throw err }
+        } else if (expected === undefined) await rename(temp, target)
+        else {
+          try { if (version(await stat(target)) !== expected) return false }
+          catch (err) { if (missing(err)) return false; throw err }
+          await rename(temp, target)
+        }
+        await syncDirectory(dirname(target))
+        return true
+      } finally { await file.close(); await rm(temp, { force: true }) }
+    },
+    async delete(key, expected, signal) {
+      signal?.throwIfAborted()
+      try {
+        if (expected !== undefined && version(await stat(path(key))) !== expected) return false
+        await rm(path(key), { force: true })
+        // Persist plaintext removal before migration checkpoints completion.
+        // Otherwise a crash could restore the directory entry after SQL has
+        // durably disabled further plaintext inventory sweeps.
+        await syncDirectory(dirname(path(key)))
+        return true
+      } catch (err) { if (missing(err)) return true; throw err }
+    },
+    async sync(key, signal) { signal?.throwIfAborted(); await syncDirectory(dirname(path(key))) },
+    async prune(prefix, signal) {
+      if (!prefix.endsWith('/')) throw new Error('Invalid storage prefix')
+      objectPath(`${prefix}validate`)
+      async function visit(folder: string): Promise<void> {
+        signal?.throwIfAborted()
+        let entries
+        try {
+          if (!(await lstat(folder)).isDirectory()) return
+          entries = await readdir(folder, { withFileTypes: true })
+        }
+        catch (err) { if (missing(err)) return; throw err }
+        for (const entry of entries) if (entry.isDirectory()) await visit(join(folder, entry.name))
+        try { await rmdir(folder); await syncDirectory(dirname(folder)) }
+        catch (err) { if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes((err as NodeJS.ErrnoException).code ?? '')) throw err }
+      }
+      await visit(join(dir, prefix))
+    },
+    list: (prefix, cursor, limit, signal) => listFiles(dir, prefix, cursor, limit, signal),
+  }
+}

package/server-managed/object-storage-vercel.ts

@@ -0,0 +1,67 @@
+import { Readable } from 'node:stream'
+import { Buffer } from 'node:buffer'
+import { type VercelBlobSdk, isNotFound, loadVercelBlobSdk } from '../server-common/vercel-blob.ts'
+import { type RawObjectStorage, objectPath } from './object-storage.ts'
+
+export async function openVercelObjectStorage(token: string, sdk?: VercelBlobSdk): Promise<RawObjectStorage> {
+  const blobs = sdk ?? await loadVercelBlobSdk()
+  const path = (key: string) => `.managed/${objectPath(key)}`
+  const storage: RawObjectStorage = {
+    async head(key, signal) {
+      try {
+        const meta = await blobs.head(path(key), { token, ...(signal ? { abortSignal: signal } : {}) })
+        if (!meta.etag) throw new Error('Missing object version')
+        return { size: meta.size, version: meta.etag, modifiedAt: new Date(meta.uploadedAt ?? Date.now()).getTime() }
+      } catch (err) { if (isNotFound(err, blobs)) return null; throw err }
+    },
+    async open(key, signal) {
+      try {
+        // Read the stored representation: transport compression can weaken its
+        // ETag, which cannot satisfy ifMatch, and changes the advertised size.
+        const result = await blobs.get(path(key), { token, access: 'private', useCache: false,
+          headers: { 'accept-encoding': 'identity' }, ...(signal ? { abortSignal: signal } : {}) })
+        if (!result) return null
+        if (result.statusCode !== 200 || !result.stream || !result.blob.etag) {
+          await result.stream?.cancel()
+          throw new Error('Unexpected blob response or missing object version')
+        }
+        return { size: result.blob.size === 0 ? null : result.blob.size, version: result.blob.etag,
+          modifiedAt: new Date(result.blob.uploadedAt ?? Date.now()).getTime(),
+          stream: Readable.fromWeb(result.stream as Parameters<typeof Readable.fromWeb>[0], signal ? { signal } : {}) }
+      } catch (err) { if (isNotFound(err, blobs)) return null; throw err }
+    },
+    async put(key, bytes, signal, expected, sizeHint) {
+      const multipart = (sizeHint ?? (Buffer.isBuffer(bytes) ? bytes.length : Infinity)) >= 5 * 1024 * 1024
+      try { await blobs.put(path(key), bytes, { token, access: 'private', addRandomSuffix: false, allowOverwrite: expected !== null,
+        ...(expected ? { ifMatch: expected } : {}), multipart, contentType: 'application/octet-stream', cacheControlMaxAge: 60, ...(signal ? { abortSignal: signal } : {}) })
+        return true
+      } catch (err) {
+        if (blobs.BlobPreconditionFailedError && err instanceof blobs.BlobPreconditionFailedError) return false
+        // The SDK uses a generic error for an occupied pathname. Reconcile a
+        // create-only failure against storage; the caller verifies the winner.
+        if (expected === null && await storage.head(key, signal)) return false
+        throw err
+      }
+    },
+    async delete(key, version, signal) {
+      signal?.throwIfAborted()
+      try { await blobs.del(path(key), { token, ...(version ? { ifMatch: version } : {}), ...(signal ? { abortSignal: signal } : {}) }); return true }
+      catch (err) {
+        if (isNotFound(err, blobs)) return true
+        if (blobs.BlobPreconditionFailedError && err instanceof blobs.BlobPreconditionFailedError) return false
+        throw err
+      }
+    },
+    async list(prefix, cursor, limit, signal) {
+      signal?.throwIfAborted()
+      if (!prefix.endsWith('/') || !Number.isInteger(limit) || limit < 1) throw new Error('Invalid storage page')
+      objectPath(`${prefix}validate`)
+      const page = await blobs.list({ prefix: `.managed/${prefix}`, token, limit, ...(cursor ? { cursor } : {}), ...(signal ? { abortSignal: signal } : {}) })
+      if (page.hasMore && (!page.cursor || page.cursor === cursor)) throw new Error('Invalid blob pagination')
+      return { objects: page.blobs.filter(blob => blob.pathname.startsWith(`.managed/${prefix}`)).map(blob => ({ key: blob.pathname.slice('.managed/'.length),
+        modifiedAt: new Date(blob.uploadedAt ?? Date.now()).getTime() })),
+      cursor: page.hasMore ? page.cursor! : null }
+    },
+  }
+  return storage
+}

package/server-managed/object-storage.ts

@@ -0,0 +1,75 @@
+import type { Buffer } from 'node:buffer'
+import type { Readable } from 'node:stream'
+import type { OpenedBlob } from './blob-store.ts'
+import { validateCacheKey } from './cache-storage.ts'
+
+export const ENCRYPTED_CACHE_PREFIX = 'cache-encrypted-v1/'
+const STORAGE_PREFIXES = ['reports/', 'bundles/', 'avatars/', 'uploads/', 'cache/', ENCRYPTED_CACHE_PREFIX]
+export interface RawObject extends OpenedBlob { version: string; modifiedAt: number }
+type ObjectMetadata = Omit<RawObject, 'stream'>
+interface ListedObject { key: string; modifiedAt: number }
+export interface ObjectPage { objects: ListedObject[]; cursor: string | null }
+export interface RawObjectStorage {
+  open(key: string, signal?: AbortSignal): Promise<RawObject | null>
+  head(key: string, signal?: AbortSignal): Promise<ObjectMetadata | null>
+  prune?(prefix: string, signal?: AbortSignal): Promise<void>
+  // Persist an observed disk replacement before its SQL migration checkpoint.
+  sync?(key: string, signal?: AbortSignal): Promise<void>
+  // A plaintext size hint selects multipart uploads without buffering streams.
+  // Publish atomically only after consuming the body through clean EOF; a body
+  // error must preserve the existing object (migration verifies while streaming).
+  // A null expected version creates only if absent, without replacing a winner.
+  put(key: string, bytes: Buffer | Readable, signal?: AbortSignal, expected?: string | null, sizeHint?: number): Promise<boolean>
+  // Return false if the original version changed; do not delete its replacement.
+  delete(key: string, version?: string, signal?: AbortSignal): Promise<boolean>
+  list(prefix: string, cursor: string | null, limit: number, signal?: AbortSignal): Promise<ObjectPage>
+}
+export interface ObjectStorage {
+  open(key: string): Promise<OpenedBlob | null>
+  get(key: string): Promise<Buffer | null>
+  exists(key: string): Promise<boolean>
+  put(key: string, bytes: Buffer, options?: { replaceReport?: boolean }): Promise<string | null>
+  delete(key: string): Promise<void>
+  // Every object under `prefix`, but those under `keep`, a prefix within it.
+  deletePrefix(prefix: string, keep?: string): Promise<void>
+}
+
+export function isBlobId(value: string): boolean {
+  return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u.test(value)
+}
+
+export function objectPath(key: string): string {
+  validateCacheKey(key)
+  if (!STORAGE_PREFIXES.some(prefix => key.startsWith(prefix))) throw new Error('Invalid managed storage namespace')
+  return key
+}
+
+// Hash verification and migration may consume a stream before it can be used.
+// Reopen only the same object version, closing any concurrent replacement.
+export async function openObjectVersion(raw: RawObjectStorage, key: string, version: string, signal?: AbortSignal): Promise<RawObject | null> {
+  const current = await raw.open(key, signal)
+  if (current?.version === version) return current
+  current?.stream.destroy()
+  return null
+}
+
+export async function deleteObjects(raw: RawObjectStorage, prefix: string, signal: AbortSignal, before?: number, keep?: string): Promise<number> {
+  const eligible = (modifiedAt: number) => before === undefined || modifiedAt < before
+  // Finish listing before deleting: provider cursors may be offset-based.
+  const cursors = new Set<string>(), keys = new Set<string>()
+  let cursor: string | null = null
+  do {
+    const page = await raw.list(prefix, cursor, 100, signal)
+    for (const object of page.objects) if (eligible(object.modifiedAt) && (keep === undefined || !object.key.startsWith(keep))) keys.add(object.key)
+    cursor = page.cursor
+    if (cursor !== null && cursors.has(cursor)) throw new Error('Invalid blob pagination')
+    if (cursor !== null) cursors.add(cursor)
+  } while (cursor !== null)
+  let removed = 0
+  for (const key of keys) {
+    const stored = await raw.head(key, signal)
+    // Ignore stale listings and preserve concurrent replacements.
+    if (stored && eligible(stored.modifiedAt) && await raw.delete(key, stored.version, signal)) removed++
+  }
+  return removed
+}

package/server-managed/package-entry-points.ts

@@ -0,0 +1,51 @@
+import { Buffer } from 'node:buffer'
+import { normalizeTeamPath } from './repo-path.ts'
+
+export const MAX_PACKAGE_BYTES = 256 * 1024
+const MAX_ENTRY_POINTS = 100
+
+// Declared paths, not a Node resolver: never guess defaults, expand export
+// patterns, or execute package code. Keep every suggestion in its package.
+export function packageEntryPoints(manifest: unknown, directory: string): string[] {
+  if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) return []
+  const pkg = manifest as { main?: unknown; exports?: unknown; bin?: unknown }
+  const paths = new Set<string>()
+  const add = (value: unknown) => {
+    if (typeof value !== 'string' || value.startsWith('/') || value.endsWith('/') || /[*?]|^[a-z][a-z\d+.-]*:/iu.test(value)) return
+    const normalized = normalizeTeamPath(value)
+    if (!normalized.ok || !normalized.path || normalized.path === 'package.json') return
+    const path = directory ? `${directory}/${normalized.path}` : normalized.path
+    if (path.length <= 500 && paths.size < MAX_ENTRY_POINTS) paths.add(path)
+  }
+  add(pkg.main)
+  const pending = [pkg.exports]
+  for (let visited = 0; pending.length > 0 && visited < 4096; visited++) {
+    const value = pending.pop()
+    if (typeof value === 'string') { if (value.startsWith('./')) add(value); continue }
+    if (Array.isArray(value)) pending.push(...value.slice(0, 4096 - visited).toReversed())
+    else if (value && typeof value === 'object') {
+      const entries = Object.entries(value).filter(([key]) => key !== 'types' && !key.startsWith('types@'))
+      pending.push(...entries.slice(0, 4096 - visited).toReversed().map(([, target]) => target))
+    }
+  }
+  if (typeof pkg.bin === 'string') add(pkg.bin)
+  else if (pkg.bin && typeof pkg.bin === 'object' && !Array.isArray(pkg.bin)) Object.values(pkg.bin).forEach(add)
+  return [...paths]
+}
+
+// Use the immutable blob from the authorized listing, never download_url or a
+// resolved symlink target. Malformed or oversized manifests are optional.
+export async function readPackageEntryPoints(sha: string, directory: string, read: (suffix: string) => Promise<unknown>): Promise<string[]> {
+  try {
+    const blob = await read(`/git/blobs/${sha}`) as { encoding?: unknown; content?: unknown; size?: unknown }
+    if (blob?.encoding !== 'base64' || typeof blob.content !== 'string' || typeof blob.size !== 'number'
+      || blob.size < 0 || blob.size > MAX_PACKAGE_BYTES || blob.content.length > MAX_PACKAGE_BYTES * 2) return []
+    const bytes = Buffer.from(blob.content, 'base64')
+    if (bytes.length !== blob.size || bytes.length > MAX_PACKAGE_BYTES) return []
+    return packageEntryPoints(JSON.parse(bytes.toString('utf8')), directory)
+  } catch {
+    // Suggestions must not prevent browsing. The handler still rechecks
+    // GitHub and managed access after this optional upstream read.
+    return []
+  }
+}