@preventive/triage 1.0.0-alpha.5 → 1.0.0-alpha.50

package/server-managed/npm-kept.ts

@@ -0,0 +1,215 @@
+// What the npm viewer and repository risk scans keep in the database
+// (managed_upstream_cache, upstream-cache.ts), beyond the instance, so that
+// neither asks for it again on every read, nor after a restart or on another
+// instance: public versions' documents, of the fields read of them
+// (npm-manifest.ts npmVersionFields), for a day (readNpmVersion asks npm again
+// then, as a version can be deprecated, its package made private, or itself
+// removed), and when each was published, for a month (npmPublishTimeStore);
+// and, for scans, each package's risk line, for as long as a scan
+// keeps it, and what a scan read of GitHub, for a day, and of npm's
+// advisories, for an hour (repository-risk.ts); and what the code viewer read
+// of a repository's GitHub, its issues, pull requests and agents' work, for a
+// day (http.ts keptAnswer), and its history, a page from a commit at a time,
+// for a month (repository-history.ts historyFrom), as are the files each of
+// its commits changes and holds (repository-changes.ts), its trees as the
+// code viewer reads them (codeTreeStore), the packages a pull request's
+// lockfiles add (pull-dependencies.ts), and the lockfiles a commit lists and
+// each one's packages, as its dependency review reads them
+// (commit-lockfiles.ts); and the commit the code viewer keeps its
+// directories at, for as long as it does (codeHeadStore). Each store removes
+// its own rows kept longer than that as it writes, at most once an hour a
+// database (pruneKept), so that rows no read will take again don't pile up.
+// The database is an optimization: a failure reads as a miss, and writes
+// nothing.
+import { createHash } from 'node:crypto'
+import { plainObject } from './npm-manifest.ts'
+import { type NpmPublishTimeStore, type NpmVersionStore, VERSION_KEPT_MS } from './npm-packages.ts'
+import type { UpstreamCacheStore } from './upstream-cache.ts'
+
+// A value kept by key, as JSON, `at` when it was.
+export interface KeptStore<T> {
+  read(key: string): Promise<{ at: number; value: T } | null>
+  // Those of `keys` kept, by key, in one read of the database.
+  readMany(keys: string[]): Promise<Map<string, { at: number; value: T }>>
+  // Those of `first` kept; where none is, those of `then`, in one read of the
+  // database (`first` whether they are `first`'s).
+  readFirst(first: string[], then: string[]): Promise<{ first: boolean; found: Map<string, { at: number; value: T }> }>
+  write(key: string, value: T, at: number): Promise<void>
+  // Removes what is kept by `key`.
+  remove(key: string): Promise<void>
+}
+
+// When each database last had the rows under each prefix removed.
+const PRUNE_EVERY_MS = 60 * 60_000
+const pruned = new WeakMap<UpstreamCacheStore, Map<string, number>>()
+// Who sees each database's prunes through (index.ts: its app, which waits
+// for them before closing it, and on serverless keeps its invocation alive
+// till they settle), as a write no longer waits for one.
+const pruneOwners = new WeakMap<UpstreamCacheStore, (prune: Promise<unknown>) => void>()
+export function ownKeptPrunes(db: UpstreamCacheStore, own: (prune: Promise<unknown>) => void) {
+  pruneOwners.set(db, own)
+}
+
+// Removes the rows under `prefix` kept `keptMs` before `at`, at most once an
+// hour a database, as rows under it are written: here, and risk's downloads
+// (npm-risk.ts), each one run handed to the database's owner. A failure is
+// `warn`ed of, and removes nothing.
+export function pruneKept(db: UpstreamCacheStore, prefix: string, keptMs: number, at: number, warn: (error: unknown) => void): Promise<void> {
+  const times = pruned.get(db) ?? pruned.set(db, new Map()).get(db)!
+  if (at - (times.get(prefix) ?? -Infinity) < PRUNE_EVERY_MS) return Promise.resolve()
+  times.set(prefix, at)
+  const prune = (async () => { try { await db.deleteUpstreamCacheEntries(prefix, at - keptMs) } catch (error) { warn(error) } })()
+  pruneOwners.get(db)?.(prune)
+  return prune
+}
+
+// Values of `type`, each read for `keptMs` after it was kept.
+function keptStore<T>(db: UpstreamCacheStore, type: string, keptMs: number, debug: boolean): KeptStore<T> {
+  const warn = (what: string, error: unknown) => { if (debug) console.warn(`managed: kept ${type} ${what} failed:`, error) }
+  // Kept for good (Infinity): none removed.
+  const prune = (at: number) => keptMs === Infinity ? Promise.resolve() : pruneKept(db, `${type}/`, keptMs, at, error => { warn('prune', error) })
+  // Keys of any length or character, as the database's: hashed.
+  const keyOf = (key: string) => `${type}/${createHash('sha256').update(key).digest('hex')}`
+  const valueOf = (row: string | null | undefined) => {
+    const kept: unknown = JSON.parse(row ?? 'null')
+    return plainObject(kept) && typeof kept['at'] === 'number' && 'value' in kept ? { at: kept['at'], value: kept['value'] as T } : null
+  }
+  return {
+    async read(key) {
+      try { return valueOf(await db.getUpstreamCacheEntry(keyOf(key))) } catch (error) { warn('read', error); return null }
+    },
+    async readMany(keys) {
+      try {
+        const rows = await db.getUpstreamCacheEntries(keys.map(keyOf))
+        return new Map(keys.flatMap(key => { const kept = valueOf(rows.get(keyOf(key))); return kept ? [[key, kept] as const] : [] }))
+      } catch (error) { warn('read', error); return new Map() }
+    },
+    async readFirst(first, then) {
+      try {
+        const { first: answered, entries } = await db.getFirstUpstreamCacheEntries(first.map(keyOf), then.map(keyOf))
+        const asked = answered ? first : then
+        return { first: answered, found: new Map(asked.flatMap(key => { const kept = valueOf(entries.get(keyOf(key))); return kept ? [[key, kept] as const] : [] })) }
+      } catch (error) { warn('read', error); return { first: false, found: new Map() } }
+    },
+    async write(key, value, at) {
+      try { await db.setUpstreamCacheEntry(keyOf(key), JSON.stringify({ at, value }), at) } catch (error) { warn('write', error) }
+      // Behind the write, not before it is done: the prune reads the whole
+      // table (no index serves its prefix), which no writer need wait on.
+      // Its hour is marked before it runs, so writes meanwhile start none,
+      // and its database's owner sees it through (ownKeptPrunes).
+      void prune(at)
+    },
+    async remove(key) {
+      try { await db.deleteUpstreamCacheEntries(keyOf(key), Number.MAX_SAFE_INTEGER) } catch (error) { warn('remove', error) }
+    },
+  }
+}
+
+export function npmVersionStore(db: UpstreamCacheStore, debug = false): NpmVersionStore {
+  const kept = keptStore<Record<string, unknown>>(db, 'npm/version-documents', VERSION_KEPT_MS, debug)
+  return {
+    async read(name, version) {
+      const hit = await kept.read(`${name}@${version}`)
+      return hit && plainObject(hit.value) ? { at: hit.at, json: hit.value } : null
+    },
+    async readMany(versions) {
+      const hits = await kept.readMany(versions.map(({ name, version }) => `${name}@${version}`))
+      return new Map([...hits].flatMap(([key, hit]) => plainObject(hit.value) ? [[key, { at: hit.at, json: hit.value }] as const] : []))
+    },
+    write: (name, version, json) => kept.write(`${name}@${version}`, json, Date.now()),
+  }
+}
+
+// When public versions were published (npm-packages.ts npmPublishTime), which
+// never changes once they are: kept a month, and kept again by the next view
+// of the version past it.
+const PUBLISH_TIME_KEPT_MS = 30 * 24 * 60 * 60_000
+export function npmPublishTimeStore(db: UpstreamCacheStore, debug = false): NpmPublishTimeStore {
+  const kept = keptStore<string>(db, 'npm/publish-times', PUBLISH_TIME_KEPT_MS, debug)
+  return {
+    async read(name, version) {
+      const hit = await kept.read(`${name}@${version}`)
+      return typeof hit?.value === 'string' ? hit.value : null
+    },
+    write: (name, version, at) => kept.write(`${name}@${version}`, at, Date.now()),
+  }
+}
+
+// Risk lines, each kept `keptMs`, for good where Infinity, a scan telling one
+// past RISK_KEPT_MS as stale (repository-risk.ts).
+export const riskLineStore = <T>(db: UpstreamCacheStore, keptMs: number, debug = false): KeptStore<T> => keptStore<T>(db, 'repository-risk/lines', keptMs, debug)
+// What a scan read, each read for keptMs (repository-risk.ts): of GitHub, a
+// repository's lockfiles as listed at a commit (`listings`) and a lockfile's
+// packages by its blob (`lockfiles`), for SCAN_READ_KEPT_MS; of npm, a
+// lockfile's advisories by its blob (`advisories`), for RISK_KEPT_MS; the
+// repositories a scan in parts told whole (`progress`); a repository's
+// Actions' files by the commit read (`action-files`), and the actions and
+// reusable workflows of others they name, each resolved (`actions`), of
+// public repositories alone (repository-actions.ts).
+export const scanReadStore = <T>(db: UpstreamCacheStore, type: 'listings' | 'lockfiles' | 'advisories' | 'progress' | 'action-files' | 'actions', keptMs: number, debug = false): KeptStore<T> =>
+  keptStore<T>(db, `repository-risk/${type}`, keptMs, debug)
+// What the code viewer read of a repository (http.ts readWholeRepository), by
+// its id and what was asked, each kept REPOSITORY_READ_KEPT_MS.
+export const REPOSITORY_READ_KEPT_MS = 24 * 60 * 60_000
+export const repositoryReadStore = (db: UpstreamCacheStore, debug = false): KeptStore<object> =>
+  keptStore<object>(db, 'repository/reads', REPOSITORY_READ_KEPT_MS, debug)
+// Pages of a repository's history (repository-history.ts historyFrom), by its
+// id and the commit each starts at, which never change: kept a month after
+// they were last read from GitHub.
+export const HISTORY_PAGE_KEPT_MS = 30 * 24 * 60 * 60_000
+export const historyPageStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/history', HISTORY_PAGE_KEPT_MS, debug)
+// The pull requests squashed into a repository's commits, by the commit
+// (repository-squashes.ts squashedPulls): kept as its history's pages are.
+export const historySquashStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/squashes', HISTORY_PAGE_KEPT_MS, debug)
+// Whether a pull request brought a commit, by its repository and commit, as
+// the Agents tab looks it up of those an agent marked (repository-agents.ts
+// pulledCommits), kept as the history is.
+export const commitPullStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/commit-pulls', HISTORY_PAGE_KEPT_MS, debug)
+// A repository's pull requests, the most recently updated first, as the
+// Pulse and the Agents tab read them (repository-pulls.ts readPulls), by its
+// id: kept as its history's pages are, those updated since read on.
+export const pullListStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/pulls', HISTORY_PAGE_KEPT_MS, debug)
+// The files each of a repository's commits changes (repository-changes.ts
+// repositoryChanges), by the commit, which never change: kept as its
+// history's pages are.
+export const historyChangeStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/changes', HISTORY_PAGE_KEPT_MS, debug)
+// The files a repository's commit holds (repository-changes.ts
+// repositoryTree), by the commit, which never change: kept as its history's
+// pages are.
+export const historyTreeStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/trees', HISTORY_PAGE_KEPT_MS, debug)
+// What the code viewer read of a repository's trees (http.ts
+// repositoryTreesOf), which never changes once a commit or tree names it: the
+// tree at a directory of a commit, by the repository's id, the commit and the
+// directory (`trees`), and how many files a tree holds and their bytes, by
+// the repository's id and the tree's (`tree-files`): kept as its history's
+// pages are.
+export const codeTreeStore = <T>(db: UpstreamCacheStore, type: 'trees' | 'tree-files', debug = false): KeptStore<T> =>
+  keptStore<T>(db, `repository/code-${type}`, HISTORY_PAGE_KEPT_MS, debug)
+// The commit the code viewer keeps a repository's directories at in byte
+// storage, its default branch's head as last read, and the repository's
+// name they are kept under (repository-code.ts repositoryCodeKept), by the
+// repository's id: kept until the head moves, the repository is renamed or
+// it is removed, one for each repository.
+export const codeHeadStore = (db: UpstreamCacheStore, debug = false): KeptStore<{ commit: string; repo: string }> =>
+  keptStore<{ commit: string; repo: string }>(db, 'repository/code-heads', Infinity, debug)
+// The packages a repository's pull request's lockfiles add
+// (pull-dependencies.ts), by the commits of its base and head, which never
+// change: kept as its history's pages are.
+export const pullDependencyStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/pull-dependencies', HISTORY_PAGE_KEPT_MS, debug)
+// The versions a repository's lockfiles locked at the commit its dependencies
+// are reviewed since (pull-dependency-reviews.ts reviewBaseline), by the
+// commit, which never change: kept as its history's pages are, never stored
+// as reviews.
+// The lines a version changes against another, or none (npm-delta.ts), by
+// both sides' integrities, each kept HISTORY_PAGE_KEPT_MS.
+export const npmDeltaStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'npm/deltas', HISTORY_PAGE_KEPT_MS, debug)
+export const reviewBaselineStore = (db: UpstreamCacheStore, debug = false): KeptStore<{ versions: string[]; unread: string[] }> =>
+  keptStore<{ versions: string[]; unread: string[] }>(db, 'repository/review-baselines', HISTORY_PAGE_KEPT_MS, debug)
+// What a repository's dependency review read of GitHub (pull-dependencies.ts
+// LockfileStore): the lockfiles a commit lists (`listings`), by the commit,
+// and a lockfile's packages with what their reviews are held to
+// (`lockfiles`), by its blob, which never change: kept as its history's
+// pages are, apart from a scan's (scanReadStore), which reads them without
+// what their reviews are held to.
+export const lockfileReadStore = <T>(db: UpstreamCacheStore, type: 'listings' | 'lockfiles', debug = false): KeptStore<T> =>
+  keptStore<T>(db, `repository/review-${type}`, HISTORY_PAGE_KEPT_MS, debug)

package/server-managed/npm-loads.ts

@@ -0,0 +1,264 @@
+// Loading a published version for the npm package viewer (npm-packages.ts):
+// its tarball, kept on disk as bundle builds keep theirs, else read from the
+// registry, its files unpacked from it, and the viewer's response built from
+// them, each shared by the readers asking for it meanwhile and held to a few
+// versions at once per process; its files kept a little while after.
+import { Buffer } from 'node:buffer'
+import { homedir } from 'node:os'
+import { join, resolve } from 'node:path'
+import { CompressionError, decompress } from '@preventive/archive/compression.js'
+import { encodeBrotli } from './brotli.ts'
+import { keptBriefly } from './kept-briefly.ts'
+import { MAX_NPM_PACKAGE_BYTES, MAX_NPM_PACKAGE_FILES, NPM_REGISTRY, NpmPackageError, type NpmPackageFile, type NpmVersionDocument, type TarReading,
+  askUpstream, npmFileRows, npmPublishTime, npmToken, readNpmTar } from './npm-packages.ts'
+import { keepFile, readKeptFile, upstreamCachePath } from './upstream-cache.ts'
+
+// The tar stream a package's files may come in, headers and padding included.
+// A tarball larger than that cannot unpack within it, so it is refused as it
+// arrives.
+export const MAX_TAR_BYTES = 96 * 1024 * 1024
+const TARBALL_TIMEOUT_MS = 120_000
+// Loads in flight per process, those of one version (its tarball, its files,
+// its response) as one, as they read its tarball and unpack it once between
+// them; each can hold a tarball, its tar, its files and their encoded JSON.
+const MAX_ACTIVE_LOADS = 4
+// How long a public version's files are kept once unpacked, and what they
+// may take between them, the least lately read going first past it:
+// Compare's requests for a pair of versions, and the Code tab's, come one
+// after another, each loading the same files. A private version's are never
+// kept, as nothing else of it is past its readers.
+const UNPACKED_KEPT_MS = 60_000
+const MAX_UNPACKED_BYTES = 128 * 1024 * 1024
+
+// A load's result, shared by its readers, and a reader's release of it.
+export interface NpmLoad<T> { result: Promise<T>; release: () => void }
+
+interface Load { job: Promise<unknown>; readers: number; settled: boolean; slot: string }
+
+// Loads by what they read; readers asking for the same share one. A load
+// keeps its place among the active ones until its work is done and each
+// reader has released it, as a reader does once its response is written or
+// abandoned, so what responses still hold counts too. Places by the slot
+// their loads share, with how many hold each and how many places it takes.
+const loads = new Map<string, Load>()
+const slots = new Map<string, { holding: number; places: number }>()
+const placesTaken = () => [...slots.values()].reduce((sum, slot) => sum + slot.places, 0)
+
+const refused = <T>(status: number, code: string): NpmLoad<T> => {
+  const result = Promise.reject(new NpmPackageError(status, code))
+  result.catch(() => {})
+  return { result, release() {} }
+}
+
+// What names a version's tarball, and so its files: its integrity, its
+// shasum and URL, and whether it is read with the token.
+const tarballKey = (doc: NpmVersionDocument) => `${doc.dist.integrity} ${doc.dist.shasum ?? ''} ${doc.dist.tarball} ${doc.private}`
+
+function shared<T>(what: string, doc: NpmVersionDocument, work: () => Promise<T>): NpmLoad<T> {
+  const { unpackedSize, fileCount } = doc.dist
+  if ((unpackedSize ?? 0) > MAX_NPM_PACKAGE_BYTES || (fileCount ?? 0) > MAX_NPM_PACKAGE_FILES) return refused(413, 'package-too-large')
+  const slot = tarballKey(doc)
+  return sharedLoad(`${what} ${slot}`, work, 'npm-busy', { slot })
+}
+
+// The places among the active ones a load takes whose files hold `files` and
+// `bytes`: as many as packages they would fill, up to every one, each place
+// holding what a package may.
+export function loadPlaces({ files, bytes }: { files: number; bytes: number }): number {
+  return Math.min(MAX_ACTIVE_LOADS, Math.max(1, Math.ceil(files / MAX_NPM_PACKAGE_FILES), Math.ceil(bytes / MAX_NPM_PACKAGE_BYTES)))
+}
+
+// A load among the active ones, by `key`, refused as `busy` past them: a
+// repository directory's (repository-code.ts) as a package's, each holding
+// alike a tarball, its tar, its files and their encoded JSON. Loads of one
+// `slot` take one place between them, or `places` (loadPlaces) where they
+// hold more.
+export function sharedLoad<T>(key: string, work: () => Promise<T>, busy: string,
+  { slot = key, places = 1 }: { slot?: string; places?: number } = {}): NpmLoad<T> {
+  let load = loads.get(key)
+  if (!load) {
+    if (!slots.has(slot) && placesTaken() + places > MAX_ACTIVE_LOADS) return refused(429, busy)
+    const created: Load = { job: work(), readers: 0, settled: false, slot }
+    const finish = () => { settle(key, created) }
+    created.job.then(finish, finish)
+    loads.set(key, created)
+    const taken = slots.get(slot)
+    slots.set(slot, { holding: (taken?.holding ?? 0) + 1, places: taken?.places ?? places })
+    load = created
+  }
+  const held = load
+  held.readers++
+  let released = false
+  return {
+    result: held.job as Promise<T>,
+    release() {
+      if (released) return
+      released = true
+      held.readers--
+      done(key, held)
+    },
+  }
+}
+
+function settle(key: string, load: Load) {
+  load.settled = true
+  done(key, load)
+}
+
+// A load gone from the active ones, and its place with it, once its work is
+// done and its readers have released it.
+function done(key: string, load: Load) {
+  if (load.readers !== 0 || !load.settled || loads.get(key) !== load) return
+  loads.delete(key)
+  const taken = slots.get(load.slot)!
+  if (taken.holding === 1) slots.delete(load.slot)
+  else taken.holding--
+}
+
+// The sha512 an integrity names, in base64, or null where it names none.
+const sha512Of = (integrity: string) => /(?:^|\s)sha512-([\d+/A-Za-z]{86}==)(?=\s|$)/u.exec(integrity)?.[1] ?? null
+// Hashed off the event loop: a tarball runs to tens of MiB.
+const digest = async (algorithm: string, bytes: Uint8Array) => Buffer.from(await crypto.subtle.digest(algorithm, bytes as Uint8Array<ArrayBuffer>))
+// Whether `bytes` are those of the sha512 an integrity names and, where there
+// is one, of the shasum (a sha1, in hex).
+async function hashedAs(bytes: Uint8Array, sha512: string, shasum: string | null): Promise<boolean> {
+  const [own512, own1] = await Promise.all([digest('SHA-512', bytes), shasum === null ? null : digest('SHA-1', bytes)])
+  return own512.toString('base64') === sha512 && (own1 === null || own1.toString('hex') === shasum)
+}
+
+// Upstream's disk cache (setCacheDir), where tarballs are kept between
+// loads, or null to keep none and read none.
+let tarballCache: string | null = null
+
+export function setNpmTarballCache(dir: string | null) {
+  tarballCache = dir === null ? null : resolve(dir)
+}
+
+// Where npm's own cache files content by its sha512, short of an .npmrc
+// moving the cache.
+function npmCachePath(hash: string): string | null {
+  const configured = process.env['npm_config_cache'] || process.env['NPM_CONFIG_CACHE']
+  let root
+  try {
+    root = configured ? resolve(configured.replace(/^~(?=[/\\])/u, homedir()))
+      : process.platform === 'win32' ? join(process.env['LOCALAPPDATA'] || homedir(), 'npm-cache') : join(homedir(), '.npm')
+  } catch { return null }
+  const hex = Buffer.from(hash, 'base64').toString('hex')
+  return join(root, '_cacache', 'content-v2', 'sha512', hex.slice(0, 2), hex.slice(2, 4), hex.slice(4))
+}
+
+// The version's tarball from the registry, at the package's own path: with
+// the token only for a private version, and its size bounded as it arrives.
+async function downloadTarball(url: string, isPrivate: boolean): Promise<Buffer> {
+  const token = isPrivate ? npmToken() : null
+  const bytes = await askUpstream(url, { headers: token ? { authorization: `Bearer ${token}` } : {}, limit: MAX_TAR_BYTES, ms: TARBALL_TIMEOUT_MS, absent: [401, 403, 404] })
+    .catch((err: unknown) => { throw err instanceof NpmPackageError && err.message === 'upstream-too-large' ? new NpmPackageError(413, 'package-too-large') : err })
+  if (bytes === null) throw new NpmPackageError(404, 'package-not-found')
+  return bytes
+}
+
+// The version's tarball, its bytes held to the document's sha512, and to its
+// shasum where it tells one, wherever they come from: kept in the cache,
+// where bundle builds keep what they fetch with the token, or in npm's; else
+// downloaded, and then kept. Bytes matching the hashes the reader's own
+// document names are the tarball that reader may download, wherever they
+// were kept and whoever fetched them.
+async function readTarball(doc: NpmVersionDocument): Promise<Uint8Array> {
+  const expected = sha512Of(doc.dist.integrity), shasum = doc.dist.shasum ?? null, url = doc.dist.tarball
+  if (!url.startsWith(`${NPM_REGISTRY}/${doc.name}/-/`) || URL.parse(url)?.href !== url || expected === null
+    || (shasum !== null && !/^[\da-f]{40}$/u.test(shasum))) throw new NpmPackageError(502, 'upstream-invalid')
+  const kept = tarballCache === null ? null : upstreamCachePath(tarballCache, 'npm/tarballs', `${doc.name}@${doc.version}.tgz`)
+  if (kept !== null) {
+    for (const path of [kept, npmCachePath(expected)]) {
+      const bytes = path === null ? null : await readKeptFile(path, MAX_TAR_BYTES)
+      if (bytes && await hashedAs(bytes, expected, shasum)) return bytes
+    }
+  }
+  const bytes = await downloadTarball(url, doc.private)
+  if (!await hashedAs(bytes, expected, shasum)) throw new NpmPackageError(502, 'upstream-invalid')
+  if (kept !== null) await keepFile(kept, bytes)
+  return bytes
+}
+
+// Tarballs being read, and versions' files being unpacked, by what names
+// them (tarballKey): readers asking for one meanwhile share it, its bytes
+// those each one's own document names.
+const reading = new Map<string, Promise<Uint8Array>>()
+const unpacking = new Map<string, Promise<Unpacked>>()
+
+function sharedRead<T>(pending: Map<string, Promise<T>>, key: string, read: () => Promise<T>): Promise<T> {
+  let job = pending.get(key)
+  if (!job) {
+    const started = read().finally(() => { if (pending.get(key) === started) pending.delete(key) })
+    pending.set(key, job = started)
+  }
+  return job
+}
+
+const sharedTarball = (doc: NpmVersionDocument) => sharedRead(reading, tarballKey(doc), () => readTarball(doc))
+
+// A version's files, and the size of the tarball they came in.
+interface Unpacked { files: NpmPackageFile[]; tarballSize: number }
+
+// Public versions' files lately unpacked, by tarballKey.
+const unpacked = keptBriefly<Unpacked>({ keptMs: UNPACKED_KEPT_MS, maxBytes: MAX_UNPACKED_BYTES })
+
+// The version's files, kept from a load of them a little while since, else
+// unpacked from its tarball (and kept, a public version's): bytes its
+// reader's own document names, as its tarball's are.
+function unpackedOf(doc: NpmVersionDocument): Promise<Unpacked> {
+  const key = tarballKey(doc)
+  const kept = unpacked.get(key)
+  if (kept) return Promise.resolve(kept)
+  return sharedRead(unpacking, key, async () => {
+    const tarball = await sharedTarball(doc)
+    const value = { files: await readTarballFiles(tarball), tarballSize: tarball.byteLength }
+    if (!doc.private) unpacked.keep(key, value, value.files.reduce((sum, file) => sum + file.path.length + file.bytes.byteLength, 0))
+    return value
+  })
+}
+
+// A gzipped tarball's files, its top directory left out, as npm extracts
+// a package's: a repository tree's as GitHub archives it, too, its symlinks
+// let climb as readNpmTar's `symlinksAbove` says, and its tar stream and
+// files bounded as `options` give, a package's bounds unless given.
+export async function readTarballFiles(tarball: Uint8Array, { maxTarBytes = MAX_TAR_BYTES, ...options }: TarReading & { maxTarBytes?: number } = {}): Promise<NpmPackageFile[]> {
+  let tar: Uint8Array
+  try { tar = await decompress(tarball, 'gzip', { limit: maxTarBytes }) }
+  catch (err) {
+    const limited = err instanceof CompressionError && err.limited
+    throw new NpmPackageError(limited ? 413 : 422, limited ? 'package-too-large' : 'bad-tarball')
+  }
+  return readNpmTar(tar, options)
+}
+
+// The version's tarball, for download.
+export function loadNpmTarball(doc: NpmVersionDocument): NpmLoad<Uint8Array> {
+  return shared('tarball', doc, () => sharedTarball(doc))
+}
+
+// The version's files, for one of them to be pretty-printed (pretty-print.ts).
+export function loadNpmFiles(doc: NpmVersionDocument): NpmLoad<NpmPackageFile[]> {
+  return shared('files', doc, async () => (await unpackedOf(doc)).files)
+}
+
+// The version as the viewer reads it, as brotli-encoded JSON: `{ name,
+// version, private, integrity, tarballSize, published, manifest, files,
+// known }`, its files as npmFileRows gives them, and with them, as its files
+// are encoded, when it was published where that is kept (`published`, asked
+// then, else null) and what `known` gives: what is kept about its package
+// (npm-insights.ts npmKnown), which the viewer need not then ask for. Encoding
+// is where a load holds the most, so it takes its place among the active
+// loads until its body is done, and readers asking meanwhile share that body,
+// the first one's manifest, `published` and `known` in it; the body is not
+// kept once its readers are done, its files a little while (unpackedOf).
+export function loadNpmPackageBody(doc: NpmVersionDocument, known: () => object = () => ({}),
+  published: () => Promise<string | null> = npmPublishTime(doc).read): NpmLoad<Buffer> {
+  return shared('package', doc, async () => {
+    const { files, tarballSize } = await unpackedOf(doc)
+    return encodeBrotli(Buffer.from(JSON.stringify({
+      name: doc.name, version: doc.version, private: doc.private, integrity: doc.dist.integrity, tarballSize,
+      published: await published(), manifest: doc.manifest, files: npmFileRows(files), known: known(),
+    })))
+  })
+}

package/server-managed/npm-manifest.ts

@@ -0,0 +1,115 @@
+// What a published version's document tells of it as the npm viewer's
+// Overview shows it (npmManifest), and the fields of the document read for it
+// and for the version itself (npm-packages.ts readNpmVersion), the rest
+// stripped where it is kept (npmVersionFields).
+import publishers from '@preventive/publishers/npm.js'
+import { getRepo } from '@preventive/upstream/package.js'
+
+export const plainObject = (value: unknown): value is Record<string, unknown> => Boolean(value) && typeof value === 'object' && !Array.isArray(value)
+export const stringRecord = (value: unknown) => plainObject(value)
+  ? Object.fromEntries(Object.entries(value).filter(([, item]) => typeof item === 'string')) as Record<string, string> : undefined
+
+interface NpmPerson { name: string | null; email: string | null }
+const NPM_ACCOUNT = /^[\w.-]{1,214}$/u
+// The npm accounts known to publish: those with standing on one of npm's
+// most downloaded packages (@preventive/publishers).
+const KNOWN_PUBLISHERS = new Set(publishers.map(name => name.toLowerCase()))
+
+// One of package.json's people: `{ name, email, url }`, or `Name <email>
+// (url)` in one string.
+function npmPerson(value: unknown): NpmPerson | null {
+  const text = (part: unknown) => typeof part === 'string' && part.trim() !== '' ? part.trim() : null
+  if (plainObject(value)) return { name: text(value['name']), email: text(value['email']) }
+  if (typeof value !== 'string') return null
+  const parts = /^([^<(]*)(?:<([^>]*)>)?\s*(?:\([^)]*\))?\s*$/u.exec(value)
+  return parts ? { name: text(parts[1]), email: text(parts[2]) } : { name: text(value), email: null }
+}
+
+// The GitLab project (`group/project`, subgroups and all) a package names as
+// its repository, in any of the forms npm takes (`gitlab:group/project`,
+// `git@gitlab.com:…`, `git+https://gitlab.com/….git`); null for another.
+// GitLab's are only on gitlab.com, as npm's provenance has them.
+function gitlabProject(repository: unknown): string | null {
+  const url = typeof repository === 'string' ? repository : plainObject(repository) && typeof repository['url'] === 'string' ? repository['url'] : ''
+  if (url.length > 2048) return null
+  const project = /^(?:gitlab:|git@gitlab\.com:|(?:git\+)?(?:https|ssh|git):\/\/(?:git@)?gitlab\.com\/)([\w.-]+(?:\/[\w.-]+)+)\/?$/u
+    .exec(url.split('/-/')[0]!)?.[1]?.replace(/\.git$/u, '')
+  return project && !project.split('/').some(part => /^\.+$/u.test(part)) ? project : null
+}
+
+// The fields the Overview shows, in the shapes package.json gives them; each
+// left out where the document has none, or has another shape. A homepage that
+// leads only where its repository does (the repository, its directory there,
+// or its readme, npm's homepage where the package names none) is left out.
+export function npmManifest(json: Record<string, unknown>): Record<string, unknown> {
+  const pick: Record<string, unknown> = {}
+  for (const key of ['description', 'license', 'homepage', 'main', 'module', 'types', 'type', 'deprecated', 'gitHead']) {
+    if (typeof json[key] === 'string') pick[key] = json[key]
+  }
+  // Its author's name, and their npm account where it can be told: the
+  // publisher's or a maintainer's whose email or name is the author's. Emails
+  // only match people up; none is kept.
+  const author = npmPerson(json['author'])
+  if (author?.name) pick['author'] = author.name
+  const accounts = [json['_npmUser'], ...Array.isArray(json['maintainers']) ? json['maintainers'] : []].map(npmPerson)
+    .filter((person): person is NpmPerson & { name: string } => person?.name != null && NPM_ACCOUNT.test(person.name))
+  // The npm account that published it, for its profile; or, published by
+  // a CI npm trusts (npm's trusted publishing), the CI, as npm names it
+  // (`GitHub Actions`).
+  const publisher = npmPerson(json['_npmUser'])?.name
+  if (publisher && NPM_ACCOUNT.test(publisher)) pick['publisher'] = publisher
+  else if (publisher && publisher.length <= 64 && plainObject(json['_npmUser']) && plainObject(json['_npmUser']['trustedPublisher'])) pick['trustedPublisher'] = publisher
+  const same = (a: string | null, b: string | null) => a !== null && b !== null && a.toLowerCase() === b.toLowerCase()
+  const account = author && accounts.find(person => same(person.email, author.email) || same(person.name, author.name))
+  if (account) pick['authorAccount'] = account.name
+  // Which of those accounts are known to publish, for the Overview to tell
+  // them from the rest.
+  const known = [...new Set([publisher, account?.name])].filter(name => name && KNOWN_PUBLISHERS.has(name.toLowerCase()))
+  if (known.length > 0) pick['knownAccounts'] = known
+  if (Array.isArray(json['keywords'])) pick['keywords'] = json['keywords'].filter(item => typeof item === 'string').slice(0, 50)
+  if (typeof json['bin'] === 'string') pick['bin'] = { [String(json['name'])]: json['bin'] }
+  else if (stringRecord(json['bin'])) pick['bin'] = stringRecord(json['bin'])
+  for (const key of ['dependencies', 'peerDependencies', 'optionalDependencies', 'engines']) {
+    const record = stringRecord(json[key])
+    if (record && Object.keys(record).length > 0) pick[key] = record
+  }
+  if (json['hasInstallScript'] === true) pick['hasInstallScript'] = true
+  // Whether npm keeps a provenance statement for it (npm-provenance.ts).
+  if (plainObject(json['dist']) && plainObject(json['dist']['attestations']) && plainObject(json['dist']['attestations']['provenance'])) pick['provenance'] = true
+  // The scripts npm runs installing it from the registry; not `prepare`,
+  // which runs only for a local or git install.
+  const scripts = stringRecord(json['scripts'])
+  const install = scripts ? ['preinstall', 'install', 'postinstall'].filter(script => script in scripts) : []
+  if (install.length > 0) pick['installScripts'] = Object.fromEntries(install.map(script => [script, scripts![script]]))
+  // A GitLab project it names, as its provenance's source is matched with
+  // the repository it names, GitHub's below.
+  const gitlab = gitlabProject(json['repository'])
+  if (gitlab) pick['gitlab'] = gitlab
+  try {
+    const repo = getRepo(json)
+    if (repo.github) {
+      pick['github'] = { github: repo.github, ...(repo.directory === undefined ? {} : { directory: repo.directory }) }
+      const home = typeof pick['homepage'] === 'string' ? getRepo({ homepage: pick['homepage'] }) : {}
+      if (home.github?.toLowerCase() === repo.github.toLowerCase() && (home.directory ?? '') === (repo.directory ?? '')) delete pick['homepage']
+    }
+  } catch {}
+  return pick
+}
+
+// Of a version's document, the fields readNpmVersion and npmManifest read,
+// of `dist` and `scripts` only theirs, and of a provenance statement only that
+// there is one: what is kept of it (npm-kept.ts), so that both read a kept one
+// as they read npm's.
+const VERSION_FIELDS = ['name', 'version', 'description', 'license', 'homepage', 'main', 'module', 'types', 'type', 'deprecated', 'gitHead', 'author', '_npmUser',
+  'maintainers', 'keywords', 'bin', 'dependencies', 'peerDependencies', 'optionalDependencies', 'engines', 'hasInstallScript', 'repository', 'bugs']
+const DIST_FIELDS = ['tarball', 'integrity', 'shasum', 'unpackedSize', 'fileCount']
+const INSTALL_SCRIPTS = ['preinstall', 'install', 'postinstall']
+const picked = (json: Record<string, unknown>, keys: string[]) => Object.fromEntries(keys.filter(key => Object.hasOwn(json, key)).map(key => [key, json[key]]))
+
+export function npmVersionFields(json: Record<string, unknown>): Record<string, unknown> {
+  const dist = plainObject(json['dist']) ? json['dist'] : {}
+  const provenance = plainObject(dist['attestations']) && plainObject(dist['attestations']['provenance'])
+  const scripts = picked(stringRecord(json['scripts']) ?? {}, INSTALL_SCRIPTS)
+  return { ...picked(json, VERSION_FIELDS), dist: { ...picked(dist, DIST_FIELDS), ...provenance && { attestations: { provenance: {} } } },
+    ...Object.keys(scripts).length > 0 && { scripts } }
+}