@preventive/triage 1.0.0-alpha.44 → 1.0.0-alpha.46

package/server-managed/npm-kept.ts

@@ -4,16 +4,22 @@
 // instance: public versions' documents, of the fields read of them
 // (npm-manifest.ts npmVersionFields), for a day (readNpmVersion asks npm again
 // then, as a version can be deprecated, its package made private, or itself
-// removed); and, for scans, each package's risk line, for as long as a scan
+// removed), and when each was published, for a month (npmPublishTimeStore);
+// and, for scans, each package's risk line, for as long as a scan
 // keeps it, and what a scan read of GitHub, for a day, and of npm's
-// advisories, for an hour (repository-risk.ts). Each store removes its own rows kept longer
+// advisories, for an hour (repository-risk.ts); and what the code viewer read
+// of a repository's GitHub, its issues, pull requests and agents' work, for a
+// day (http.ts keptAnswer), and its history, a page from a commit at a time,
+// for a month (repository-history.ts historyFrom), as are the files each of
+// its commits changes and holds (repository-changes.ts), and the packages a
+// pull request's lockfiles add (pull-dependencies.ts). Each store removes its own rows kept longer
 // than that as it writes, at most once an hour a database (pruneKept), so
 // that rows no read will take again don't pile up.
 // The database is an optimization: a failure reads as a miss, and writes
 // nothing.
 import { createHash } from 'node:crypto'
 import { plainObject } from './npm-manifest.ts'
-import { type NpmVersionStore, VERSION_KEPT_MS } from './npm-packages.ts'
+import { type NpmPublishTimeStore, type NpmVersionStore, VERSION_KEPT_MS } from './npm-packages.ts'
 import type { UpstreamCacheStore } from './upstream-cache.ts'
 
 // A value kept by key, as JSON, `at` when it was.
@@ -21,7 +27,12 @@ export interface KeptStore<T> {
   read(key: string): Promise<{ at: number; value: T } | null>
   // Those of `keys` kept, by key, in one read of the database.
   readMany(keys: string[]): Promise<Map<string, { at: number; value: T }>>
+  // Those of `first` kept; where none is, those of `then`, in one read of the
+  // database (`first` whether they are `first`'s).
+  readFirst(first: string[], then: string[]): Promise<{ first: boolean; found: Map<string, { at: number; value: T }> }>
   write(key: string, value: T, at: number): Promise<void>
+  // Removes what is kept by `key`.
+  remove(key: string): Promise<void>
 }
 
 // When each database last had the rows under each prefix removed.
@@ -59,10 +70,20 @@ function keptStore<T>(db: UpstreamCacheStore, type: string, keptMs: number, debu
         return new Map(keys.flatMap(key => { const kept = valueOf(rows.get(keyOf(key))); return kept ? [[key, kept] as const] : [] }))
       } catch (error) { warn('read', error); return new Map() }
     },
+    async readFirst(first, then) {
+      try {
+        const { first: answered, entries } = await db.getFirstUpstreamCacheEntries(first.map(keyOf), then.map(keyOf))
+        const asked = answered ? first : then
+        return { first: answered, found: new Map(asked.flatMap(key => { const kept = valueOf(entries.get(keyOf(key))); return kept ? [[key, kept] as const] : [] })) }
+      } catch (error) { warn('read', error); return { first: false, found: new Map() } }
+    },
     async write(key, value, at) {
       try { await db.setUpstreamCacheEntry(keyOf(key), JSON.stringify({ at, value }), at) } catch (error) { warn('write', error) }
       await prune(at)
     },
+    async remove(key) {
+      try { await db.deleteUpstreamCacheEntries(keyOf(key), Number.MAX_SAFE_INTEGER) } catch (error) { warn('remove', error) }
+    },
   }
 }
 
@@ -73,17 +94,73 @@ export function npmVersionStore(db: UpstreamCacheStore, debug = false): NpmVersi
       const hit = await kept.read(`${name}@${version}`)
       return hit && plainObject(hit.value) ? { at: hit.at, json: hit.value } : null
     },
+    async readMany(versions) {
+      const hits = await kept.readMany(versions.map(({ name, version }) => `${name}@${version}`))
+      return new Map([...hits].flatMap(([key, hit]) => plainObject(hit.value) ? [[key, { at: hit.at, json: hit.value }] as const] : []))
+    },
     write: (name, version, json) => kept.write(`${name}@${version}`, json, Date.now()),
   }
 }
 
+// When public versions were published (npm-packages.ts npmPublishTime), which
+// never changes once they are: kept a month, and kept again by the next view
+// of the version past it.
+const PUBLISH_TIME_KEPT_MS = 30 * 24 * 60 * 60_000
+export function npmPublishTimeStore(db: UpstreamCacheStore, debug = false): NpmPublishTimeStore {
+  const kept = keptStore<string>(db, 'npm/publish-times', PUBLISH_TIME_KEPT_MS, debug)
+  return {
+    async read(name, version) {
+      const hit = await kept.read(`${name}@${version}`)
+      return typeof hit?.value === 'string' ? hit.value : null
+    },
+    write: (name, version, at) => kept.write(`${name}@${version}`, at, Date.now()),
+  }
+}
+
 // Risk lines, each kept `keptMs`, for good where Infinity, a scan telling one
 // past RISK_KEPT_MS as stale (repository-risk.ts).
 export const riskLineStore = <T>(db: UpstreamCacheStore, keptMs: number, debug = false): KeptStore<T> => keptStore<T>(db, 'repository-risk/lines', keptMs, debug)
 // What a scan read, each read for keptMs (repository-risk.ts): of GitHub, a
 // repository's lockfiles as listed at a commit (`listings`) and a lockfile's
 // packages by its blob (`lockfiles`), for SCAN_READ_KEPT_MS; of npm, a
-// lockfile's advisories by its blob (`advisories`), for RISK_KEPT_MS; and the
-// repositories a scan in parts told whole (`progress`).
-export const scanReadStore = <T>(db: UpstreamCacheStore, type: 'listings' | 'lockfiles' | 'advisories' | 'progress', keptMs: number, debug = false): KeptStore<T> =>
+// lockfile's advisories by its blob (`advisories`), for RISK_KEPT_MS; the
+// repositories a scan in parts told whole (`progress`); a repository's
+// Actions' files by the commit read (`action-files`), and the actions and
+// reusable workflows of others they name, each resolved (`actions`), of
+// public repositories alone (repository-actions.ts).
+export const scanReadStore = <T>(db: UpstreamCacheStore, type: 'listings' | 'lockfiles' | 'advisories' | 'progress' | 'action-files' | 'actions', keptMs: number, debug = false): KeptStore<T> =>
   keptStore<T>(db, `repository-risk/${type}`, keptMs, debug)
+// What the code viewer read of a repository (http.ts readWholeRepository), by
+// its id and what was asked, each kept REPOSITORY_READ_KEPT_MS.
+export const REPOSITORY_READ_KEPT_MS = 24 * 60 * 60_000
+export const repositoryReadStore = (db: UpstreamCacheStore, debug = false): KeptStore<object> =>
+  keptStore<object>(db, 'repository/reads', REPOSITORY_READ_KEPT_MS, debug)
+// Pages of a repository's history (repository-history.ts historyFrom), by its
+// id and the commit each starts at, which never change: kept a month after
+// they were last read from GitHub.
+export const HISTORY_PAGE_KEPT_MS = 30 * 24 * 60 * 60_000
+export const historyPageStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/history', HISTORY_PAGE_KEPT_MS, debug)
+// The pull requests squashed into a repository's commits, by the commit
+// (repository-history.ts squashedPulls): kept as its history's pages are.
+export const historySquashStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/squashes', HISTORY_PAGE_KEPT_MS, debug)
+// The files each of a repository's commits changes (repository-changes.ts
+// repositoryChanges), by the commit, which never change: kept as its
+// history's pages are.
+export const historyChangeStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/changes', HISTORY_PAGE_KEPT_MS, debug)
+// The files a repository's commit holds (repository-changes.ts
+// repositoryTree), by the commit, which never change: kept as its history's
+// pages are.
+export const historyTreeStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/trees', HISTORY_PAGE_KEPT_MS, debug)
+// The packages a repository's pull request's lockfiles add
+// (pull-dependencies.ts), by the commits of its base and head, which never
+// change: kept as its history's pages are.
+export const pullDependencyStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/pull-dependencies', HISTORY_PAGE_KEPT_MS, debug)
+// The versions a repository's lockfiles locked at the commit its dependencies
+// are reviewed since (pull-dependency-reviews.ts reviewBaseline), by the
+// commit, which never change: kept as its history's pages are, never stored
+// as reviews.
+// The lines a version changes against another, or none (npm-delta.ts), by
+// both sides' integrities, each kept HISTORY_PAGE_KEPT_MS.
+export const npmDeltaStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'npm/deltas', HISTORY_PAGE_KEPT_MS, debug)
+export const reviewBaselineStore = (db: UpstreamCacheStore, debug = false): KeptStore<{ versions: string[]; unread: string[] }> =>
+  keptStore<{ versions: string[]; unread: string[] }>(db, 'repository/review-baselines', HISTORY_PAGE_KEPT_MS, debug)

package/server-managed/npm-loads.ts

@@ -1,13 +1,14 @@
 // Loading a published version for the npm package viewer (npm-packages.ts):
 // its tarball, kept on disk as bundle builds keep theirs, else read from the
-// registry, and the viewer's response built from it, each shared by the
-// readers asking for it meanwhile and held to a few at once per process.
+// registry, its files unpacked from it, and the viewer's response built from
+// them, each shared by the readers asking for it meanwhile and held to a few
+// versions at once per process; its files kept a little while after.
 import { Buffer } from 'node:buffer'
 import { homedir } from 'node:os'
 import { join, resolve } from 'node:path'
-import { promisify } from 'node:util'
-import { gunzip } from 'node:zlib'
+import { CompressionError, decompress } from '@preventive/archive/compression.js'
 import { encodeBrotli } from './brotli.ts'
+import { keptBriefly } from './kept-briefly.ts'
 import { MAX_NPM_PACKAGE_BYTES, MAX_NPM_PACKAGE_FILES, NPM_REGISTRY, NpmPackageError, type NpmPackageFile, type NpmVersionDocument, askUpstream,
   npmFileRows, npmPublishTime, npmToken, readNpmTar } from './npm-packages.ts'
 import { keepFile, readKeptFile, upstreamCachePath } from './upstream-cache.ts'
@@ -17,22 +18,30 @@ import { keepFile, readKeptFile, upstreamCachePath } from './upstream-cache.ts'
 // arrives.
 export const MAX_TAR_BYTES = 96 * 1024 * 1024
 const TARBALL_TIMEOUT_MS = 120_000
-// Loads in flight per process; each can hold a tarball, its tar, its files
-// and their encoded JSON.
+// Loads in flight per process, those of one version (its tarball, its files,
+// its response) as one, as they read its tarball and unpack it once between
+// them; each can hold a tarball, its tar, its files and their encoded JSON.
 const MAX_ACTIVE_LOADS = 4
-
-const gunzipAsync = promisify(gunzip)
+// How long a public version's files are kept once unpacked, and what they
+// may take between them, the least lately read going first past it:
+// Compare's requests for a pair of versions, and the Code tab's, come one
+// after another, each loading the same files. A private version's are never
+// kept, as nothing else of it is past its readers.
+const UNPACKED_KEPT_MS = 60_000
+const MAX_UNPACKED_BYTES = 128 * 1024 * 1024
 
 // A load's result, shared by its readers, and a reader's release of it.
 export interface NpmLoad<T> { result: Promise<T>; release: () => void }
 
-interface Load { job: Promise<unknown>; readers: number; settled: boolean }
+interface Load { job: Promise<unknown>; readers: number; settled: boolean; slot: string }
 
 // Loads by what they read; readers asking for the same share one. A load
 // keeps its place among the active ones until its work is done and each
 // reader has released it, as a reader does once its response is written or
-// abandoned, so what responses still hold counts too.
+// abandoned, so what responses still hold counts too. Places by the slot
+// their loads share, with how many hold each.
 const loads = new Map<string, Load>()
+const slots = new Map<string, number>()
 
 const refused = <T>(status: number, code: string): NpmLoad<T> => {
   const result = Promise.reject(new NpmPackageError(status, code))
@@ -40,19 +49,30 @@ const refused = <T>(status: number, code: string): NpmLoad<T> => {
   return { result, release() {} }
 }
 
-function shared<T>(key: string, doc: NpmVersionDocument, work: () => Promise<T>): NpmLoad<T> {
+// What names a version's tarball, and so its files: its integrity and URL,
+// and whether it is read with the token.
+const tarballKey = (doc: NpmVersionDocument) => `${doc.dist.integrity} ${doc.dist.tarball} ${doc.private}`
+
+function shared<T>(what: string, doc: NpmVersionDocument, work: () => Promise<T>): NpmLoad<T> {
   const { unpackedSize, fileCount } = doc.dist
   if ((unpackedSize ?? 0) > MAX_NPM_PACKAGE_BYTES || (fileCount ?? 0) > MAX_NPM_PACKAGE_FILES) return refused(413, 'package-too-large')
+  const slot = tarballKey(doc)
+  return sharedLoad(`${what} ${slot}`, work, 'npm-busy', slot)
+}
+
+// A load among the active ones, by `key`, refused as `busy` past them: a
+// repository directory's (repository-code.ts) as a package's, each holding
+// alike a tarball, its tar, its files and their encoded JSON. Loads of one
+// `slot` take one place between them.
+export function sharedLoad<T>(key: string, work: () => Promise<T>, busy: string, slot = key): NpmLoad<T> {
   let load = loads.get(key)
   if (!load) {
-    if (loads.size >= MAX_ACTIVE_LOADS) return refused(429, 'npm-busy')
-    const created: Load = { job: work(), readers: 0, settled: false }
-    const settle = () => {
-      created.settled = true
-      if (created.readers === 0 && loads.get(key) === created) loads.delete(key)
-    }
-    created.job.then(settle, settle)
+    if (!slots.has(slot) && slots.size >= MAX_ACTIVE_LOADS) return refused(429, busy)
+    const created: Load = { job: work(), readers: 0, settled: false, slot }
+    const finish = () => { settle(key, created) }
+    created.job.then(finish, finish)
     loads.set(key, created)
+    slots.set(slot, (slots.get(slot) ?? 0) + 1)
     load = created
   }
   const held = load
@@ -63,11 +83,27 @@ function shared<T>(key: string, doc: NpmVersionDocument, work: () => Promise<T>)
     release() {
       if (released) return
       released = true
-      if (--held.readers === 0 && held.settled && loads.get(key) === held) loads.delete(key)
+      held.readers--
+      done(key, held)
     },
   }
 }
 
+function settle(key: string, load: Load) {
+  load.settled = true
+  done(key, load)
+}
+
+// A load gone from the active ones, and its place with it, once its work is
+// done and its readers have released it.
+function done(key: string, load: Load) {
+  if (load.readers !== 0 || !load.settled || loads.get(key) !== load) return
+  loads.delete(key)
+  const holding = slots.get(load.slot)! - 1
+  if (holding === 0) slots.delete(load.slot)
+  else slots.set(load.slot, holding)
+}
+
 // The sha512 an integrity names, in base64, or null where it names none.
 const sha512Of = (integrity: string) => /(?:^|\s)sha512-([\d+/A-Za-z]{86}==)(?=\s|$)/u.exec(integrity)?.[1] ?? null
 // Hashed off the event loop: a tarball runs to tens of MiB.
@@ -125,43 +161,84 @@ async function readTarball(doc: NpmVersionDocument): Promise<Uint8Array> {
   return bytes
 }
 
-async function readFiles(tarball: Uint8Array): Promise<NpmPackageFile[]> {
-  let tar: Buffer
-  try { tar = await gunzipAsync(tarball, { maxOutputLength: MAX_TAR_BYTES }) }
+// Tarballs being read, and versions' files being unpacked, by what names
+// them (tarballKey): readers asking for one meanwhile share it, its bytes
+// those each one's own document names.
+const reading = new Map<string, Promise<Uint8Array>>()
+const unpacking = new Map<string, Promise<Unpacked>>()
+
+function sharedRead<T>(pending: Map<string, Promise<T>>, key: string, read: () => Promise<T>): Promise<T> {
+  let job = pending.get(key)
+  if (!job) {
+    const started = read().finally(() => { if (pending.get(key) === started) pending.delete(key) })
+    pending.set(key, job = started)
+  }
+  return job
+}
+
+const sharedTarball = (doc: NpmVersionDocument) => sharedRead(reading, tarballKey(doc), () => readTarball(doc))
+
+// A version's files, and the size of the tarball they came in.
+interface Unpacked { files: NpmPackageFile[]; tarballSize: number }
+
+// Public versions' files lately unpacked, by tarballKey.
+const unpacked = keptBriefly<Unpacked>({ keptMs: UNPACKED_KEPT_MS, maxBytes: MAX_UNPACKED_BYTES })
+
+// The version's files, kept from a load of them a little while since, else
+// unpacked from its tarball (and kept, a public version's): bytes its
+// reader's own document names, as its tarball's are.
+function unpackedOf(doc: NpmVersionDocument): Promise<Unpacked> {
+  const key = tarballKey(doc)
+  const kept = unpacked.get(key)
+  if (kept) return Promise.resolve(kept)
+  return sharedRead(unpacking, key, async () => {
+    const tarball = await sharedTarball(doc)
+    const value = { files: await readTarballFiles(tarball), tarballSize: tarball.byteLength }
+    if (!doc.private) unpacked.keep(key, value, value.files.reduce((sum, file) => sum + file.path.length + file.bytes.byteLength, 0))
+    return value
+  })
+}
+
+// A gzipped tarball's files, its top directory left out, as npm extracts
+// a package's: a repository tree's as GitHub archives it, too, its symlinks
+// let climb as readNpmTar's `symlinksAbove` says.
+export async function readTarballFiles(tarball: Uint8Array, options: { symlinksAbove?: number } = {}): Promise<NpmPackageFile[]> {
+  let tar: Uint8Array
+  try { tar = await decompress(tarball, 'gzip', { limit: MAX_TAR_BYTES }) }
   catch (err) {
-    throw new NpmPackageError((err as { code?: string }).code === 'ERR_BUFFER_TOO_LARGE' ? 413 : 422,
-      (err as { code?: string }).code === 'ERR_BUFFER_TOO_LARGE' ? 'package-too-large' : 'bad-tarball')
+    const limited = err instanceof CompressionError && err.limited
+    throw new NpmPackageError(limited ? 413 : 422, limited ? 'package-too-large' : 'bad-tarball')
   }
-  return readNpmTar(tar)
+  return readNpmTar(tar, options)
 }
 
 // The version's tarball, for download.
 export function loadNpmTarball(doc: NpmVersionDocument): NpmLoad<Uint8Array> {
-  return shared(`tarball ${doc.dist.integrity} ${doc.dist.tarball}`, doc, () => readTarball(doc))
+  return shared('tarball', doc, () => sharedTarball(doc))
 }
 
 // The version's files, for one of them to be pretty-printed (pretty-print.ts).
 export function loadNpmFiles(doc: NpmVersionDocument): NpmLoad<NpmPackageFile[]> {
-  return shared(`files ${doc.dist.integrity} ${doc.dist.tarball}`, doc, async () => readFiles(await readTarball(doc)))
+  return shared('files', doc, async () => (await unpackedOf(doc)).files)
 }
 
 // The version as the viewer reads it, as brotli-encoded JSON: `{ name,
 // version, private, integrity, tarballSize, published, manifest, files,
 // known }`, its files as npmFileRows gives them, and with them, as its files
-// are encoded, when it was published where that is kept (npmPublishTime,
-// else null) and what `known` gives: what is kept about its package
+// are encoded, when it was published where that is kept (`published`, asked
+// then, else null) and what `known` gives: what is kept about its package
 // (npm-insights.ts npmKnown), which the viewer need not then ask for. Encoding
 // is where a load holds the most, so it takes its place among the active
 // loads until its body is done, and readers asking meanwhile share that body,
-// the first one's manifest and `known` in it; nothing is kept once its readers
-// are done.
-export function loadNpmPackageBody(doc: NpmVersionDocument, known: () => object = () => ({})): NpmLoad<Buffer> {
-  return shared(`package ${doc.dist.integrity} ${doc.dist.tarball} ${doc.private}`, doc, async () => {
-    const tarball = await readTarball(doc)
-    const files = await readFiles(tarball)
+// the first one's manifest, `published` and `known` in it; the body is not
+// kept once its readers are done, its files a little while (unpackedOf).
+export function loadNpmPackageBody(doc: NpmVersionDocument, known: () => object = () => ({}),
+  published: () => Promise<string | null> = npmPublishTime(doc).read): NpmLoad<Buffer> {
+  return shared('package', doc, async () => {
+    const { files, tarballSize } = await unpackedOf(doc)
     return encodeBrotli(Buffer.from(JSON.stringify({
-      name: doc.name, version: doc.version, private: doc.private, integrity: doc.dist.integrity, tarballSize: tarball.byteLength,
-      published: npmPublishTime(doc.name, doc.version), manifest: doc.manifest, files: npmFileRows(files), known: known(),
+      name: doc.name, version: doc.version, private: doc.private, integrity: doc.dist.integrity, tarballSize,
+      published: await published(), manifest: doc.manifest, files: npmFileRows(files), known: known(),
     })))
   })
 }

package/server-managed/npm-packages.ts

@@ -14,6 +14,7 @@ import { Buffer } from 'node:buffer'
 import { createHash } from 'node:crypto'
 import { rm } from 'node:fs/promises'
 import { resolve } from 'node:path'
+import { ArchiveError, type Entry, unpack } from '@preventive/archive/tar.js'
 import { isExactVersion } from '@preventive/upstream/semver.js'
 import { isNpmPackageName, isNpmPackageSpec, npmPackageScope } from '../common/managed/npm-packages.js'
 import type { Role } from '../common/managed/roles.ts'
@@ -33,6 +34,9 @@ const MAX_DOCUMENT_BYTES = 4 * PACKUMENT_BYTES
 // stream holding them is bounded in npm-loads.ts.
 export const MAX_NPM_PACKAGE_BYTES = 64 * 1024 * 1024
 export const MAX_NPM_PACKAGE_FILES = 20_000
+// The entries its tar stream may hold, read before any is counted as a file:
+// a directory's too, as a GitHub tree's tarball has one for each, npm's none.
+export const MAX_NPM_TAR_ENTRIES = 2 * MAX_NPM_PACKAGE_FILES
 // What the files may take as the viewer's JSON. Escaping grows a text up to
 // six times (a control character becomes `\u0001`), so it is counted before
 // anything is serialized.
@@ -149,14 +153,60 @@ async function publicWholeDocument(name: string, url: string, { limit, signal }:
   return json
 }
 
+// Whole documents being asked of npm as anyone asks, by URL and limit:
+// readers asking for one meanwhile share it, as opening a version asks for
+// its package's for its versions, its advisories and its risk at once; each
+// gives up on it alone once its own signal aborts, and it is given up on once
+// they all have. None is asked with the token. Read alike by every reader,
+// none changing it.
+const wholeAsks = new Map<string, { asked: Promise<Record<string, unknown> | null>; controller: AbortController; readers: number }>()
+
+function sharedWholeDocument(name: string, url: string, { limit, signal }: { limit: number; signal: AbortSignal }): Promise<Record<string, unknown> | null> {
+  if (signal.aborted) return Promise.reject(signal.reason)
+  const key = `${url} ${limit}`
+  let ask = wholeAsks.get(key)
+  if (!ask) {
+    const controller = new AbortController()
+    const created = { asked: publicWholeDocument(name, url, { limit, signal: controller.signal }), controller, readers: 0 }
+    const settle = () => { if (wholeAsks.get(key) === created) wholeAsks.delete(key) }
+    created.asked.then(settle, settle)
+    wholeAsks.set(key, ask = created)
+  }
+  const held = ask
+  held.readers++
+  return new Promise((answer, fail) => {
+    let left = false
+    const leave = () => {
+      if (left) return false
+      left = true
+      held.readers--
+      signal.removeEventListener('abort', onAbort)
+      return true
+    }
+    const onAbort = () => {
+      if (!leave()) return
+      if (held.readers === 0) {
+        if (wholeAsks.get(key) === held) wholeAsks.delete(key)
+        held.controller.abort(signal.reason)
+      }
+      fail(signal.reason)
+    }
+    const answered = (json: Record<string, unknown> | null) => { if (leave()) answer(json) }
+    const failed = (err: unknown) => { if (leave()) fail(err) }
+    signal.addEventListener('abort', onAbort, { once: true })
+    held.asked.then(answered, failed)
+  })
+}
+
 // Asked anonymously first, unless its caller knows npm has none for anyone
 // (`anonymous` false): a reader without private access stops there. One
 // with it asks again with the token where npmTokenReads says so, and the
 // answer is private. A whole document (`whole`) anyone may read is kept
-// (publicWholeDocument).
+// (publicWholeDocument), and asked once for those asking at once
+// (sharedWholeDocument).
 async function readDocument(name: string, url: string, privileged: boolean,
   { whole = false, anonymous = true, ...options }: { accept: string; limit: number; signal: AbortSignal; whole?: boolean; anonymous?: boolean }) {
-  const open = anonymous ? whole ? await publicWholeDocument(name, url, options) : await registryDocument(url, { ...options, token: null }) : null
+  const open = anonymous ? whole ? await sharedWholeDocument(name, url, options) : await registryDocument(url, { ...options, token: null }) : null
   if (open) return { json: open, private: false }
   if (!npmTokenReads(name, privileged)) return null
   const json = await registryDocument(url, { ...options, token: npmToken() })
@@ -199,6 +249,8 @@ function versionDocument(name: string, json: Record<string, unknown>, isPrivate:
 // (npm-manifest.ts npmVersionFields), as the database keeps them (npm-kept.ts).
 export interface NpmVersionStore {
   read(name: string, version: string): Promise<{ at: number; json: Record<string, unknown> } | null>
+  // Those of `versions` kept, by `name@version`, in one read.
+  readMany?(versions: { name: string; version: string }[]): Promise<Map<string, { at: number; json: Record<string, unknown> }>>
   write(name: string, version: string, json: Record<string, unknown>): Promise<void>
 }
 
@@ -224,6 +276,20 @@ export function readNpmVersion(name: string, spec: string, privileged: boolean,
   })
 }
 
+// `store`, where it reads many at once, with the exact ones of `versions`
+// read from it in one read, for readNpmVersion to read each of those from as
+// it asks: a version and the one it is compared with, read together.
+export function readTogether(store: NpmVersionStore, versions: { name: string; version: string }[]): NpmVersionStore {
+  const exact = versions.filter(({ version }) => isExactVersion(version))
+  if (!store.readMany || exact.length < 2) return store
+  const keys = new Set(exact.map(({ name, version }) => `${name}@${version}`))
+  const kept = store.readMany(exact)
+  return { ...store, async read(name, version) {
+    const key = `${name}@${version}`
+    return keys.has(key) ? (await kept).get(key) ?? null : store.read(name, version)
+  } }
+}
+
 // The package's dist-tags and versions, newest published first, from its
 // abbreviated document; with `times`, from its whole one, which alone has
 // when each was published, `times` by version (ISO dates), and `created`,
@@ -273,77 +339,57 @@ const isPublishTime = (value: string | undefined) => typeof value === 'string' &
 const PUBLISH_TIMES = 100_000
 const publishTimes = new Map<string, string>()
 
-export const npmPublishTime = (name: string, version: string): string | null => publishTimes.get(`${name}@${version}`) ?? null
-
-export interface NpmPackageFile { path: string; bytes: Uint8Array }
-
-const textOf = (bytes: Uint8Array, start: number, length: number) => {
-  const field = bytes.subarray(start, start + length)
-  const end = field.indexOf(0)
-  return Buffer.from(end === -1 ? field : field.subarray(0, end)).toString('utf8')
+// Where public versions' publish times are kept beyond the instance, as the
+// database keeps them (npm-kept.ts npmPublishTimeStore).
+export interface NpmPublishTimeStore {
+  read(name: string, version: string): Promise<string | null>
+  write(name: string, version: string, at: string): Promise<void>
 }
 
-function sizeOf(bytes: Uint8Array, offset: number): number {
-  // GNU base-256 for sizes past the 11 octal digits.
-  if (bytes[offset]! & 0x80) {
-    let size = bytes[offset]! & 0x7f
-    for (let i = 1; i < 12; i++) size = size * 256 + bytes[offset + i]!
-    return size
+// When `doc` was published: as noted here, else as `store` keeps a public
+// version's, so that another instance, or this one restarted, has it before
+// it reads the whole document again. The store is read once, at once, beside
+// whatever its caller reads meanwhile, a failure read as none; `read` looks
+// here again as it is asked, as the whole document read meanwhile may have
+// noted it by then, and `keep` keeps there the time noted here where it read
+// none.
+export function npmPublishTime(doc: Pick<NpmVersionDocument, 'name' | 'version' | 'private'>, store?: NpmPublishTimeStore) {
+  const noted = () => publishTimes.get(`${doc.name}@${doc.version}`) ?? null
+  const kept = store && !doc.private ? store.read(doc.name, doc.version).catch(() => null) : Promise.resolve(null)
+  return {
+    read: async (): Promise<string | null> => noted() ?? await kept ?? noted(),
+    async keep() {
+      const at = noted()
+      if (at !== null && store && !doc.private && await kept === null) await store.write(doc.name, doc.version, at)
+    },
   }
-  const text = textOf(bytes, offset, 12).trim()
-  return /^[0-7]+$/u.test(text) ? Number.parseInt(text, 8) : Number.NaN
 }
 
-// The path a package's file is installed at: npm drops the first component
-// (`package/`, or whatever the archive used), and a path that would escape the
-// package, or names nothing, is not extracted.
-function packagePath(raw: string): string | null {
-  const parts = raw.split('/').slice(1).filter(part => part !== '' && part !== '.')
-  return parts.length === 0 || parts.includes('..') ? null : parts.join('/')
-}
+export interface NpmPackageFile { path: string; bytes: Uint8Array }
 
-// A pax header's path. Each record is `<length> <key>=<value>\n`, its length
-// in bytes, so records are walked as bytes and only their values decoded.
-function paxPath(data: Uint8Array): string | null {
-  let path = null
-  let at = 0
-  while (at < data.length) {
-    const space = data.indexOf(0x20, at)
-    const length = Number.parseInt(Buffer.from(data.subarray(at, space)).toString('latin1'), 10)
-    if (space === -1 || !Number.isSafeInteger(length) || length <= 0) break
-    const record = Buffer.from(data.subarray(space + 1, at + length - 1)).toString('utf8')
-    if (record.startsWith('path=')) path = record.slice(5)
-    at += length
+// The regular files of a tar stream, by package path: npm drops the first
+// component (`package/`, or whatever the archive used), and a file at the top
+// is not extracted. The stream is read by @preventive/archive, which refuses
+// one malformed, a name or a symlink that would climb out of it -- a symlink
+// past `symlinksAbove` levels above it, none unless given -- and a name
+// repeated as another entry, and gives up past MAX_NPM_TAR_ENTRIES before
+// reading on; a file under another top directory at the same path replaces
+// an earlier one, as extraction would. Directories, links and devices hold no
+// file content and are skipped.
+export function readNpmTar(tar: Uint8Array, { symlinksAbove = 0 }: { symlinksAbove?: number } = {}): NpmPackageFile[] {
+  let entries: Entry[]
+  try { entries = unpack(tar, { maxEntries: MAX_NPM_TAR_ENTRIES, symlinksAbove }) }
+  catch (err) {
+    if (err instanceof ArchiveError) throw err.limited ? new NpmPackageError(413, 'package-too-large') : new NpmPackageError(422, 'bad-tarball')
+    throw err
   }
-  return path
-}
-
-// The regular files of a tar stream, by package path, a later entry
-// replacing an earlier one as extraction would. Directories, links and
-// devices hold no file content and are skipped.
-export function readNpmTar(tar: Uint8Array): NpmPackageFile[] {
   const files = new Map<string, Uint8Array>()
   let total = 0
-  let longName: string | null = null
-  let offset = 0
-  while (offset + 512 <= tar.length) {
-    const header = tar.subarray(offset, offset + 512)
-    if (header.every(byte => byte === 0)) break
-    const size = sizeOf(header, 124)
-    const type = String.fromCodePoint(header[156]!)
-    const start = offset + 512
-    if (!Number.isSafeInteger(size) || size < 0 || start + size > tar.length) throw new NpmPackageError(422, 'bad-tarball')
-    const data = tar.subarray(start, start + size)
-    offset = start + Math.ceil(size / 512) * 512
-    if (type === 'x' || type === 'L') { longName = type === 'x' ? paxPath(data) : textOf(data, 0, data.length); continue }
-    if (type === 'g' || type === 'K') continue
-    const prefix = textOf(header, 257, 6) === 'ustar' ? textOf(header, 345, 155) : ''
-    const name = longName ?? (prefix ? `${prefix}/${textOf(header, 0, 100)}` : textOf(header, 0, 100))
-    longName = null
-    if (!['0', '\0', '7'].includes(type)) continue
-    const path = packagePath(name)
-    if (path === null) continue
-    total += size - (files.get(path)?.length ?? 0)
+  for (const { name, type, data } of entries) {
+    const slash = name.indexOf('/')
+    if ((type !== 'file' && type !== 'contiguous-file') || slash === -1) continue
+    const path = name.slice(slash + 1)
+    total += data.length - (files.get(path)?.length ?? 0)
     files.set(path, data)
     if (files.size > MAX_NPM_PACKAGE_FILES || total > MAX_NPM_PACKAGE_BYTES) throw new NpmPackageError(413, 'package-too-large')
   }

package/server-managed/npm-risk.ts

@@ -35,6 +35,7 @@ import { buffer } from 'node:stream/consumers'
 import type { CacheStore } from '@preventive/upstream/npm.js'
 import { DEFAULT_MINIMUM_RELEASE_AGE, type NpmAssessment, assessNpm, npmReadyDownload, npmReadyDownloads } from '../risk/index.js'
 import { CacheMissError, type CacheStorage } from './cache-storage.ts'
+import { npmVersionStore } from './npm-kept.ts'
 import { loadNpmTarball } from './npm-loads.ts'
 import { NpmPackageError, type NpmVersionDocument, npmTokenReads, readNpmPackument, readNpmVersion } from './npm-packages.ts'
 import { npmPackageScope } from '../common/managed/npm-packages.js'
@@ -176,7 +177,9 @@ export function npmRisk(doc: NpmVersionDocument, old: NpmVersionDocument | 'pick
     return scopes.get(scope)!
   }
   const given = old === 'pick' ? null : old
-  const documentOf = async (version: string) => [doc, given].find(read => read?.version === version) ?? await readNpmVersion(doc.name, version, privileged, signal) ?? undefined
+  // The one picked, read as the viewer reads a version, kept a day where public.
+  const documentOf = async (version: string) => [doc, given].find(read => read?.version === version)
+    ?? await readNpmVersion(doc.name, version, privileged, signal, npmVersionStore(stores.db, stores.debug)) ?? undefined
   return bounded(spare, () => assessNpm(doc.name, doc.version, { oldVersion: old === 'pick' ? true : old?.version ?? null, fromEmpty,
     minimumReleaseAge: teamScoped ? 0 : DEFAULT_MINIMUM_RELEASE_AGE,
     popularity: !teamScoped, cache: npmRiskCache(stores, !doc.private && !given?.private), signal, dependencies: !spare, advisories: !spare, osv: !doc.private, owned,