@preventive/triage 1.0.0-alpha.44 → 1.0.0-alpha.45

package/server-managed/index.ts

@@ -11,6 +11,7 @@ import { initializeApp } from '../server-common/initialize.ts'
 import { type ManagedConfig, loadManagedConfig } from './config.ts'
 import { type ManagedHttpDeps, createManagedRequestHandler } from './http.ts'
 import { setNpmTarballCache } from './npm-loads.ts'
+import { assembledReviewedDependencies } from './reviewed-dependencies.ts'
 import { setNpmDocumentCache } from './npm-packages.ts'
 import { loadManagedStatic } from './static.ts'
 import { openManagedStorage } from './storage.ts'
@@ -21,7 +22,7 @@ import { setCacheDir } from '@preventive/upstream/npm.js'
 const REAP_INTERVAL_MS = 3_600_000
 const REAP_RETRY_MS = 60_000
 
-type ManagedAppOptions = Partial<Pick<ManagedHttpDeps, 'next' | 'serverInfo' | 'isShuttingDown'>>
+type ManagedAppOptions = Partial<Pick<ManagedHttpDeps, 'next' | 'serverInfo' | 'isShuttingDown' | 'reviewedDependencies'>>
 
 export async function createManagedApp(config: ManagedConfig, options: ManagedAppOptions = {}) {
   // Audits keep upstream's records on disk where the config has a cache
@@ -34,7 +35,10 @@ export async function createManagedApp(config: ManagedConfig, options: ManagedAp
   // Load the optional peer only for serverless managed deployments. Embedding
   // hosts can discard listener promises, so retain maintenance inside the app.
   const waitUntil = config.serverless ? (await import('@vercel/functions')).waitUntil : undefined
-  return await initializeApp(rollback => assembleManagedApp(config, options, rollback, waitUntil))
+  // The host's reviewed dependencies, registered before; none after.
+  const reviewedDependencies = assembledReviewedDependencies()
+  const assembled = { ...options, ...reviewedDependencies ? { reviewedDependencies } : {} }
+  return await initializeApp(rollback => assembleManagedApp(config, assembled, rollback, waitUntil))
 }
 
 async function assembleManagedApp(config: ManagedConfig, options: ManagedAppOptions, rollback: AsyncDisposableStack, waitUntil?: (promise: Promise<unknown>) => void) {

package/server-managed/locked-packages.ts

@@ -7,6 +7,7 @@ import { LockfileError, parseNpmLockfile } from '@preventive/lockfile/npm.js'
 import { parsePnpmLockfile } from '@preventive/lockfile/pnpm.js'
 import { parseYarn1Lockfile } from '@preventive/lockfile/yarn1.js'
 import type { LockfileFormat } from '@preventive/upstream/github.js'
+import type { ReviewedLockfile } from './reviewed-dependencies.ts'
 
 export interface LockedPackage {
   name: string
@@ -19,9 +20,16 @@ export interface LockedPackage {
   // Where one from elsewhere than npm's registry comes from, without any
   // credentials its URL has.
   from?: string
-  // A mirror's tarball's, as the lockfile has it: npm's package of its name
-  // and version only where npm's integrity is among it.
+  // As the lockfile has it: a mirror's tarball's, npm's package of its name
+  // and version only where npm's integrity is among it; and, where its
+  // reviews ask (`review`), npm's registry's too.
   integrity?: string
+  // Where its reviews ask: the sha1 Yarn 1 writes after a tarball's `#`, in
+  // hex (`shasum`); the hash of the patch pnpm applies to it
+  // (`patchHash`, its patchedDependencies'); and the lockfile's format.
+  shasum?: string
+  patchHash?: string
+  lockfile?: ReviewedLockfile
   // Where only the importers' devDependencies reach it, as npm's `dev` flag
   // says, and an install of no devDependencies leaves it out: in npm's and
   // pnpm's lockfiles. Yarn 1's says nothing of it, which only the
@@ -32,10 +40,10 @@ export interface LockedPackage {
 // A package as each reader resolves one, its name the one it is published
 // under, and a registry tarball's URL held to its name and version.
 type Resolution =
-  | { type: 'tarball'; tarball: string | undefined; integrity: string | undefined; gitHosted?: boolean }
+  | { type: 'tarball'; tarball: string | undefined; integrity: string | undefined; sha1?: string | undefined; gitHosted?: boolean }
   | { type: 'git'; repo: string }
   | { type: 'directory' }
-interface ReadPackage { name: string; version: string | undefined; resolution: Resolution | undefined; dev?: boolean | undefined }
+interface ReadPackage { name: string; version: string | undefined; resolution: Resolution | undefined; dev?: boolean | undefined; patchHash?: string | undefined }
 
 // The packages of a lockfile of `format`, as its reader resolves them.
 // Versions go unchecked against ranges, as only what is locked is asked for.
@@ -54,41 +62,54 @@ function packagesOf(format: LockfileFormat, text: string): ReadPackage[] {
 }
 
 // npm's registry, by either of its names, and yarn's mirror of it, as
-// @preventive/lockfile holds their tarballs to a package's name and version.
+// @preventive/lockfile holds their tarballs to a package's name and version:
+// its link to one, as it names it, a hash after it alone (Yarn 1's sha1).
 const REGISTRY_HOSTS = new Set(['registry.npmjs.org', 'registry.npmjs.com', 'registry.yarnpkg.com'])
-const isRegistry = (url: URL) => url.protocol === 'https:' && !url.port && !url.username && !url.password
+const isRegistry = (url: URL) => url.protocol === 'https:' && !url.port && !url.username && !url.password && !url.search
   && REGISTRY_HOSTS.has(url.hostname.replace(/\.+$/u, ''))
 // Where a URL is, as it may be shown: its credentials, query and fragment
 // left out.
 const shown = (url: URL | null) => url ? { from: `${url.protocol}//${url.host}${url.pathname}` } : {}
 
-// Where a package read comes from (LockedPackage); null for one a package
-// bundles, with no resolution, or one of the project's own directories.
-function locked({ name, version, resolution }: ReadPackage): LockedPackage | null {
+// Where a package read comes from (LockedPackage), with npm's registry's
+// integrity where `review`; null for one a package bundles, with no
+// resolution, or one of the project's own directories.
+function located({ name, version, resolution }: ReadPackage, review: boolean): LockedPackage | null {
   const at = { name, version: version ?? null }
   if (resolution === undefined || resolution.type === 'directory') return null
   if (resolution.type === 'git') return { ...at, source: 'git', ...shown(URL.parse(resolution.repo)) }
   const { tarball, integrity } = resolution
   if (resolution.gitHosted) return { ...at, source: 'git', ...shown(URL.parse(tarball ?? '')) }
-  if (tarball === undefined) return { ...at, source: 'registry' }
+  const registry = { ...at, source: 'registry' as const, ...review && integrity ? { integrity } : {} }
+  if (tarball === undefined) return registry
   if (tarball.startsWith('file:')) return { ...at, source: 'file' }
   const url = URL.parse(tarball)
-  if (url && isRegistry(url)) return { ...at, source: 'registry' }
+  if (url && isRegistry(url)) return registry
   return integrity ? { ...at, ...shown(url), source: 'mirror', integrity } : { ...at, ...shown(url), source: 'tarball' }
 }
+// As above, and with what its reviews ask (LockedPackage `shasum`,
+// `patchHash`, `lockfile`) where `review`.
+function locked(pkg: ReadPackage, format: LockfileFormat, review: boolean): LockedPackage | null {
+  const one = located(pkg, review)
+  if (!one || !review) return one
+  const sha1 = pkg.resolution?.type === 'tarball' ? pkg.resolution.sha1 : undefined
+  return { ...one, ...sha1 ? { shasum: sha1 } : {}, ...pkg.patchHash ? { patchHash: pkg.patchHash } : {}, lockfile: format === 'yarn' ? 'yarn1' : format as ReviewedLockfile }
+}
 
 // The packages a lockfile of `format` (npm, pnpm or yarn, as
 // findRepoLockfiles names them) locks, each once, in the lockfile's order:
 // `dev` only where it is wherever it is installed, at each path of npm's
-// tree, and with each of pnpm's peers. Throws the reader's LockfileError or
-// YamlError for one it refuses.
-export function lockedPackages(format: LockfileFormat, text: string): LockedPackage[] {
+// tree, and with each of pnpm's peers; with `review`, each with what its
+// reviews hold it to (LockedPackage `integrity`, `shasum`, `patchHash`),
+// each patch of it apart. Throws the reader's LockfileError or YamlError for
+// one it refuses.
+export function lockedPackages(format: LockfileFormat, text: string, { review = false } = {}): LockedPackage[] {
   const found = new Map<string, LockedPackage>()
   for (const pkg of packagesOf(format, text)) {
-    const one = locked(pkg)
+    const one = locked(pkg, format, review)
     if (!one) continue
     if (pkg.dev === true) one.dev = true
-    const key = `${one.source}\0${one.name}@${one.version ?? ''}\0${one.from ?? ''}\0${one.integrity ?? ''}`
+    const key = `${one.source}\0${one.name}@${one.version ?? ''}\0${one.from ?? ''}\0${one.integrity ?? ''}\0${one.shasum ?? ''}\0${one.patchHash ?? ''}`
     const had = found.get(key)
     if (!had) found.set(key, one)
     else if (had.dev && !one.dev) delete had.dev

package/server-managed/lockfile-scope.ts

@@ -0,0 +1,49 @@
+// The lockfiles a repository's dependency review holds to, as an admin sets
+// it (repository-dependency-reviews.ts): every one but those it names
+// (`exclude`, naming none for all of them), or those it names alone
+// (`include`). Each is named by its path in the repository, or, ending in
+// `/`, by a directory, every lockfile under it. A directory of a pull
+// request's lockfiles (pull-dependencies.ts) is reviewed where one of its
+// lockfiles is held to; one none of whose are, not at all (`excluded`), nor
+// failing the review where it couldn't be read.
+export interface LockfileScope { mode: 'exclude' | 'include'; paths: string[] }
+export const EVERY_LOCKFILE: LockfileScope = { mode: 'exclude', paths: [] }
+export const MAX_SCOPED_LOCKFILES = 100
+const MAX_PATH = 300
+
+// A path as one is named: relative, of `/`-separated names none of which is
+// empty, `.` or `..`, nor holding a backslash or a control character; a
+// directory's ending in `/`.
+function namedPath(path: unknown): path is string {
+  if (typeof path !== 'string' || path.length === 0 || path.length > MAX_PATH || path.startsWith('/')) return false
+  if (/[\\\p{Cc}]/u.test(path)) return false
+  const names = path.endsWith('/') ? path.slice(0, -1).split('/') : path.split('/')
+  return names.every(name => name !== '' && name !== '.' && name !== '..')
+}
+
+// A scope as a request sends it, each path once, in order; null for one
+// that isn't one.
+export function scopeOf(value: unknown): LockfileScope | null {
+  const { mode, paths } = (value ?? {}) as { mode?: unknown; paths?: unknown }
+  if ((mode !== 'exclude' && mode !== 'include') || !Array.isArray(paths) || paths.length > MAX_SCOPED_LOCKFILES || !paths.every(namedPath)) return null
+  return { mode, paths: [...new Set(paths as string[])].toSorted() }
+}
+
+// A scope as stored (JSON, none for every lockfile), every lockfile where
+// it can't be read.
+export function storedScope(text: string | null): LockfileScope {
+  if (!text) return EVERY_LOCKFILE
+  try { return scopeOf(JSON.parse(text)) ?? EVERY_LOCKFILE } catch { return EVERY_LOCKFILE }
+}
+export const scopeText = (scope: LockfileScope) => isEvery(scope) ? null : JSON.stringify({ mode: scope.mode, paths: scope.paths })
+export const isEvery = (scope: LockfileScope) => scope.mode === 'exclude' && scope.paths.length === 0
+
+const names = (path: string, named: string) => named.endsWith('/') ? path.startsWith(named) : path === named
+// Whether the lockfile at `path` is held to.
+export function heldTo(scope: LockfileScope, path: string) {
+  const named = scope.paths.some(one => names(path, one))
+  return scope.mode === 'include' ? named : !named
+}
+// Whether a directory of lockfiles at `paths` is reviewed: where one is
+// held to, or it names none.
+export const reviewedIn = (scope: LockfileScope, paths: string[]) => paths.length === 0 || paths.some(path => heldTo(scope, path))

package/server-managed/npm-delta-worker.js

@@ -0,0 +1,23 @@
+// The lines a package's files change from one side to another (npm-delta.ts),
+// as @preventive/compare's compareTrees counts them (git's --numstat), off
+// the server's event loop: of its text files, renames found as git finds
+// them, a side not there empty; and, apart, those of its changes it stamps
+// `typeOnly`, declaration files changed in types alone.
+import { parentPort, workerData } from 'node:worker_threads'
+import { compareTrees } from '@preventive/compare'
+
+const tree = files => files.map(([name, data]) => ({ name, data }))
+const changes = compareTrees(tree(workerData.base), tree(workerData.other), { paths: 'keys', lines: true, renames: true })
+const answer = { additions: 0, deletions: 0, typeAdditions: 0, typeDeletions: 0, typeFiles: 0 }
+for (const change of changes) {
+  if (change.binary) continue
+  answer.additions += change.insertions ?? 0
+  answer.deletions += change.deletions ?? 0
+  if (change.typeOnly !== true) continue
+  answer.typeAdditions += change.insertions ?? 0
+  answer.typeDeletions += change.deletions ?? 0
+  answer.typeFiles++
+}
+// Node worker messages have no browser target origin.
+// oxlint-disable-next-line unicorn/require-post-message-target-origin
+parentPort.postMessage(answer)

package/server-managed/npm-delta.ts

@@ -0,0 +1,173 @@
+// The lines a version of a package changes against another, or against none,
+// its whole package's (GET /api/npm/delta), as a list of versions wanting
+// reviews tells them beside each: added and removed (`additions`,
+// `deletions`), as git's --numstat counts them, of their text files, a
+// renamed one diffed with what it was; and, where some of the files changed
+// are declaration files @preventive/compare's compareTrees proves changed in
+// types alone (`typeOnly`, proveTypeOnly: types and comments alone erased,
+// the runtime they leave as it was, an added one's empty or a single bare
+// `export {}`), those of the rest (`typeless`), what a reviewer may pass by
+// leaving those out; and whether a binary file, which they count none of,
+// was added, removed or changed (`binary`), one moved as it was not. Worked
+// out in a worker, off the event loop, as many at once per process as
+// MAX_ACTIVE, their files read only once their turn has come, those waiting
+// past MAX_WAITING refused, each waiting a minute at most and counting in a
+// minute at most; one asked again meanwhile sharing the first's; and kept by
+// both sides' integrities, in this instance and beyond it (`kept`), what a
+// version holds never changing. Access is the caller's to check.
+import { Worker } from 'node:worker_threads'
+import { createHash } from 'node:crypto'
+import type { KeptStore } from './npm-kept.ts'
+import { type Lru, lru } from './lru.ts'
+import { NpmPackageError, type NpmPackageFile, npmFileText } from './npm-packages.ts'
+
+export interface NpmDelta { additions: number; deletions: number; typeless: { additions: number; deletions: number } | null; binary: boolean }
+
+const MAX_ACTIVE = 2
+const MAX_WAITING = 64
+const DELTA_TIMEOUT_MS = 60_000
+// The text the two sides may hold together, past which none is counted.
+const MAX_TEXT_LENGTH = 64 * 1024 * 1024
+
+const deltas: Lru<NpmDelta> = lru(20_000)
+const working = new Map<string, Promise<NpmDelta>>()
+let active = 0
+const waiting: Array<() => void> = []
+
+async function turn(deadline: number): Promise<void> {
+  if (active < MAX_ACTIVE) { active++; return }
+  if (waiting.length >= MAX_WAITING) throw new NpmPackageError(429, 'delta-busy')
+  await new Promise<void>((resolve, reject) => {
+    const next = () => { clearTimeout(timer); resolve() }
+    const timer = setTimeout(() => {
+      waiting.splice(waiting.indexOf(next), 1)
+      reject(new NpmPackageError(504, 'delta-timeout'))
+    }, deadline - Date.now())
+    waiting.push(next)
+  })
+}
+function done() {
+  const next = waiting.shift()
+  if (next) next()
+  else active--
+}
+
+const lines = (told: unknown): told is { additions: number; deletions: number } => typeof told === 'object' && told !== null
+  && Number.isSafeInteger((told as { additions?: unknown }).additions) && Number.isSafeInteger((told as { deletions?: unknown }).deletions)
+// A delta as kept, none where it isn't one, or was kept before it told of
+// binary files.
+const deltaOf = (told: unknown): NpmDelta | null => lines(told) && ((told as NpmDelta).typeless === null || lines((told as NpmDelta).typeless))
+  && typeof (told as NpmDelta).binary === 'boolean' ? { additions: told.additions, deletions: told.deletions, typeless: (told as NpmDelta).typeless,
+    binary: (told as NpmDelta).binary } : null
+
+type Counted = { additions: number; deletions: number; typeAdditions: number; typeDeletions: number; typeFiles: number }
+const isCounted = (told: unknown): told is Counted => typeof told === 'object' && told !== null
+  && ['additions', 'deletions', 'typeAdditions', 'typeDeletions', 'typeFiles'].every(key => Number.isSafeInteger((told as Record<string, unknown>)[key]))
+
+// What the worker counts of `base` against `other`, each its text files' paths and texts.
+function counted(base: [string, string][], other: [string, string][], deadline: number): Promise<Counted> {
+  let worker: Worker
+  try {
+    worker = new Worker(new URL('./npm-delta-worker.js', import.meta.url), {
+      workerData: { base, other }, execArgv: ['--disable-warning=ExperimentalWarning'], resourceLimits: { maxOldGenerationSizeMb: 1536 },
+    })
+  } catch (err) {
+    console.warn('managed: a package delta could not start:', err)
+    throw new NpmPackageError(503, 'unavailable')
+  }
+  return new Promise((resolve, reject) => {
+    let answer: Counted | null = null, failure = new NpmPackageError(422, 'delta-failed')
+    const timer = setTimeout(() => { failure = new NpmPackageError(504, 'delta-timeout'); void worker.terminate() }, deadline - Date.now())
+    worker.once('message', (told: unknown) => {
+      if (isCounted(told)) answer = told
+      void worker.terminate()
+    })
+    worker.once('error', err => { console.warn('managed: a package delta failed:', err) })
+    worker.once('exit', () => {
+      clearTimeout(timer)
+      if (answer) resolve(answer)
+      else reject(failure)
+    })
+  })
+}
+
+// A side's text files, by path, how long their texts are, and its other,
+// binary, files.
+function texts(files: NpmPackageFile[] | null) {
+  let length = 0
+  const binaries: NpmPackageFile[] = []
+  const read = (files ?? []).flatMap(file => {
+    const text = npmFileText(file.bytes)
+    if (text === null) { binaries.push(file); return [] }
+    length += text.length
+    return [[file.path, text] as [string, string]]
+  })
+  return { read, length, binaries }
+}
+
+const same = (a: Uint8Array, b: Uint8Array) => Buffer.from(a.buffer, a.byteOffset, a.byteLength).equals(b)
+const digest = (bytes: Uint8Array) => createHash('sha256').update(bytes).digest('base64')
+
+// Whether `news`, a side's binary files, differ from `olds`, the other's:
+// one added, removed or changed, one moved where it is the same as one that
+// was elsewhere not, matched by their digests, each file looked at once.
+function binaryChanged(olds: NpmPackageFile[], news: NpmPackageFile[]): boolean {
+  if (olds.length !== news.length) return true
+  const left = new Map(olds.map(({ path, bytes }) => [path, bytes]))
+  const moved: Uint8Array[] = []
+  for (const { path, bytes } of news) {
+    const was = left.get(path)
+    if (was && same(was, bytes)) left.delete(path)
+    else moved.push(bytes)
+  }
+  // Those not where they were, by their digests, as many of each as there are.
+  const unmoved = new Map<string, number>()
+  for (const bytes of left.values()) {
+    const key = digest(bytes)
+    unmoved.set(key, (unmoved.get(key) ?? 0) + 1)
+  }
+  return moved.some(bytes => {
+    const key = digest(bytes)
+    const count = unmoved.get(key) ?? 0
+    if (count === 0) return true
+    unmoved.set(key, count - 1)
+    return false
+  })
+}
+
+type Side = { integrity: string; files: () => Promise<NpmPackageFile[]> }
+
+// The delta of `other` (its integrity, and how to read its files) against
+// `base`, or none, as held, else shared with one asked before it, else
+// looked for where it is kept, else worked out and kept: shared from before
+// it is looked for, lest those asked together each look, find none, and work
+// it out.
+export function npmDelta(base: Side | null, other: Side, kept: KeptStore<NpmDelta> | null = null): Promise<NpmDelta> {
+  const key = `${base?.integrity ?? '(empty)'}\0${other.integrity}`
+  const known = deltas.get(key)
+  if (known) return Promise.resolve(known)
+  const shared = working.get(key)
+  if (shared) return shared
+  const work = worked(key, base, other, kept).finally(() => working.delete(key))
+  working.set(key, work)
+  return work
+}
+
+// As kept, else its turn awaited, then both sides' files read, counted and kept.
+async function worked(key: string, base: Side | null, other: Side, kept: KeptStore<NpmDelta> | null): Promise<NpmDelta> {
+  const known = deltaOf((await kept?.read(key))?.value)
+  if (known) { deltas.set(key, known); return known }
+  await turn(Date.now() + DELTA_TIMEOUT_MS)
+  let binary, told
+  try {
+    const [olds, news] = [texts(base ? await base.files() : null), texts(await other.files())]
+    if (olds.length + news.length > MAX_TEXT_LENGTH) throw new NpmPackageError(413, 'package-too-large')
+    binary = binaryChanged(olds.binaries, news.binaries)
+    told = await counted(olds.read, news.read, Date.now() + DELTA_TIMEOUT_MS)
+  } finally { done() }
+  const delta = { additions: told.additions, deletions: told.deletions,
+    typeless: told.typeFiles > 0 ? { additions: told.additions - told.typeAdditions, deletions: told.deletions - told.typeDeletions } : null, binary }
+  deltas.set(key, delta)
+  await kept?.write(key, delta, Date.now())
+  return delta
+}

package/server-managed/npm-insights.ts

@@ -362,25 +362,34 @@ async function askAdvisories(name: string, versions: string[], { githubToken, re
   }
   if (result.status !== 200) throw new NpmPackageError(502, 'upstream-unavailable')
   const index = new Map(versions.map((version, i) => [version, i]))
-  // npm's registry answers an advisory once a range it covers: here it is one
-  // row, its ranges joined as semver joins them, the versions and CWEs they
-  // cover together, and the highest score.
+  // npm's registry answers an advisory once a range it covers, and upstream
+  // answers the versions OSV's record of it, or its repository, covers beyond
+  // npm's under the same id, in a row of their own (as hermes-engine's
+  // GHSA-f5x2-xv93-4p23: npm's `<=0.4.3`, OSV's fixed in 0.5.2): here it is
+  // one row, the first's (npm's, then OSV's, then the repository's, as
+  // upstream orders them) with what it lacks taken from the rest, its ranges
+  // joined as semver joins them, the versions and CWEs they cover together,
+  // and the worst severity and highest score of them: a version only the
+  // repository reports keeps the severity it gives.
   const merged = new Map<string, NpmAdvisory>()
   for (const row of result.body) {
     const affected = row.versions.map(version => index.get(version)).filter(at => at !== undefined)
-    const key = `${row.source}\0${row.id}`
-    const known = merged.get(key)
+    const own: NpmAdvisory = {
+      id: row.id, source: row.source, ...row.ghsa && { ghsa: row.ghsa }, ...row.url && { url: row.url }, ...row.title && { title: row.title },
+      ...row.severity && { severity: row.severity }, ...row.cvss !== undefined && { cvss: row.cvss }, cwe: row.cwe, ...row.range && { range: row.range },
+      ...row.details && { details: row.details }, affected,
+    }
+    const known = merged.get(row.id)
     if (!known) {
-      merged.set(key, {
-        id: row.id, source: row.source, ...row.ghsa && { ghsa: row.ghsa }, ...row.url && { url: row.url }, ...row.title && { title: row.title },
-        ...row.severity && { severity: row.severity }, ...row.cvss !== undefined && { cvss: row.cvss }, cwe: row.cwe, ...row.range && { range: row.range },
-        ...row.details && { details: row.details }, affected,
-      })
+      merged.set(row.id, own)
       continue
     }
     const range = [known.range, row.range].filter(Boolean).join(' || ')
     const cvss = Math.max(known.cvss ?? -1, row.cvss ?? -1)
-    merged.set(key, { ...known, ...range && { range }, ...cvss >= 0 && { cvss }, cwe: [...new Set([...known.cwe, ...row.cwe])], affected: [...known.affected, ...affected] })
+    const rank = (severity: string | undefined) => SEVERITY_RANK[severity ?? ''] ?? 4
+    const severity = rank(row.severity) < rank(known.severity) ? row.severity : known.severity
+    merged.set(row.id, { ...own, ...known, ...range && { range }, ...severity && { severity }, ...cvss >= 0 && { cvss }, cwe: [...new Set([...known.cwe, ...row.cwe])],
+      affected: [...known.affected, ...affected] })
   }
   return {
     versions, repository, ...keptAt && { keptAt },

package/server-managed/npm-kept.ts

@@ -6,7 +6,12 @@
 // then, as a version can be deprecated, its package made private, or itself
 // removed); and, for scans, each package's risk line, for as long as a scan
 // keeps it, and what a scan read of GitHub, for a day, and of npm's
-// advisories, for an hour (repository-risk.ts). Each store removes its own rows kept longer
+// advisories, for an hour (repository-risk.ts); and what the code viewer read
+// of a repository's GitHub, its issues, pull requests and agents' work, for a
+// day (http.ts keptAnswer), and its history, a page from a commit at a time,
+// for a month (repository-history.ts historyFrom), as are the files each of
+// its commits changes and holds (repository-changes.ts), and the packages a
+// pull request's lockfiles add (pull-dependencies.ts). Each store removes its own rows kept longer
 // than that as it writes, at most once an hour a database (pruneKept), so
 // that rows no read will take again don't pile up.
 // The database is an optimization: a failure reads as a miss, and writes
@@ -22,6 +27,8 @@ export interface KeptStore<T> {
   // Those of `keys` kept, by key, in one read of the database.
   readMany(keys: string[]): Promise<Map<string, { at: number; value: T }>>
   write(key: string, value: T, at: number): Promise<void>
+  // Removes what is kept by `key`.
+  remove(key: string): Promise<void>
 }
 
 // When each database last had the rows under each prefix removed.
@@ -63,6 +70,9 @@ function keptStore<T>(db: UpstreamCacheStore, type: string, keptMs: number, debu
       try { await db.setUpstreamCacheEntry(keyOf(key), JSON.stringify({ at, value }), at) } catch (error) { warn('write', error) }
       await prune(at)
     },
+    async remove(key) {
+      try { await db.deleteUpstreamCacheEntries(keyOf(key), Number.MAX_SAFE_INTEGER) } catch (error) { warn('remove', error) }
+    },
   }
 }
 
@@ -83,7 +93,44 @@ export const riskLineStore = <T>(db: UpstreamCacheStore, keptMs: number, debug =
 // What a scan read, each read for keptMs (repository-risk.ts): of GitHub, a
 // repository's lockfiles as listed at a commit (`listings`) and a lockfile's
 // packages by its blob (`lockfiles`), for SCAN_READ_KEPT_MS; of npm, a
-// lockfile's advisories by its blob (`advisories`), for RISK_KEPT_MS; and the
-// repositories a scan in parts told whole (`progress`).
-export const scanReadStore = <T>(db: UpstreamCacheStore, type: 'listings' | 'lockfiles' | 'advisories' | 'progress', keptMs: number, debug = false): KeptStore<T> =>
+// lockfile's advisories by its blob (`advisories`), for RISK_KEPT_MS; the
+// repositories a scan in parts told whole (`progress`); a repository's
+// Actions' files by the commit read (`action-files`), and the actions and
+// reusable workflows of others they name, each resolved (`actions`), of
+// public repositories alone (repository-actions.ts).
+export const scanReadStore = <T>(db: UpstreamCacheStore, type: 'listings' | 'lockfiles' | 'advisories' | 'progress' | 'action-files' | 'actions', keptMs: number, debug = false): KeptStore<T> =>
   keptStore<T>(db, `repository-risk/${type}`, keptMs, debug)
+// What the code viewer read of a repository (http.ts readWholeRepository), by
+// its id and what was asked, each kept REPOSITORY_READ_KEPT_MS.
+export const REPOSITORY_READ_KEPT_MS = 24 * 60 * 60_000
+export const repositoryReadStore = (db: UpstreamCacheStore, debug = false): KeptStore<object> =>
+  keptStore<object>(db, 'repository/reads', REPOSITORY_READ_KEPT_MS, debug)
+// Pages of a repository's history (repository-history.ts historyFrom), by its
+// id and the commit each starts at, which never change: kept a month after
+// they were last read from GitHub.
+export const HISTORY_PAGE_KEPT_MS = 30 * 24 * 60 * 60_000
+export const historyPageStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/history', HISTORY_PAGE_KEPT_MS, debug)
+// The pull requests squashed into a repository's commits, by the commit
+// (repository-history.ts squashedPulls): kept as its history's pages are.
+export const historySquashStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/squashes', HISTORY_PAGE_KEPT_MS, debug)
+// The files each of a repository's commits changes (repository-changes.ts
+// repositoryChanges), by the commit, which never change: kept as its
+// history's pages are.
+export const historyChangeStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/changes', HISTORY_PAGE_KEPT_MS, debug)
+// The files a repository's commit holds (repository-changes.ts
+// repositoryTree), by the commit, which never change: kept as its history's
+// pages are.
+export const historyTreeStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/trees', HISTORY_PAGE_KEPT_MS, debug)
+// The packages a repository's pull request's lockfiles add
+// (pull-dependencies.ts), by the commits of its base and head, which never
+// change: kept as its history's pages are.
+export const pullDependencyStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/pull-dependencies', HISTORY_PAGE_KEPT_MS, debug)
+// The versions a repository's lockfiles locked at the commit its dependencies
+// are reviewed since (pull-dependency-reviews.ts reviewBaseline), by the
+// commit, which never change: kept as its history's pages are, never stored
+// as reviews.
+// The lines a version changes against another, or none (npm-delta.ts), by
+// both sides' integrities, each kept HISTORY_PAGE_KEPT_MS.
+export const npmDeltaStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'npm/deltas', HISTORY_PAGE_KEPT_MS, debug)
+export const reviewBaselineStore = (db: UpstreamCacheStore, debug = false): KeptStore<{ versions: string[]; unread: string[] }> =>
+  keptStore<{ versions: string[]; unread: string[] }>(db, 'repository/review-baselines', HISTORY_PAGE_KEPT_MS, debug)

package/server-managed/npm-loads.ts

@@ -5,8 +5,7 @@
 import { Buffer } from 'node:buffer'
 import { homedir } from 'node:os'
 import { join, resolve } from 'node:path'
-import { promisify } from 'node:util'
-import { gunzip } from 'node:zlib'
+import { CompressionError, decompress } from '@preventive/archive/compression.js'
 import { encodeBrotli } from './brotli.ts'
 import { MAX_NPM_PACKAGE_BYTES, MAX_NPM_PACKAGE_FILES, NPM_REGISTRY, NpmPackageError, type NpmPackageFile, type NpmVersionDocument, askUpstream,
   npmFileRows, npmPublishTime, npmToken, readNpmTar } from './npm-packages.ts'
@@ -21,8 +20,6 @@ const TARBALL_TIMEOUT_MS = 120_000
 // and their encoded JSON.
 const MAX_ACTIVE_LOADS = 4
 
-const gunzipAsync = promisify(gunzip)
-
 // A load's result, shared by its readers, and a reader's release of it.
 export interface NpmLoad<T> { result: Promise<T>; release: () => void }
 
@@ -43,9 +40,16 @@ const refused = <T>(status: number, code: string): NpmLoad<T> => {
 function shared<T>(key: string, doc: NpmVersionDocument, work: () => Promise<T>): NpmLoad<T> {
   const { unpackedSize, fileCount } = doc.dist
   if ((unpackedSize ?? 0) > MAX_NPM_PACKAGE_BYTES || (fileCount ?? 0) > MAX_NPM_PACKAGE_FILES) return refused(413, 'package-too-large')
+  return sharedLoad(key, work, 'npm-busy')
+}
+
+// A load among the active ones, by `key`, refused as `busy` past them: a
+// repository directory's (repository-code.ts) as a package's, each holding
+// alike a tarball, its tar, its files and their encoded JSON.
+export function sharedLoad<T>(key: string, work: () => Promise<T>, busy: string): NpmLoad<T> {
   let load = loads.get(key)
   if (!load) {
-    if (loads.size >= MAX_ACTIVE_LOADS) return refused(429, 'npm-busy')
+    if (loads.size >= MAX_ACTIVE_LOADS) return refused(429, busy)
     const created: Load = { job: work(), readers: 0, settled: false }
     const settle = () => {
       created.settled = true
@@ -125,14 +129,17 @@ async function readTarball(doc: NpmVersionDocument): Promise<Uint8Array> {
   return bytes
 }
 
-async function readFiles(tarball: Uint8Array): Promise<NpmPackageFile[]> {
-  let tar: Buffer
-  try { tar = await gunzipAsync(tarball, { maxOutputLength: MAX_TAR_BYTES }) }
+// A gzipped tarball's files, its top directory left out, as npm extracts
+// a package's: a repository tree's as GitHub archives it, too, its symlinks
+// let climb as readNpmTar's `symlinksAbove` says.
+export async function readTarballFiles(tarball: Uint8Array, options: { symlinksAbove?: number } = {}): Promise<NpmPackageFile[]> {
+  let tar: Uint8Array
+  try { tar = await decompress(tarball, 'gzip', { limit: MAX_TAR_BYTES }) }
   catch (err) {
-    throw new NpmPackageError((err as { code?: string }).code === 'ERR_BUFFER_TOO_LARGE' ? 413 : 422,
-      (err as { code?: string }).code === 'ERR_BUFFER_TOO_LARGE' ? 'package-too-large' : 'bad-tarball')
+    const limited = err instanceof CompressionError && err.limited
+    throw new NpmPackageError(limited ? 413 : 422, limited ? 'package-too-large' : 'bad-tarball')
   }
-  return readNpmTar(tar)
+  return readNpmTar(tar, options)
 }
 
 // The version's tarball, for download.
@@ -142,7 +149,7 @@ export function loadNpmTarball(doc: NpmVersionDocument): NpmLoad<Uint8Array> {
 
 // The version's files, for one of them to be pretty-printed (pretty-print.ts).
 export function loadNpmFiles(doc: NpmVersionDocument): NpmLoad<NpmPackageFile[]> {
-  return shared(`files ${doc.dist.integrity} ${doc.dist.tarball}`, doc, async () => readFiles(await readTarball(doc)))
+  return shared(`files ${doc.dist.integrity} ${doc.dist.tarball}`, doc, async () => readTarballFiles(await readTarball(doc)))
 }
 
 // The version as the viewer reads it, as brotli-encoded JSON: `{ name,
@@ -158,7 +165,7 @@ export function loadNpmFiles(doc: NpmVersionDocument): NpmLoad<NpmPackageFile[]>
 export function loadNpmPackageBody(doc: NpmVersionDocument, known: () => object = () => ({})): NpmLoad<Buffer> {
   return shared(`package ${doc.dist.integrity} ${doc.dist.tarball} ${doc.private}`, doc, async () => {
     const tarball = await readTarball(doc)
-    const files = await readFiles(tarball)
+    const files = await readTarballFiles(tarball)
     return encodeBrotli(Buffer.from(JSON.stringify({
       name: doc.name, version: doc.version, private: doc.private, integrity: doc.dist.integrity, tarballSize: tarball.byteLength,
       published: npmPublishTime(doc.name, doc.version), manifest: doc.manifest, files: npmFileRows(files), known: known(),