@preventive/triage 1.0.0-alpha.46 → 1.0.0-alpha.48

package/server-managed/index.ts

@@ -44,7 +44,7 @@ export async function createManagedApp(config: ManagedConfig, options: ManagedAp
 
 async function assembleManagedApp(config: ManagedConfig, { ui = true, ...options }: ManagedAppOptions, rollback: AsyncDisposableStack, waitUntil?: (promise: Promise<unknown>) => void) {
   const storage = await openManagedStorage(config)
-  const { db, avatarStore, reportStore, bundleStore, bundleCache, reportSourcesCache, prettyCache, npmCacheStorage } = storage
+  const { db, avatarStore, reportStore, bundleStore, bundleCache, reportSourcesCache, prettyCache, npmCacheStorage, repositoryCodeStorage } = storage
   rollback.defer(() => db.close())
   const originGate = createOriginGate(config.host, config.trustProxyEnv)
 
@@ -60,7 +60,7 @@ async function assembleManagedApp(config: ManagedConfig, { ui = true, ...options
     indexOnly: options.next != null, scanServer: options.serverInfo?.deepviewScanServer ?? null,
   }) : () => false
   const routeRequest = createManagedRequestHandler({
-    ...options, config, db, avatarStore, reportStore, bundleStore, bundleCache, reportSourcesCache, prettyCache, npmCacheStorage, originGate, serveStatic,
+    ...options, config, db, avatarStore, reportStore, bundleStore, bundleCache, reportSourcesCache, prettyCache, npmCacheStorage, repositoryCodeStorage, originGate, serveStatic,
     ...(storage.uploadStore ? { uploadStore: storage.uploadStore } : {}), ...(waitUntil ? { waitUntil } : {}),
     isShuttingDown, track,
   })

package/server-managed/link-reports.ts

@@ -39,7 +39,11 @@ function decryptGroups(key: StorageKey | null, id: string, payload: string): str
   try { return JSON.parse(plain.toString('utf8')) as string[][] } finally { plain.fill(0) }
 }
 export async function linkRevision(db: ManagedSql): Promise<string> {
-  const rows = await db.prepare('SELECT id, sha256 FROM managed_link_report WHERE enabled = 1 ORDER BY id').all()
+  return linkRevisionOf(await db.prepare('SELECT id, sha256 FROM managed_link_report WHERE enabled = 1 ORDER BY id').all())
+}
+// The revision of the enabled link reports' `{ id, sha256 }` rows, by id,
+// however they were read.
+export function linkRevisionOf(rows: unknown[]): string {
   return rows.length > 0 ? createHash('sha256').update(JSON.stringify(rows)).digest('base64url') : ''
 }
 function parseLinkReport(content: string) {

package/server-managed/managed-issues.ts

@@ -53,6 +53,9 @@ export function managedIssueMethods(db: ManagedSql): ManagedIssueStore {
     issue_url AS issueUrl, auto_fix_url AS autoFixUrl, auto_fix_checked_at AS autoFixCheckedAt, created_by AS createdBy, created_at AS createdAt`
   const get = db.prepare(`SELECT ${fields} FROM managed_finding_issue WHERE finding_id = ?`)
   const list = db.prepare(`SELECT ${fields} FROM managed_finding_issue WHERE finding_id IN (SELECT value FROM json_each(?)) ORDER BY finding_id`)
+  // A lookup's derived Fix replaces the one it read, unless one started later
+  // changed it since: the check time, which only changes stamp
+  // (github-issue-links.ts issueUpdates), orders them.
   const autoFix = db.prepare(`UPDATE managed_finding_issue SET auto_fix_url = ?, auto_fix_checked_at = ?
     WHERE finding_id = ? AND issue_url = ? AND COALESCE(auto_fix_url, '') = COALESCE(?, '')
       AND (auto_fix_checked_at IS NULL OR auto_fix_checked_at <= ?)`)

package/server-managed/npm-delta.ts

@@ -154,6 +154,22 @@ export function npmDelta(base: Side | null, other: Side, kept: KeptStore<NpmDelt
   return work
 }
 
+// The deltas of those of `sides` (each the integrities of the version it is
+// against, or none, and of the version) held or kept, by their keys
+// (npmDelta's), in one read of `kept` for those not held: none worked out.
+export async function keptNpmDeltas(sides: [string | null, string][], kept: KeptStore<NpmDelta>): Promise<Map<string, NpmDelta>> {
+  const keys = [...new Set(sides.map(([base, other]) => `${base ?? '(empty)'}\0${other}`))]
+  const found = new Map(keys.flatMap(key => { const known = deltas.get(key); return known ? [[key, known] as const] : [] }))
+  const unheld = keys.filter(key => !found.has(key))
+  for (const [key, row] of unheld.length > 0 ? await kept.readMany(unheld) : []) {
+    const known = deltaOf(row.value)
+    if (!known) continue
+    deltas.set(key, known)
+    found.set(key, known)
+  }
+  return found
+}
+
 // As kept, else its turn awaited, then both sides' files read, counted and kept.
 async function worked(key: string, base: Side | null, other: Side, kept: KeptStore<NpmDelta> | null): Promise<NpmDelta> {
   const known = deltaOf((await kept?.read(key))?.value)

package/server-managed/npm-kept.ts

@@ -11,10 +11,14 @@
 // of a repository's GitHub, its issues, pull requests and agents' work, for a
 // day (http.ts keptAnswer), and its history, a page from a commit at a time,
 // for a month (repository-history.ts historyFrom), as are the files each of
-// its commits changes and holds (repository-changes.ts), and the packages a
-// pull request's lockfiles add (pull-dependencies.ts). Each store removes its own rows kept longer
-// than that as it writes, at most once an hour a database (pruneKept), so
-// that rows no read will take again don't pile up.
+// its commits changes and holds (repository-changes.ts), its trees as the
+// code viewer reads them (codeTreeStore), the packages a pull request's
+// lockfiles add (pull-dependencies.ts), and the lockfiles a commit lists and
+// each one's packages, as its dependency review reads them
+// (commit-lockfiles.ts); and the commit the code viewer keeps its
+// directories at, for as long as it does (codeHeadStore). Each store removes
+// its own rows kept longer than that as it writes, at most once an hour a
+// database (pruneKept), so that rows no read will take again don't pile up.
 // The database is an optimization: a failure reads as a miss, and writes
 // nothing.
 import { createHash } from 'node:crypto'
@@ -38,15 +42,25 @@ export interface KeptStore<T> {
 // When each database last had the rows under each prefix removed.
 const PRUNE_EVERY_MS = 60 * 60_000
 const pruned = new WeakMap<UpstreamCacheStore, Map<string, number>>()
+// Who sees each database's prunes through (index.ts: its app, which waits
+// for them before closing it, and on serverless keeps its invocation alive
+// till they settle), as a write no longer waits for one.
+const pruneOwners = new WeakMap<UpstreamCacheStore, (prune: Promise<unknown>) => void>()
+export function ownKeptPrunes(db: UpstreamCacheStore, own: (prune: Promise<unknown>) => void) {
+  pruneOwners.set(db, own)
+}
 
 // Removes the rows under `prefix` kept `keptMs` before `at`, at most once an
 // hour a database, as rows under it are written: here, and risk's downloads
-// (npm-risk.ts). A failure is `warn`ed of, and removes nothing.
-export async function pruneKept(db: UpstreamCacheStore, prefix: string, keptMs: number, at: number, warn: (error: unknown) => void) {
+// (npm-risk.ts), each one run handed to the database's owner. A failure is
+// `warn`ed of, and removes nothing.
+export function pruneKept(db: UpstreamCacheStore, prefix: string, keptMs: number, at: number, warn: (error: unknown) => void): Promise<void> {
   const times = pruned.get(db) ?? pruned.set(db, new Map()).get(db)!
-  if (at - (times.get(prefix) ?? -Infinity) < PRUNE_EVERY_MS) return
+  if (at - (times.get(prefix) ?? -Infinity) < PRUNE_EVERY_MS) return Promise.resolve()
   times.set(prefix, at)
-  try { await db.deleteUpstreamCacheEntries(prefix, at - keptMs) } catch (error) { warn(error) }
+  const prune = (async () => { try { await db.deleteUpstreamCacheEntries(prefix, at - keptMs) } catch (error) { warn(error) } })()
+  pruneOwners.get(db)?.(prune)
+  return prune
 }
 
 // Values of `type`, each read for `keptMs` after it was kept.
@@ -79,7 +93,11 @@ function keptStore<T>(db: UpstreamCacheStore, type: string, keptMs: number, debu
     },
     async write(key, value, at) {
       try { await db.setUpstreamCacheEntry(keyOf(key), JSON.stringify({ at, value }), at) } catch (error) { warn('write', error) }
-      await prune(at)
+      // Behind the write, not before it is done: the prune reads the whole
+      // table (no index serves its prefix), which no writer need wait on.
+      // Its hour is marked before it runs, so writes meanwhile start none,
+      // and its database's owner sees it through (ownKeptPrunes).
+      void prune(at)
     },
     async remove(key) {
       try { await db.deleteUpstreamCacheEntries(keyOf(key), Number.MAX_SAFE_INTEGER) } catch (error) { warn('remove', error) }
@@ -141,7 +159,7 @@ export const repositoryReadStore = (db: UpstreamCacheStore, debug = false): Kept
 export const HISTORY_PAGE_KEPT_MS = 30 * 24 * 60 * 60_000
 export const historyPageStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/history', HISTORY_PAGE_KEPT_MS, debug)
 // The pull requests squashed into a repository's commits, by the commit
-// (repository-history.ts squashedPulls): kept as its history's pages are.
+// (repository-squashes.ts squashedPulls): kept as its history's pages are.
 export const historySquashStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/squashes', HISTORY_PAGE_KEPT_MS, debug)
 // The files each of a repository's commits changes (repository-changes.ts
 // repositoryChanges), by the commit, which never change: kept as its
@@ -151,6 +169,21 @@ export const historyChangeStore = <T>(db: UpstreamCacheStore, debug = false): Ke
 // repositoryTree), by the commit, which never change: kept as its history's
 // pages are.
 export const historyTreeStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/trees', HISTORY_PAGE_KEPT_MS, debug)
+// What the code viewer read of a repository's trees (http.ts
+// repositoryTreesOf), which never changes once a commit or tree names it: the
+// tree at a directory of a commit, by the repository's id, the commit and the
+// directory (`trees`), and how many files a tree holds and their bytes, by
+// the repository's id and the tree's (`tree-files`): kept as its history's
+// pages are.
+export const codeTreeStore = <T>(db: UpstreamCacheStore, type: 'trees' | 'tree-files', debug = false): KeptStore<T> =>
+  keptStore<T>(db, `repository/code-${type}`, HISTORY_PAGE_KEPT_MS, debug)
+// The commit the code viewer keeps a repository's directories at in byte
+// storage, its default branch's head as last read, and the repository's
+// name they are kept under (repository-code.ts repositoryCodeKept), by the
+// repository's id: kept until the head moves, the repository is renamed or
+// it is removed, one for each repository.
+export const codeHeadStore = (db: UpstreamCacheStore, debug = false): KeptStore<{ commit: string; repo: string }> =>
+  keptStore<{ commit: string; repo: string }>(db, 'repository/code-heads', Infinity, debug)
 // The packages a repository's pull request's lockfiles add
 // (pull-dependencies.ts), by the commits of its base and head, which never
 // change: kept as its history's pages are.
@@ -164,3 +197,11 @@ export const pullDependencyStore = <T>(db: UpstreamCacheStore, debug = false): K
 export const npmDeltaStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'npm/deltas', HISTORY_PAGE_KEPT_MS, debug)
 export const reviewBaselineStore = (db: UpstreamCacheStore, debug = false): KeptStore<{ versions: string[]; unread: string[] }> =>
   keptStore<{ versions: string[]; unread: string[] }>(db, 'repository/review-baselines', HISTORY_PAGE_KEPT_MS, debug)
+// What a repository's dependency review read of GitHub (pull-dependencies.ts
+// LockfileStore): the lockfiles a commit lists (`listings`), by the commit,
+// and a lockfile's packages with what their reviews are held to
+// (`lockfiles`), by its blob, which never change: kept as its history's
+// pages are, apart from a scan's (scanReadStore), which reads them without
+// what their reviews are held to.
+export const lockfileReadStore = <T>(db: UpstreamCacheStore, type: 'listings' | 'lockfiles', debug = false): KeptStore<T> =>
+  keptStore<T>(db, `repository/review-${type}`, HISTORY_PAGE_KEPT_MS, debug)

package/server-managed/npm-loads.ts

@@ -9,8 +9,8 @@ import { join, resolve } from 'node:path'
 import { CompressionError, decompress } from '@preventive/archive/compression.js'
 import { encodeBrotli } from './brotli.ts'
 import { keptBriefly } from './kept-briefly.ts'
-import { MAX_NPM_PACKAGE_BYTES, MAX_NPM_PACKAGE_FILES, NPM_REGISTRY, NpmPackageError, type NpmPackageFile, type NpmVersionDocument, askUpstream,
-  npmFileRows, npmPublishTime, npmToken, readNpmTar } from './npm-packages.ts'
+import { MAX_NPM_PACKAGE_BYTES, MAX_NPM_PACKAGE_FILES, NPM_REGISTRY, NpmPackageError, type NpmPackageFile, type NpmVersionDocument, type TarReading,
+  askUpstream, npmFileRows, npmPublishTime, npmToken, readNpmTar } from './npm-packages.ts'
 import { keepFile, readKeptFile, upstreamCachePath } from './upstream-cache.ts'
 
 // The tar stream a package's files may come in, headers and padding included.
@@ -39,9 +39,10 @@ interface Load { job: Promise<unknown>; readers: number; settled: boolean; slot:
 // keeps its place among the active ones until its work is done and each
 // reader has released it, as a reader does once its response is written or
 // abandoned, so what responses still hold counts too. Places by the slot
-// their loads share, with how many hold each.
+// their loads share, with how many hold each and how many places it takes.
 const loads = new Map<string, Load>()
-const slots = new Map<string, number>()
+const slots = new Map<string, { holding: number; places: number }>()
+const placesTaken = () => [...slots.values()].reduce((sum, slot) => sum + slot.places, 0)
 
 const refused = <T>(status: number, code: string): NpmLoad<T> => {
   const result = Promise.reject(new NpmPackageError(status, code))
@@ -49,30 +50,40 @@ const refused = <T>(status: number, code: string): NpmLoad<T> => {
   return { result, release() {} }
 }
 
-// What names a version's tarball, and so its files: its integrity and URL,
-// and whether it is read with the token.
-const tarballKey = (doc: NpmVersionDocument) => `${doc.dist.integrity} ${doc.dist.tarball} ${doc.private}`
+// What names a version's tarball, and so its files: its integrity, its
+// shasum and URL, and whether it is read with the token.
+const tarballKey = (doc: NpmVersionDocument) => `${doc.dist.integrity} ${doc.dist.shasum ?? ''} ${doc.dist.tarball} ${doc.private}`
 
 function shared<T>(what: string, doc: NpmVersionDocument, work: () => Promise<T>): NpmLoad<T> {
   const { unpackedSize, fileCount } = doc.dist
   if ((unpackedSize ?? 0) > MAX_NPM_PACKAGE_BYTES || (fileCount ?? 0) > MAX_NPM_PACKAGE_FILES) return refused(413, 'package-too-large')
   const slot = tarballKey(doc)
-  return sharedLoad(`${what} ${slot}`, work, 'npm-busy', slot)
+  return sharedLoad(`${what} ${slot}`, work, 'npm-busy', { slot })
+}
+
+// The places among the active ones a load takes whose files hold `files` and
+// `bytes`: as many as packages they would fill, up to every one, each place
+// holding what a package may.
+export function loadPlaces({ files, bytes }: { files: number; bytes: number }): number {
+  return Math.min(MAX_ACTIVE_LOADS, Math.max(1, Math.ceil(files / MAX_NPM_PACKAGE_FILES), Math.ceil(bytes / MAX_NPM_PACKAGE_BYTES)))
 }
 
 // A load among the active ones, by `key`, refused as `busy` past them: a
 // repository directory's (repository-code.ts) as a package's, each holding
 // alike a tarball, its tar, its files and their encoded JSON. Loads of one
-// `slot` take one place between them.
-export function sharedLoad<T>(key: string, work: () => Promise<T>, busy: string, slot = key): NpmLoad<T> {
+// `slot` take one place between them, or `places` (loadPlaces) where they
+// hold more.
+export function sharedLoad<T>(key: string, work: () => Promise<T>, busy: string,
+  { slot = key, places = 1 }: { slot?: string; places?: number } = {}): NpmLoad<T> {
   let load = loads.get(key)
   if (!load) {
-    if (!slots.has(slot) && slots.size >= MAX_ACTIVE_LOADS) return refused(429, busy)
+    if (!slots.has(slot) && placesTaken() + places > MAX_ACTIVE_LOADS) return refused(429, busy)
     const created: Load = { job: work(), readers: 0, settled: false, slot }
     const finish = () => { settle(key, created) }
     created.job.then(finish, finish)
     loads.set(key, created)
-    slots.set(slot, (slots.get(slot) ?? 0) + 1)
+    const taken = slots.get(slot)
+    slots.set(slot, { holding: (taken?.holding ?? 0) + 1, places: taken?.places ?? places })
     load = created
   }
   const held = load
@@ -99,15 +110,21 @@ function settle(key: string, load: Load) {
 function done(key: string, load: Load) {
   if (load.readers !== 0 || !load.settled || loads.get(key) !== load) return
   loads.delete(key)
-  const holding = slots.get(load.slot)! - 1
-  if (holding === 0) slots.delete(load.slot)
-  else slots.set(load.slot, holding)
+  const taken = slots.get(load.slot)!
+  if (taken.holding === 1) slots.delete(load.slot)
+  else taken.holding--
 }
 
 // The sha512 an integrity names, in base64, or null where it names none.
 const sha512Of = (integrity: string) => /(?:^|\s)sha512-([\d+/A-Za-z]{86}==)(?=\s|$)/u.exec(integrity)?.[1] ?? null
 // Hashed off the event loop: a tarball runs to tens of MiB.
-const sha512 = async (bytes: Uint8Array) => Buffer.from(await crypto.subtle.digest('SHA-512', bytes as Uint8Array<ArrayBuffer>)).toString('base64')
+const digest = async (algorithm: string, bytes: Uint8Array) => Buffer.from(await crypto.subtle.digest(algorithm, bytes as Uint8Array<ArrayBuffer>))
+// Whether `bytes` are those of the sha512 an integrity names and, where there
+// is one, of the shasum (a sha1, in hex).
+async function hashedAs(bytes: Uint8Array, sha512: string, shasum: string | null): Promise<boolean> {
+  const [own512, own1] = await Promise.all([digest('SHA-512', bytes), shasum === null ? null : digest('SHA-1', bytes)])
+  return own512.toString('base64') === sha512 && (own1 === null || own1.toString('hex') === shasum)
+}
 
 // Upstream's disk cache (setCacheDir), where tarballs are kept between
 // loads, or null to keep none and read none.
@@ -140,23 +157,25 @@ async function downloadTarball(url: string, isPrivate: boolean): Promise<Buffer>
   return bytes
 }
 
-// The version's tarball, its bytes held to the document's sha512 wherever
-// they come from: kept in the cache, where bundle builds keep what they
-// fetch with the token, or in npm's; else downloaded, and then kept. Bytes
-// matching the sha512 the reader's own document names are the tarball that
-// reader may download, wherever they were kept and whoever fetched them.
+// The version's tarball, its bytes held to the document's sha512, and to its
+// shasum where it tells one, wherever they come from: kept in the cache,
+// where bundle builds keep what they fetch with the token, or in npm's; else
+// downloaded, and then kept. Bytes matching the hashes the reader's own
+// document names are the tarball that reader may download, wherever they
+// were kept and whoever fetched them.
 async function readTarball(doc: NpmVersionDocument): Promise<Uint8Array> {
-  const expected = sha512Of(doc.dist.integrity), url = doc.dist.tarball
-  if (!url.startsWith(`${NPM_REGISTRY}/${doc.name}/-/`) || URL.parse(url)?.href !== url || expected === null) throw new NpmPackageError(502, 'upstream-invalid')
+  const expected = sha512Of(doc.dist.integrity), shasum = doc.dist.shasum ?? null, url = doc.dist.tarball
+  if (!url.startsWith(`${NPM_REGISTRY}/${doc.name}/-/`) || URL.parse(url)?.href !== url || expected === null
+    || (shasum !== null && !/^[\da-f]{40}$/u.test(shasum))) throw new NpmPackageError(502, 'upstream-invalid')
   const kept = tarballCache === null ? null : upstreamCachePath(tarballCache, 'npm/tarballs', `${doc.name}@${doc.version}.tgz`)
   if (kept !== null) {
     for (const path of [kept, npmCachePath(expected)]) {
       const bytes = path === null ? null : await readKeptFile(path, MAX_TAR_BYTES)
-      if (bytes && await sha512(bytes) === expected) return bytes
+      if (bytes && await hashedAs(bytes, expected, shasum)) return bytes
     }
   }
   const bytes = await downloadTarball(url, doc.private)
-  if (await sha512(bytes) !== expected) throw new NpmPackageError(502, 'upstream-invalid')
+  if (!await hashedAs(bytes, expected, shasum)) throw new NpmPackageError(502, 'upstream-invalid')
   if (kept !== null) await keepFile(kept, bytes)
   return bytes
 }
@@ -201,10 +220,11 @@ function unpackedOf(doc: NpmVersionDocument): Promise<Unpacked> {
 
 // A gzipped tarball's files, its top directory left out, as npm extracts
 // a package's: a repository tree's as GitHub archives it, too, its symlinks
-// let climb as readNpmTar's `symlinksAbove` says.
-export async function readTarballFiles(tarball: Uint8Array, options: { symlinksAbove?: number } = {}): Promise<NpmPackageFile[]> {
+// let climb as readNpmTar's `symlinksAbove` says, and its tar stream and
+// files bounded as `options` give, a package's bounds unless given.
+export async function readTarballFiles(tarball: Uint8Array, { maxTarBytes = MAX_TAR_BYTES, ...options }: TarReading & { maxTarBytes?: number } = {}): Promise<NpmPackageFile[]> {
   let tar: Uint8Array
-  try { tar = await decompress(tarball, 'gzip', { limit: MAX_TAR_BYTES }) }
+  try { tar = await decompress(tarball, 'gzip', { limit: maxTarBytes }) }
   catch (err) {
     const limited = err instanceof CompressionError && err.limited
     throw new NpmPackageError(limited ? 413 : 422, limited ? 'package-too-large' : 'bad-tarball')

package/server-managed/npm-manifest.ts

@@ -102,7 +102,7 @@ export function npmManifest(json: Record<string, unknown>): Record<string, unkno
 // as they read npm's.
 const VERSION_FIELDS = ['name', 'version', 'description', 'license', 'homepage', 'main', 'module', 'types', 'type', 'deprecated', 'gitHead', 'author', '_npmUser',
   'maintainers', 'keywords', 'bin', 'dependencies', 'peerDependencies', 'optionalDependencies', 'engines', 'hasInstallScript', 'repository', 'bugs']
-const DIST_FIELDS = ['tarball', 'integrity', 'unpackedSize', 'fileCount']
+const DIST_FIELDS = ['tarball', 'integrity', 'shasum', 'unpackedSize', 'fileCount']
 const INSTALL_SCRIPTS = ['preinstall', 'install', 'postinstall']
 const picked = (json: Record<string, unknown>, keys: string[]) => Object.fromEntries(keys.filter(key => Object.hasOwn(json, key)).map(key => [key, json[key]]))
 

package/server-managed/npm-packages.ts

@@ -31,11 +31,12 @@ const PACKUMENT_BYTES = 64 * 1024 * 1024
 // limit until it is read and parsed: four packuments, or 32 version documents.
 const MAX_DOCUMENT_BYTES = 4 * PACKUMENT_BYTES
 // What a package may unpack to: the files' bytes and their count; the tar
-// stream holding them is bounded in npm-loads.ts.
+// stream holding them is bounded in npm-loads.ts. A repository directory's
+// are its own (repository-code.ts).
 export const MAX_NPM_PACKAGE_BYTES = 64 * 1024 * 1024
 export const MAX_NPM_PACKAGE_FILES = 20_000
 // The entries its tar stream may hold, read before any is counted as a file:
-// a directory's too, as a GitHub tree's tarball has one for each, npm's none.
+// a directory's too, though npm packs none.
 export const MAX_NPM_TAR_ENTRIES = 2 * MAX_NPM_PACKAGE_FILES
 // What the files may take as the viewer's JSON. Escaping grows a text up to
 // six times (a control character becomes `\u0001`), so it is counted before
@@ -227,7 +228,9 @@ export interface NpmVersionDocument {
   name: string
   version: string
   private: boolean
-  dist: { tarball: string; integrity: string; unpackedSize: number | null; fileCount: number | null }
+  // `shasum`, the tarball's sha1 in hex, where npm tells it: held, with the
+  // integrity, to the tarball its files are read from (npm-loads.ts).
+  dist: { tarball: string; integrity: string; shasum: string | null; unpackedSize: number | null; fileCount: number | null }
   manifest: Record<string, unknown>
 }
 
@@ -237,10 +240,12 @@ const count = (value: unknown) => Number.isSafeInteger(value) && (value as numbe
 function versionDocument(name: string, json: Record<string, unknown>, isPrivate: boolean): NpmVersionDocument {
   const dist = json['dist']
   if (json['name'] !== name || typeof json['version'] !== 'string' || !isNpmPackageSpec(json['version']) || !plainObject(dist)
-      || typeof dist['tarball'] !== 'string' || typeof dist['integrity'] !== 'string') throw new NpmPackageError(502, 'upstream-invalid')
+      || typeof dist['tarball'] !== 'string' || typeof dist['integrity'] !== 'string'
+      || (dist['shasum'] !== undefined && typeof dist['shasum'] !== 'string')) throw new NpmPackageError(502, 'upstream-invalid')
   return {
     name, version: json['version'], private: isPrivate,
-    dist: { tarball: dist['tarball'], integrity: dist['integrity'], unpackedSize: count(dist['unpackedSize']), fileCount: count(dist['fileCount']) },
+    dist: { tarball: dist['tarball'], integrity: dist['integrity'], shasum: typeof dist['shasum'] === 'string' ? dist['shasum'] : null,
+      unpackedSize: count(dist['unpackedSize']), fileCount: count(dist['fileCount']) },
     manifest: npmManifest(json),
   }
 }
@@ -261,13 +266,18 @@ export interface NpmVersionStore {
 // changes a version's files, but its document can change: a deprecation, the
 // package made private (a scoped one can be), the version removed (which whoever
 // reads it from a store learns elsewhere too: risk does). A day bounds each.
+// One kept with no shasum, as none was before it was kept (npmVersionFields),
+// is asked of npm again, which tells one of each version.
 export const VERSION_KEPT_MS = 24 * 60 * 60_000
 export function readNpmVersion(name: string, spec: string, privileged: boolean, signal: AbortSignal, store?: NpmVersionStore): Promise<NpmVersionDocument | null> {
   if (!isNpmPackageName(name) || !isNpmPackageSpec(spec)) return Promise.reject(new NpmPackageError(400, 'bad-package'))
   // A kept one is read within the same budget as npm's answer.
   return withDocumentBudget(VERSION_DOCUMENT_BYTES, async () => {
     const kept = store && isExactVersion(spec) ? await store.read(name, spec) : null
-    if (kept && Date.now() - kept.at < VERSION_KEPT_MS) try { return versionDocument(name, kept.json, false) } catch {}
+    const keptDist = kept?.json['dist']
+    if (kept && Date.now() - kept.at < VERSION_KEPT_MS && plainObject(keptDist) && typeof keptDist['shasum'] === 'string') {
+      try { return versionDocument(name, kept.json, false) } catch {}
+    }
     const found = await readDocument(name, registryUrl(name, spec), privileged, { accept: 'application/json', limit: VERSION_DOCUMENT_BYTES, signal })
     if (!found) return null
     const doc = versionDocument(name, found.json, found.private)
@@ -367,18 +377,23 @@ export function npmPublishTime(doc: Pick<NpmVersionDocument, 'name' | 'version'
 
 export interface NpmPackageFile { path: string; bytes: Uint8Array }
 
+// How a tar stream is read (readNpmTar): how far its symlinks may climb above
+// it, and the entries, files and bytes it may hold, a package's unless given.
+export interface TarReading { symlinksAbove?: number; maxEntries?: number; maxFiles?: number; maxBytes?: number }
+
 // The regular files of a tar stream, by package path: npm drops the first
 // component (`package/`, or whatever the archive used), and a file at the top
 // is not extracted. The stream is read by @preventive/archive, which refuses
 // one malformed, a name or a symlink that would climb out of it -- a symlink
 // past `symlinksAbove` levels above it, none unless given -- and a name
-// repeated as another entry, and gives up past MAX_NPM_TAR_ENTRIES before
-// reading on; a file under another top directory at the same path replaces
-// an earlier one, as extraction would. Directories, links and devices hold no
+// repeated as another entry, and gives up past `maxEntries` before reading
+// on; a file under another top directory at the same path replaces an
+// earlier one, as extraction would. Directories, links and devices hold no
 // file content and are skipped.
-export function readNpmTar(tar: Uint8Array, { symlinksAbove = 0 }: { symlinksAbove?: number } = {}): NpmPackageFile[] {
+export function readNpmTar(tar: Uint8Array, { symlinksAbove = 0, maxEntries = MAX_NPM_TAR_ENTRIES, maxFiles = MAX_NPM_PACKAGE_FILES,
+  maxBytes = MAX_NPM_PACKAGE_BYTES }: TarReading = {}): NpmPackageFile[] {
   let entries: Entry[]
-  try { entries = unpack(tar, { maxEntries: MAX_NPM_TAR_ENTRIES, symlinksAbove }) }
+  try { entries = unpack(tar, { maxEntries, symlinksAbove }) }
   catch (err) {
     if (err instanceof ArchiveError) throw err.limited ? new NpmPackageError(413, 'package-too-large') : new NpmPackageError(422, 'bad-tarball')
     throw err
@@ -391,7 +406,7 @@ export function readNpmTar(tar: Uint8Array, { symlinksAbove = 0 }: { symlinksAbo
     const path = name.slice(slash + 1)
     total += data.length - (files.get(path)?.length ?? 0)
     files.set(path, data)
-    if (files.size > MAX_NPM_PACKAGE_FILES || total > MAX_NPM_PACKAGE_BYTES) throw new NpmPackageError(413, 'package-too-large')
+    if (files.size > maxFiles || total > maxBytes) throw new NpmPackageError(413, 'package-too-large')
   }
   return [...files].map(([path, bytes]) => ({ path, bytes })).toSorted((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0)
 }
@@ -421,13 +436,14 @@ const ROW_LENGTH = 80
 
 // The files as the viewer gets them: `[path, bytes, text]`, text null for a
 // file that is not UTF-8, which instead carries its bytes' sha256, so a
-// comparison of two versions tells a changed one from an unchanged one.
-export function npmFileRows(files: NpmPackageFile[]): ([string, number, string] | [string, number, null, string])[] {
+// comparison of two versions tells a changed one from an unchanged one. They
+// may take `maxLength` characters of JSON, a package's unless given.
+export function npmFileRows(files: NpmPackageFile[], maxLength = MAX_NPM_JSON_LENGTH): ([string, number, string] | [string, number, null, string])[] {
   let length = 0
   return files.map(({ path, bytes }) => {
     const text = npmFileText(bytes)
     length += ROW_LENGTH + jsonLength(path) + (text === null ? 0 : jsonLength(text))
-    if (length > MAX_NPM_JSON_LENGTH) throw new NpmPackageError(413, 'package-too-large')
+    if (length > maxLength) throw new NpmPackageError(413, 'package-too-large')
     return text === null ? [path, bytes.byteLength, null, `sha256-${createHash('sha256').update(bytes).digest('base64')}`] : [path, bytes.byteLength, text]
   })
 }

package/server-managed/npm-risk.ts

@@ -14,8 +14,15 @@
 //   instance, or this one restarted, asks npm's downloads API, which answers
 //   some 40 lookups a minute, none of them again that week, each removed a
 //   day past it (npm-kept.ts pruneKept); a private one's in memory.
-// - `risk/npm/packages`, `risk/npm/public` and `risk/npm/advisories`, kept an
-//   hour or a day by risk itself: in memory.
+// - `risk/npm/packages`, `risk/npm/old-versions`, `risk/npm/public` and
+//   `risk/npm/advisories`, which risk asks again after an hour: a public
+//   package's in memory and in the database too, read from memory first, so
+//   that a cold instance, as a serverless one starts, reads no whole document
+//   of npm's again that one before it read within the hour; by a hash of
+//   their type and key, as a dependency's spec runs to any length, each
+//   removed a day on, and one past 256 KiB as JSON (a package's deprecated
+//   versions, its messages and all, run long) in memory alone. A private
+//   one's in memory.
 // - `npm/tarballs`: none asked, as tarballs are read as the viewer's loads
 //   read them, kept where they keep them (npm-loads.ts loadNpmTarball), and
 //   an assessment kept needs none.
@@ -69,11 +76,20 @@ function remember(key: string, value: unknown) {
   while (memory.size > MEMORY_ENTRIES || memoryLength > MEMORY_LENGTH) forget(memory.keys().next().value!)
 }
 
-const hashed = (type: string, key: string) => `risk/${createHash('sha256').update(`${type}\0${key}`).digest('hex')}`
-// What of a public package is kept in the database, and for how long its
-// downloads are: risk's week, and a day more.
+const digest = (type: string, key: string) => createHash('sha256').update(`${type}\0${key}`).digest('hex')
+const hashed = (type: string, key: string) => `risk/${digest(type, key)}`
+// What of a public package is kept in the database by its key, and for how
+// long its downloads are: risk's week, and a day more.
 const IN_DATABASE = new Set(['risk/npm/versions', 'risk/npm/downloads'])
 const DOWNLOADS_KEPT_MS = 8 * 24 * 60 * 60_000
+// What of a public package risk asks again after an hour, kept in memory and
+// in the database (HOURLY_PREFIX), each a day, none past HOURLY_LENGTH as
+// JSON there.
+const HOURLY = new Set(['risk/npm/packages', 'risk/npm/old-versions', 'risk/npm/public', 'risk/npm/advisories'])
+const HOURLY_PREFIX = 'risk/hourly/'
+const HOURLY_KEPT_MS = 24 * 60 * 60_000
+const HOURLY_LENGTH = 256 * 1024
+const databaseKey = (type: string, key: string) => HOURLY.has(type) ? `${HOURLY_PREFIX}${digest(type, key)}` : `${type}/${key}`
 
 export interface NpmRiskStores {
   db: UpstreamCacheStore
@@ -82,10 +98,13 @@ export interface NpmRiskStores {
 }
 
 // A cache that reads and writes many of a type at once too, as risk's
-// readying of downloads asks (npmReadyDownloads): one read and one write of
-// the database each, not one a key, as a request's queries take turns on its
-// connection (db-neon.ts).
+// readying of downloads asks (npmReadyDownloads), and reads records of any
+// types at once, as risk reads a package's record and its versions'
+// (registry.js npmPackageFacts): one read and one write of the database each,
+// not one a key, as a request's queries take turns on its connection
+// (db-neon.ts).
 export interface NpmRiskCache extends CacheStore {
+  readEach(entries: [type: string, key: string][]): Promise<unknown[]>
   readMany(type: string, keys: string[]): Promise<Map<string, unknown>>
   writeMany(type: string, entries: [key: string, value: unknown][]): Promise<void>
 }
@@ -94,13 +113,32 @@ export interface NpmRiskCache extends CacheStore {
 // that what is kept of it can be shared.
 export function npmRiskCache({ db, npmCache, debug }: NpmRiskStores, isPublic: boolean): NpmRiskCache {
   const warn = (what: string, error: unknown) => { if (debug) console.warn(`managed: risk cache ${what} failed:`, error) }
+  const memoryKey = (type: string, key: string) => `${isPublic ? '' : 'private\0'}${type}\0${key}`
+  // Whether a record of `type` is read of the database: unless it is one
+  // kept in memory too (HOURLY) and there.
+  const ofDatabase = (type: string, key: string) => isPublic && (IN_DATABASE.has(type) || HOURLY.has(type) && !memory.has(memoryKey(type, key)))
+  // A record as the database keeps it, read: an hourly one kept in memory
+  // too, for the next to ask.
+  const readOf = (type: string, key: string, row: string | null | undefined) => {
+    const value: unknown = row == null ? null : JSON.parse(row)
+    if (value !== null && HOURLY.has(type)) remember(memoryKey(type, key), value)
+    return value
+  }
   const cache: NpmRiskCache = {
+    async readEach(entries) {
+      const asked = new Set(entries.flatMap(([type, key], i) => ofDatabase(type, key) ? [i] : []))
+      let rows = new Map<string, string>()
+      if (asked.size > 0) {
+        try { rows = await db.getUpstreamCacheEntries([...asked].map(i => databaseKey(...entries[i]!))) } catch (error) { warn('read', error) }
+      }
+      return Promise.all(entries.map(([type, key], i) => {
+        if (!asked.has(i)) return cache.read(type, key)
+        try { return readOf(type, key, rows.get(databaseKey(type, key))) } catch (error) { warn('read', error); return null }
+      }))
+    },
     async readMany(type, keys) {
-      if (!isPublic || !IN_DATABASE.has(type)) return new Map(await Promise.all(keys.map(async key => [key, await cache.read(type, key)] as const)))
-      try {
-        const found = await db.getUpstreamCacheEntries(keys.map(key => `${type}/${key}`))
-        return new Map(keys.map(key => { const value = found.get(`${type}/${key}`); return [key, value === undefined ? null : JSON.parse(value)] }))
-      } catch (error) { warn('read', error); return new Map() }
+      const values = await cache.readEach(keys.map(key => [type, key]))
+      return new Map(keys.map((key, i) => [key, values[i]]))
     },
     async writeMany(type, entries) {
       if (!isPublic || !IN_DATABASE.has(type)) { for (const [key, value] of entries) await cache.write(type, key, value); return }
@@ -112,22 +150,31 @@ export function npmRiskCache({ db, npmCache, debug }: NpmRiskStores, isPublic: b
       if (type === 'npm/tarballs') return null
       try {
         if (isPublic && type === 'risk/npm/assessments' && npmCache) return JSON.parse((await buffer((await npmCache.open(hashed(type, key))).stream)).toString('utf8'))
-        if (isPublic && IN_DATABASE.has(type)) { const value = await db.getUpstreamCacheEntry(`${type}/${key}`); return value === null ? null : JSON.parse(value) }
+        if (ofDatabase(type, key)) return readOf(type, key, await db.getUpstreamCacheEntry(databaseKey(type, key)))
       } catch (error) { if (!(error instanceof CacheMissError)) warn('read', error); return null }
-      return memory.get(`${isPublic ? '' : 'private\0'}${type}\0${key}`)?.value ?? null
+      return memory.get(memoryKey(type, key))?.value ?? null
     },
     async write(type, key, value) {
       if (type === 'npm/tarballs') return
+      if (isPublic && HOURLY.has(type)) remember(memoryKey(type, key), value)
       try {
         if (isPublic && type === 'risk/npm/assessments' && npmCache) { await npmCache.put(hashed(type, key), Buffer.from(JSON.stringify(value))); return }
         if (isPublic && IN_DATABASE.has(type)) {
           const at = Date.now()
-          await db.setUpstreamCacheEntry(`${type}/${key}`, JSON.stringify(value), at)
+          await db.setUpstreamCacheEntry(databaseKey(type, key), JSON.stringify(value), at)
           if (type === 'risk/npm/downloads') await pruneKept(db, `${type}/`, DOWNLOADS_KEPT_MS, at, error => { warn('prune', error) })
           return
         }
+        if (isPublic && HOURLY.has(type)) {
+          const json = JSON.stringify(value)
+          if (json.length > HOURLY_LENGTH) return
+          const at = Date.now()
+          await db.setUpstreamCacheEntry(databaseKey(type, key), json, at)
+          await pruneKept(db, HOURLY_PREFIX, HOURLY_KEPT_MS, at, error => { warn('prune', error) })
+          return
+        }
       } catch (error) { warn('write', error); return }
-      remember(`${isPublic ? '' : 'private\0'}${type}\0${key}`, value)
+      remember(memoryKey(type, key), value)
     },
   }
   return cache

package/server-managed/object-storage.ts

@@ -30,7 +30,8 @@ export interface ObjectStorage {
   exists(key: string): Promise<boolean>
   put(key: string, bytes: Buffer, options?: { replaceReport?: boolean }): Promise<string | null>
   delete(key: string): Promise<void>
-  deletePrefix(prefix: string): Promise<void>
+  // Every object under `prefix`, but those under `keep`, a prefix within it.
+  deletePrefix(prefix: string, keep?: string): Promise<void>
 }
 
 export function isBlobId(value: string): boolean {
@@ -52,14 +53,14 @@ export async function openObjectVersion(raw: RawObjectStorage, key: string, vers
   return null
 }
 
-export async function deleteObjects(raw: RawObjectStorage, prefix: string, signal: AbortSignal, before?: number): Promise<number> {
+export async function deleteObjects(raw: RawObjectStorage, prefix: string, signal: AbortSignal, before?: number, keep?: string): Promise<number> {
   const eligible = (modifiedAt: number) => before === undefined || modifiedAt < before
   // Finish listing before deleting: provider cursors may be offset-based.
   const cursors = new Set<string>(), keys = new Set<string>()
   let cursor: string | null = null
   do {
     const page = await raw.list(prefix, cursor, 100, signal)
-    for (const object of page.objects) if (eligible(object.modifiedAt)) keys.add(object.key)
+    for (const object of page.objects) if (eligible(object.modifiedAt) && (keep === undefined || !object.key.startsWith(keep))) keys.add(object.key)
     cursor = page.cursor
     if (cursor !== null && cursors.has(cursor)) throw new Error('Invalid blob pagination')
     if (cursor !== null) cursors.add(cursor)