@preventive/upstream 1.0.0-alpha.9 → 1.0.1

package/src/advisories/github.js

@@ -1,13 +1,21 @@
 import assert from 'node:assert/strict'
 
 import { assertRepo, assertSoldeerName, assertSoldeerVersion, assertion, isGhsa, isRefName, isStrings } from '../args.js'
+import { readRecord, writeRecord } from '../cache.js'
 import { isGone } from '../github/client.js'
 import { recover } from '../http.js'
 import { pool } from '../pool.js'
+import { isExactVersion } from '../semver.js'
 import { advisoryUrl, covered, detailsOf, isText, mayBeInRange, metrics } from './common.js'
 import { soldeerRepos } from './repos.js'
 
 const REPOS_AT_ONCE = 4
+const DIR = 'github/advisories'
+// A maintainer publishes one whenever they are ready, and seeing it before
+// GitHub reviews it is what a repository is asked for: 90 minutes, not a month.
+const LISTING_TTL_MS = 90 * 60 * 1000
+// Stamped on each entry, and raised when a listing is kept differently.
+const VERSION = 2
 
 const isRepoAdvisory = (advisory) => advisory && typeof advisory === 'object' && isGhsa(advisory.ghsa_id)
   && advisory.state === 'published' && isText(advisory.summary)
@@ -15,7 +23,63 @@ const isRepoAdvisory = (advisory) => advisory && typeof advisory === 'object' &&
 
 export const assertClient = assertion('a GitHub client from createClient', (value) => typeof value?.listRepoAdvisories === 'function')
 
-const listAdvisories = (github, repo) => github.listRepoAdvisories({ repo }).catch(recover(isGone, []))
+// A maintainer may give a vulnerable range only its lower bound, and the
+// version that fixed it apart (`>= 5.0.0-beta.1`, patched in `5.0.0-rc.2`),
+// as GitHub's page shows them side by side: such a range ends below its one
+// patched version. A range bounded above, or patched in more than one
+// version, is as given.
+function boundedRange(range, patched) {
+  const fixed = typeof patched === 'string' ? patched.trim() : null
+  if (!isExactVersion(fixed) || range.includes('<')) return range
+  return range.trim() === '' ? `< ${fixed}` : `${range.trim()}, < ${fixed}`
+}
+
+// What rows are made from, and all that is kept of a listing: each advisory
+// not withdrawn, with its CVSS vector GitHub prefers, and each vulnerable
+// range with the package it names, ended below its patched version where it
+// has no end of its own (boundedRange). A listing that is malformed is
+// refused whole, and never kept; one whose only fault is a description
+// (isDigest) is answered, but not kept either.
+function digest(repo, list) {
+  return list.flatMap((advisory) => {
+    assert.ok(isRepoAdvisory(advisory), `advisories: malformed advisory from ${repo}`)
+    if (advisory.withdrawn_at) return []
+    const cvss = [advisory.cvss_severities?.cvss_v3, advisory.cvss_severities?.cvss_v4, advisory.cvss].find((entry) => typeof entry?.vector_string === 'string')
+    const ranges = (advisory.vulnerabilities ?? []).map((vulnerability) => {
+      const range = vulnerability?.vulnerable_version_range ?? ''
+      assert.ok(typeof range === 'string', `advisories: malformed range in ${advisory.ghsa_id}`)
+      const { ecosystem, name } = vulnerability?.package ?? {}
+      return { range: boundedRange(range, vulnerability?.patched_versions), ...(typeof ecosystem === 'string' && { ecosystem }), ...(typeof name === 'string' && { name }) }
+    })
+    return [{
+      ghsa: advisory.ghsa_id,
+      title: advisory.summary,
+      ...(advisory.description != null && advisory.description !== '' && { description: advisory.description }),
+      ...metrics({ severity: advisory.severity, score: cvss?.score, vector: cvss?.vector_string, cwe: advisory.cwe_ids }),
+      ranges,
+    }]
+  })
+}
+
+// A description is checked only for `details` (detailsOf), so a malformed
+// one leaves the listing usable, but not kept.
+const isDigest = (advisory) => advisory && typeof advisory === 'object' && isGhsa(advisory.ghsa) && isText(advisory.title)
+  && (advisory.description === undefined || isText(advisory.description))
+  && Array.isArray(advisory.ranges) && advisory.ranges.every((entry) => typeof entry?.range === 'string')
+
+// A repository's listing, digested, through the cache: GitHub's names are
+// case-insensitive, so one entry answers every spelling. A repository gone,
+// renamed or blocked has none, and that is not kept.
+async function listAdvisories(github, repo, cache) {
+  const name = repo.toLowerCase()
+  const entry = await readRecord(DIR, name, cache, LISTING_TTL_MS)
+  if (entry?.v === VERSION && Array.isArray(entry.advisories) && entry.advisories.every(isDigest)) return entry.advisories
+  const list = await github.listRepoAdvisories({ repo }).catch(recover(isGone, null))
+  if (list === null) return []
+  const advisories = digest(repo, list)
+  if (advisories.every(isDigest)) await writeRecord(DIR, name, { v: VERSION, advisories }, cache)
+  return advisories
+}
 
 // GitHub's `>= 1.0.0, < 1.2.6` is npm's with the commas dropped.
 // Maintainers write these unreviewed: one without a range, or with one
@@ -23,17 +87,16 @@ const listAdvisories = (github, repo) => github.listRepoAdvisories({ repo }).cat
 // may publish it long before the advisory database has one, and its
 // `description` is the text `details` asks for: the listing has it all.
 function fromRepository(repo, name, advisory, range, asked, { covers, details }) {
-  const cvss = [advisory.cvss_severities?.cvss_v3, advisory.cvss_severities?.cvss_v4, advisory.cvss].find((entry) => typeof entry?.vector_string === 'string')
   return {
     name,
     source: 'repository',
-    id: advisory.ghsa_id,
-    ghsa: advisory.ghsa_id,
-    url: advisoryUrl(advisory.ghsa_id, repo),
+    id: advisory.ghsa,
+    ghsa: advisory.ghsa,
+    url: advisoryUrl(advisory.ghsa, repo),
     aliases: [],
-    title: advisory.summary,
-    ...(details && detailsOf(advisory.description, advisory.ghsa_id)),
-    ...metrics({ severity: advisory.severity, score: cvss?.score, vector: cvss?.vector_string, cwe: advisory.cwe_ids }),
+    title: advisory.title,
+    ...(details && detailsOf(advisory.description, advisory.ghsa)),
+    ...metrics({ severity: advisory.severity, score: advisory.cvss, vector: advisory.cvssVector, cwe: advisory.cwe }),
     range,
     versions: covered(asked, range.replaceAll(',', ' '), covers),
   }
@@ -41,21 +104,18 @@ function fromRepository(repo, name, advisory, range, asked, { covers, details })
 
 // Each asked name's repository, `repoOf` it, is asked once, and its
 // advisories' vulnerable ranges become rows for the names that `takes` a
-// vulnerability, one per name, advisory and range, holding the asked
+// range's package, one per name, advisory and range, holding the asked
 // versions it `covers`, and with `details`, its text.
-async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers, details }) {
+async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers, details, cache }) {
   // GitHub's names are case-insensitive: one spelling asks for all.
   const namesOf = Map.groupBy([...asked.keys()].filter(repoOf), (name) => repoOf(name).toLowerCase())
-  const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ repo: repoOf(names[0]), names, list: await listAdvisories(github, repoOf(names[0])) }))
+  const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ repo: repoOf(names[0]), names, list: await listAdvisories(github, repoOf(names[0]), cache) }))
   const rows = new Map()
   for (const { repo, names, list } of listed) {
     for (const advisory of list) {
-      assert.ok(isRepoAdvisory(advisory), `advisories: malformed advisory from ${repo}`)
-      for (const vulnerability of advisory.withdrawn_at ? [] : advisory.vulnerabilities ?? []) {
-        const range = vulnerability?.vulnerable_version_range ?? ''
-        assert.ok(typeof range === 'string', `advisories: malformed range in ${advisory.ghsa_id}`)
-        for (const name of names.filter((candidate) => takes(candidate, vulnerability?.package))) {
-          rows.set(`${name} ${advisory.ghsa_id} ${range}`, fromRepository(repo, name, advisory, range, asked.get(name), { covers, details }))
+      for (const { range, ...pkg } of advisory.ranges) {
+        for (const name of names.filter((candidate) => takes(candidate, pkg))) {
+          rows.set(`${name} ${advisory.ghsa} ${range}`, fromRepository(repo, name, advisory, range, asked.get(name), { covers, details }))
         }
       }
     }
@@ -63,9 +123,10 @@ async function repositoryAdvisories(github, asked, { repoOf, takes = () => true,
   return [...rows.values()]
 }
 
-// Each asked name's repository: as given, `known`, or else looked up.
-async function reposOf(asked, known, lookUp) {
-  const found = await lookUp([...asked.keys()].filter((name) => !known.has(name)))
+// Each asked name's repository: as given, `known`, or else looked up,
+// through `cache`.
+async function reposOf(asked, known, lookUp, cache) {
+  const found = await lookUp([...asked.keys()].filter((name) => !known.has(name)), { asked, cache })
   return (name) => known.get(name) ?? found.get(name)
 }
 
@@ -74,12 +135,12 @@ async function reposOf(asked, known, lookUp) {
 // each package's repository, `known` or else looked up, adds what it
 // publishes for GitHub's `ecosystem` entries naming the package, for the
 // versions `rows` do not already report under that GHSA.
-export async function withRepositories(rows, asked, { github, repoAdvisories, known, details }, { ecosystem, lookUp, covers }) {
+export async function withRepositories(rows, asked, { github, repoAdvisories, known, details, cache }, { ecosystem, lookUp, covers }) {
   if (!repoAdvisories) return rows
   const reported = new Set(rows.flatMap((row) => [row.id, row.ghsa, ...row.aliases].filter(isGhsa).flatMap((id) => row.versions.map((version) => `${row.name} ${id} ${version}`))))
-  const repoOf = await reposOf(asked, known, lookUp)
-  const takes = (name, pkg) => pkg?.ecosystem === ecosystem && pkg.name === name
-  const added = await repositoryAdvisories(github, asked, { repoOf, takes, covers, details })
+  const repoOf = await reposOf(asked, known, lookUp, cache)
+  const takes = (name, pkg) => pkg.ecosystem === ecosystem && pkg.name === name
+  const added = await repositoryAdvisories(github, asked, { repoOf, takes, covers, details, cache })
   return [...rows, ...added.map((row) => ({ ...row, versions: row.versions.filter((version) => !reported.has(`${row.name} ${row.id} ${version}`)) }))]
 }
 
@@ -93,7 +154,7 @@ export const GITHUB = {
   repositoryOnly: true,
   assertName: assertRepo,
   assertVersion: assertion('a version or a branch name', isRefName),
-  advisories: (asked, { github, details }) => repositoryAdvisories(github, asked, { repoOf: (repo) => repo, covers: coversPlaceholder, details }),
+  advisories: (asked, { github, details, cache }) => repositoryAdvisories(github, asked, { repoOf: (repo) => repo, covers: coversPlaceholder, details, cache }),
 }
 
 // Soldeer packages, which no advisory database has: the repository each
@@ -104,5 +165,5 @@ export const SOLDEER = {
   repositoryOnly: true,
   assertName: assertSoldeerName,
   assertVersion: assertSoldeerVersion,
-  advisories: async (asked, { github, known, details }) => await repositoryAdvisories(github, asked, { repoOf: await reposOf(asked, known, soldeerRepos), details }),
+  advisories: async (asked, { github, known, details, cache }) => await repositoryAdvisories(github, asked, { repoOf: await reposOf(asked, known, soldeerRepos, cache), details, cache }),
 }

package/src/advisories/index.js

@@ -1,6 +1,7 @@
 import assert from 'node:assert/strict'
 
 import { assertArgs, assertBoolean, assertRepo, assertion, isIterable, optional, sameName } from '../args.js'
+import { assertCache } from '../cache.js'
 import { order } from './common.js'
 import { GITHUB, SOLDEER, assertClient } from './github.js'
 import { NPM } from './npm.js'
@@ -11,7 +12,7 @@ const byNumbers = new Intl.Collator('en', { numeric: true }).compare
 const assertEcosystem = assertion(`one of ${Object.keys(ECOSYSTEMS).join(', ')}`, (value) => typeof value === 'string' && Object.hasOwn(ECOSYSTEMS, value))
 const assertVersions = assertion('a non-empty array', (value) => Array.isArray(value) && value.length > 0)
 
-// Ecosystem → name → { versions, github? }, a name given twice merged.
+// Ecosystem -> name -> { versions, github? }, a name given twice merged.
 // Every package is checked before any is used.
 function collect(packages) {
   assert.ok(isIterable(packages), 'advisories: packages must be an iterable of { ecosystem, name, github?, versions }')
@@ -34,7 +35,7 @@ function collect(packages) {
 }
 
 export async function advisories(packages, options = {}) {
-  assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean) })
+  assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean), cache: optional(assertCache) })
   assert.ok(!options.repoAdvisories || options.github, 'advisories: repoAdvisories needs a github client')
   const byEcosystem = collect(packages)
   for (const ecosystem of byEcosystem.keys()) assert.ok(!ECOSYSTEMS[ecosystem].repositoryOnly || options.github, `advisories: ${ecosystem} packages need a github client`)
@@ -43,7 +44,7 @@ export async function advisories(packages, options = {}) {
     const names = [...named.keys()].toSorted()
     const asked = new Map(names.map((name) => [name, [...named.get(name).versions].toSorted(compare)]))
     const known = new Map(names.filter((name) => named.get(name).github).map((name) => [name, named.get(name).github]))
-    const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, known })
+    const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, known, cache: options.cache })
     return rows.map((row) => ({ ecosystem, ...row }))
   }))
   return found.flat().filter((row) => row.versions.length > 0).toSorted((a, b) => order(a.ecosystem, b.ecosystem) || order(a.name, b.name))

package/src/advisories/osv.js

@@ -13,12 +13,12 @@ const QUERIES_PER_REQUEST = 1000
 export const RECORDS_AT_ONCE = 8
 const isOsvId = matches(/^(?=.{1,128}$)[A-Z][\dA-Z]*(?:-[\dA-Za-z]+)+$/u)
 // osv.dev's page for one of its records. An id that is only another's
-// alias (PKSA-…) has none, and redirects to a search.
+// alias (PKSA-...) has none, and redirects to a search.
 const osvPage = (id) => `https://osv.dev/vulnerability/${id}`
 const INFORMATIONAL = new Set(['unmaintained', 'unsound', 'notice'])
 // Of the records that publish one advisory, the first answered for a
 // version stands for the rest on it: RustSec's, what `cargo audit` reads
-// and GitHub mirrors, then the GHSA, then any other (MAL-…), then by id.
+// and GitHub mirrors, then the GHSA, then any other (MAL-...), then by id.
 const rank = (id) => (id.startsWith('RUSTSEC-') ? 0 : (isGhsa(id) ? 1 : 2))
 const first = (a, b) => rank(a.id) - rank(b.id) || order(a.id, b.id)
 const named = (a, b) => a.aliases.includes(b.id) || b.aliases.includes(a.id)
@@ -88,7 +88,7 @@ function toAdvisory(ecosystem, name, versions, record, details) {
 async function osvAdvisories(ecosystem, asked, options) {
   const { osv, github, lookUp, covers } = ecosystem
   const list = [...asked].flatMap(([name, versions]) => versions.map((version) => ({ name, version })))
-  const hits = new Map() // id → name → versions, in `list` order
+  const hits = new Map() // id -> name -> versions, in `list` order
   for (const chunk of chunks(list, QUERIES_PER_REQUEST)) {
     const body = { queries: chunk.map(({ name, version }) => ({ package: { name, ecosystem: osv }, version })) }
     const answer = await request(QUERY_URL, { method: 'POST', body, as: 'json' })

package/src/advisories/repos.js

@@ -4,18 +4,19 @@ import { show } from '../args.js'
 import { addRepos, readRepos } from '../cache.js'
 import { lookUpCrateRepos } from '../cargo/repos.js'
 import { PACKAGIST_REPO, SOLDEER_API, buildUrl, isNotFound, recover, request } from '../http.js'
-import { lookUpPackageRepo } from '../npm/repos.js'
+import { getVersionDocument } from '../npm/versions.js'
+import { getRepo } from '../package.js'
 import { pool } from '../pool.js'
 import { githubRepoOfUrl } from '../remote.js'
 
 const PACKAGES_AT_ONCE = 8
 
-// Name → its GitHub repo, or null where the registry has no such package
+// Name -> its GitHub repo, or null where the registry has no such package
 // or it names no GitHub repo; any other failure throws. Only a repo found
-// is cached, for a month.
-async function cachedRepos(dir, names, fetchMissing) {
-  const repos = await readRepos(dir, names)
-  await addRepos(dir, repos, await fetchMissing(names.filter((name) => !repos.has(name))))
+// is cached, for a month, in the cache set or `cache` (readRecord).
+async function cachedRepos(dir, names, fetchMissing, cache) {
+  const repos = await readRepos(dir, names, cache)
+  await addRepos(dir, repos, await fetchMissing(names.filter((name) => !repos.has(name))), cache)
   return repos
 }
 
@@ -42,7 +43,17 @@ async function fetchSoldeerRepo(name) {
   return githubRepoOfUrl(answer.data[0]?.github_url) ?? null
 }
 
-export const crateRepos = (names) => lookUpCrateRepos('advisories', names)
-export const composerRepos = (names) => cachedRepos('composer/repos', names, (missing) => fetchEach(missing, fetchComposerRepo))
-export const npmRepos = (names) => fetchEach(names, async (name) => (await lookUpPackageRepo(name))?.github ?? null)
-export const soldeerRepos = (names) => cachedRepos('soldeer/repos', names, (missing) => fetchEach(missing, fetchSoldeerRepo))
+// The repo the newest version asked of each package names, from that
+// version's document, kept for good (getVersionDocument): null where the
+// registry does not have that version, or it names no GitHub repo.
+async function npmRepo(name, version, cache) {
+  const json = await getVersionDocument('advisories', name, version, cache)
+  return (json && getRepo(json).github) ?? null
+}
+
+// Each takes the names to look up, the versions `asked` of each, and the
+// call's `cache`.
+export const crateRepos = (names, { cache }) => lookUpCrateRepos('advisories', names, { cache })
+export const composerRepos = (names, { cache }) => cachedRepos('composer/repos', names, (missing) => fetchEach(missing, fetchComposerRepo), cache)
+export const npmRepos = (names, { asked, cache }) => fetchEach(names, (name) => npmRepo(name, asked.get(name).at(-1), cache))
+export const soldeerRepos = (names, { cache }) => cachedRepos('soldeer/repos', names, (missing) => fetchEach(missing, fetchSoldeerRepo), cache)

package/src/args.js

@@ -7,7 +7,7 @@ export const matches = (regex) => (value) => typeof value === 'string' && regex.
 // Bidi controls can make a log line read as something it doesn't say.
 const isBidi = (char) => (char >= '\u202A' && char <= '\u202E') || (char >= '\u2066' && char <= '\u2069')
 // A prototype's properties would be read as options without being checked.
-const isPlainObject = (value) => value != null && [Object.prototype, null].includes(Object.getPrototypeOf(value))
+export const isPlainObject = (value) => value != null && [Object.prototype, null].includes(Object.getPrototypeOf(value))
 // Lone surrogates make encodeURIComponent throw.
 const isString = (value, max, allowed = '') => typeof value === 'string' && value.length <= max && value.isWellFormed()
   && ![...value].some((char) => isControl(char) && !allowed.includes(char))

package/src/cache.js

@@ -3,12 +3,22 @@ import { constants } from 'node:fs'
 import { mkdir, open, rename, rm, writeFile } from 'node:fs/promises'
 import { homedir } from 'node:os'
 import { dirname, isAbsolute, join, resolve } from 'node:path'
+import { promisify } from 'node:util'
+import { gunzip, gzip } from 'node:zlib'
 
-import { assertCacheName, assertDirectoryPath, isRepo } from './args.js'
+import { assertCacheName, assertDirectoryPath, assertion, isRepo } from './args.js'
 import { MAX_BYTES, decode } from './http.js'
+import { pool } from './pool.js'
 
-const DIRS = new Set(['npm/repos', 'npm/tarballs', 'cargo/repos', 'cargo/crates', 'composer/repos', 'soldeer/repos', 'soldeer/zips', 'github/trees'])
+const DIRS = new Set(['npm/repos', 'npm/tarballs', 'npm/versions', 'cargo/repos', 'cargo/crates', 'composer/repos', 'soldeer/repos', 'soldeer/zips', 'github/trees', 'github/advisories'])
 const RECORD_TTL_MS = 30 * 24 * 60 * 60 * 1000 // A link only moves on a transfer or rename, and GitHub redirects those.
+const RECORDS_AT_ONCE = 8
+// JSON filed gzipped, as `<name>.json.gz`: a listing carries each
+// advisory's full text, Markdown that compresses well, and a version
+// document all npm has of that version.
+const GZIP_DIRS = new Set(['github/advisories', 'npm/versions'])
+const compress = promisify(gzip)
+const decompress = promisify(gunzip)
 
 // An environment variable may hold anything: only an absolute path is one.
 const absolute = (path) => (typeof path === 'string' && isAbsolute(path) ? path : undefined)
@@ -95,10 +105,12 @@ export async function readCache(dir, key) {
   return path && await readRegularFile(path)
 }
 
-export async function readCacheJSON(dir, key) {
+// With `compressed`, the file is decompressed first, to no more than a file
+// read; one that is not gzip, fails its CRC, or runs past that, is a miss.
+export async function readCacheJSON(dir, key, { compressed = false } = {}) {
   const bytes = await readCache(dir, key)
   try {
-    return bytes && JSON.parse(decode(bytes, key))
+    return bytes && JSON.parse(decode(compressed ? await decompress(bytes, { maxOutputLength: MAX_BYTES }) : bytes, key))
   } catch {
     return null
   }
@@ -119,33 +131,74 @@ export async function writeCache(dir, key, data) {
   }
 }
 
-export async function writeCacheJSON(dir, key, value) {
-  return await writeCache(dir, key, JSON.stringify(value))
+export async function writeCacheJSON(dir, key, value, { compressed = false } = {}) {
+  const json = JSON.stringify(value)
+  return await writeCache(dir, key, compressed ? await compress(json) : json)
 }
 
-export async function readRecord(dir, name) {
-  const record = await readCacheJSON(dir, `${name}.json`)
+// A call's `cache` option: a store of the caller's, with read and write,
+// to keep what the call caches in place of the cache set; or false, for
+// the cache set to be read and nothing written. `cache` below is that.
+export const assertCache = assertion('false, or a store with read and write', (value) => value === false || (typeof value?.read === 'function' && typeof value?.write === 'function'))
+
+// What is kept as `dir` and `name`: in a `cache` store, else in the cache
+// set, as `fromDisk` and `toDisk` read and write it.
+async function read(dir, name, cache, fromDisk) {
+  assert.ok(DIRS.has(dir), `Unexpected cache entry: ${dir}`)
+  return cache ? await cache.read(dir, name) : await fromDisk()
+}
+
+async function write(dir, name, value, cache, toDisk) {
+  assert.ok(DIRS.has(dir), `Unexpected cache entry: ${dir}`)
+  if (cache === false) return false
+  if (!cache) return await toDisk()
+  await cache.write(dir, name, value)
+  return true
+}
+
+// JSON, as it is in a store, and in the cache set gzipped where `dir` is.
+const jsonFile = (dir, name) => (GZIP_DIRS.has(dir) ? { file: `${name}.json.gz`, compressed: true } : { file: `${name}.json` })
+
+export function readJSON(dir, name, cache) {
+  const { file, compressed } = jsonFile(dir, name)
+  return read(dir, name, cache, () => readCacheJSON(dir, file, { compressed }))
+}
+
+export function writeJSON(dir, name, value, cache) {
+  const { file, compressed } = jsonFile(dir, name)
+  return write(dir, name, value, cache, () => writeCacheJSON(dir, file, value, { compressed }))
+}
+
+// Bytes, in the cache set as `file`. A store may keep what it is given and
+// answer what it keeps, so bytes cross it as a copy each way; what it
+// answers that is not bytes is none.
+export async function readBytes(dir, name, file, cache) {
+  const bytes = await read(dir, name, cache, () => readCache(dir, file))
+  if (!(bytes instanceof Uint8Array)) return null
+  return cache ? new Uint8Array(bytes) : bytes
+}
+
+export const writeBytes = (dir, name, file, bytes, cache) => write(dir, name, cache ? new Uint8Array(bytes) : bytes, cache, () => writeCache(dir, file, bytes))
+
+// A record, kept as readJSON and writeJSON keep it, for `ttl`.
+export async function readRecord(dir, name, cache, ttl = RECORD_TTL_MS) {
+  const record = await readJSON(dir, name, cache)
   const age = typeof record?.at === 'number' ? Date.now() - record.at : Number.NaN
-  const fresh = age >= 0 && age <= RECORD_TTL_MS // An entry from the future is not fresh forever.
+  const fresh = age >= 0 && age <= ttl // An entry from the future is not fresh forever.
   return fresh && record.name === name ? record : null
 }
 
-export const writeRecord = (dir, name, value) => writeCacheJSON(dir, `${name}.json`, { at: Date.now(), name, ...value })
+export const writeRecord = (dir, name, value, cache) => writeJSON(dir, name, { at: Date.now(), name, ...value }, cache)
 
-// Name → the GitHub repo cached for it, for those of `names` that have one.
-export async function readRepos(dir, names) {
-  const repos = new Map()
-  for (const name of names) {
-    const entry = await readRecord(dir, name)
-    if (isRepo(entry?.github)) repos.set(name, entry.github)
-  }
-  return repos
+// Name -> the GitHub repo kept for it (readRecord), for those of `names`
+// that have one.
+export async function readRepos(dir, names, cache) {
+  const entries = await pool(names, RECORDS_AT_ONCE, (name) => readRecord(dir, name, cache))
+  return new Map(names.flatMap((name, i) => (isRepo(entries[i]?.github) ? [[name, entries[i].github]] : [])))
 }
 
-// An answer into `repos`, and into the cache where it found a repo.
-export async function addRepos(dir, repos, answer) {
-  for (const [name, github] of answer) {
-    repos.set(name, github)
-    if (github) await writeRecord(dir, name, { github })
-  }
+// An answer into `repos`, and kept (writeRecord) where it found a repo.
+export async function addRepos(dir, repos, answer, cache) {
+  for (const [name, github] of answer) repos.set(name, github)
+  await pool([...answer].filter(([, github]) => github), RECORDS_AT_ONCE, ([name, github]) => writeRecord(dir, name, { github }, cache))
 }

package/src/cargo/repos.js

@@ -12,7 +12,7 @@ const PER_REQUEST = 100
 const PACE_MS = 1000 // crates.io asks for a request a second at most, and a user agent.
 const USER_AGENT = '@preventive/upstream (https://github.com/PreventiveMeasures/libraries)'
 
-// One request for up to a hundred crates: each name → the GitHub repo its
+// One request for up to a hundred crates: each name -> the GitHub repo its
 // `repository` names, or null where it names none. A crate crates.io
 // leaves out of its answer is one it does not have, null too.
 async function askCrates(method, names) {
@@ -27,12 +27,12 @@ async function askCrates(method, names) {
   return found
 }
 
-// Name → its GitHub repo, or null where crates.io has no such crate or it
-// names no GitHub repo, through the cache: only a repo found is cached,
-// for a month. A failed request throws, or with `soft` leaves its names
-// out; `cachedOnly` asks nothing.
-export async function lookUpCrateRepos(method, names, { soft = false, cachedOnly = false } = {}) {
-  const repos = await readRepos(DIR, names)
+// Name -> its GitHub repo, or null where crates.io has no such crate or it
+// names no GitHub repo, through the cache, or a `cache` store (readRecord):
+// only a repo found is kept, for a month. A failed request throws, or with
+// `soft` leaves its names out; `cachedOnly` asks nothing.
+export async function lookUpCrateRepos(method, names, { soft = false, cachedOnly = false, cache } = {}) {
+  const repos = await readRepos(DIR, names, cache)
   if (cachedOnly) return repos
   for (const [i, chunk] of chunks(names.filter((name) => !repos.has(name)), PER_REQUEST).entries()) {
     if (i > 0) await sleep(PACE_MS)
@@ -40,7 +40,7 @@ export async function lookUpCrateRepos(method, names, { soft = false, cachedOnly
       if (soft) return new Map()
       throw error
     })
-    await addRepos(DIR, repos, answer)
+    await addRepos(DIR, repos, answer, cache)
   }
   return repos
 }

package/src/download.js

@@ -1,7 +1,7 @@
 import assert from 'node:assert/strict'
 import { createHash } from 'node:crypto'
 
-import { ourCachePaths, readCache, readRegularFile, writeCache } from './cache.js'
+import { ourCachePaths, readBytes, readRegularFile, writeBytes } from './cache.js'
 import { request } from './http.js'
 import { gitTreeOfTarball } from './tree.js'
 
@@ -18,9 +18,10 @@ const DIGESTS = {
 // caller gives, never a disk; without it nothing is read. From `local`,
 // other tools' caches, and with `ours` our default cache and stasis's,
 // whether set or not (read, never written; a mismatch is passed over),
-// else the cache set (a mismatch throws), else what `locate` answers,
-// checked and then cached.
-export async function verifiedDownload({ method, dir, what, ext, algorithm, expected, local = [], ours = false, locate, options = {}, list }) {
+// else the cache set, or a `cache` store in its place (readBytes; a
+// mismatch throws), else what `locate` answers, checked and then kept as
+// writeBytes keeps it.
+export async function verifiedDownload({ method, dir, what, ext, algorithm, expected, local = [], ours = false, cache, locate, options = {}, list }) {
   assert.ok(Object.hasOwn(DIGESTS, algorithm) && typeof expected === 'string' && expected !== '', `${method}: nothing to check ${what} against`)
   const digest = (bytes) => DIGESTS[algorithm](bytes, { expected, list })
   const key = `${what}.${ext}`
@@ -33,10 +34,10 @@ export async function verifiedDownload({ method, dir, what, ext, algorithm, expe
     assert.ok(actual === expected, `${method}: integrity mismatch for ${what} from ${from}: expected ${expected}, got ${actual}`)
     return bytes
   }
-  const cached = await readCache(dir, key)
-  if (cached) return await check(cached, 'the cache')
+  const cached = await readBytes(dir, what, key, cache)
+  if (cached) return await check(cached, cache ? 'the store' : 'the cache')
   const url = await locate()
   const bytes = await check(await request(url, { ...options, as: 'bytes' }), url)
-  await writeCache(dir, key, bytes)
+  await writeBytes(dir, what, key, bytes, cache)
   return bytes
 }

package/src/git.js

@@ -54,5 +54,5 @@ export async function findGitCheckout(dir) {
   const directory = relative(dirs.root, start).split(sep).join('/')
   const github = await originGitHub(dirs.commonDir)
   const commit = await headCommit(dirs.gitDir, dirs.commonDir)
-  return { ...(github && { github }), ...(directory && { directory }), ...(github && { url: `https://github.com/${github}` }), ...(commit && { commit }) }
+  return { ...(github && { github }), directory, ...(github && { url: `https://github.com/${github}` }), ...(commit && { commit }) }
 }