@preventive/upstream 1.0.0-alpha.10 → 1.0.0-alpha.12

package/advisories.d.ts

@@ -2,8 +2,10 @@
 // other.
 
 import type { Client, RepoName } from './github.js'
+import type { CacheStore } from './npm.js'
 
 export { HttpError } from './npm.js'
+export type { CacheStore } from './npm.js'
 
 // As purl and stasis name them.
 export type Ecosystem = 'npm' | 'cargo' | 'composer' | 'soldeer' | 'github'
@@ -36,9 +38,16 @@ export interface AdvisoryOptions {
   // Also asks each npm, cargo and composer package's GitHub repository
   // for its published advisories, which it has before GitHub reviews them
   // into the databases above: the package's `github` where given, else
-  // the one npm's metadata, crates.io or Packagist names, looked up
-  // through the cache. Needs `github`.
+  // the one looked up. An npm package's is the one the document of its
+  // newest version asked names, kept for good as getMeta keeps it (none
+  // where the registry does not have that version); a crate's and a
+  // composer package's, the one crates.io or Packagist names, kept a
+  // month. Needs `github`.
   repoAdvisories?: boolean
+  // Where what this call caches is kept, as CacheOptions (npm.js) has it:
+  // a repository's listing, and each package's repository looked up (an
+  // npm package's as its version's document). See `advisories`.
+  cache?: CacheStore | false
   // Also returns each advisory's full text, as `details`. OSV's records
   // and a repository's listing already carry it; npm's registry does not,
   // and its rows take OSV's record of their GHSA, one more request each
@@ -108,6 +117,11 @@ export interface Advisory {
 // nothing, and a GHSA OSV does not have adds no `details`. Any other
 // failure, a malformed answer, or one about something not asked, throws:
 // nothing is left out quietly.
+// A repository's listing is kept for 90 minutes, as one entry for every
+// spelling of its name, holding only what rows are made from, `details`
+// included: in setCacheDir's cache gzipped, in a `cache` store as it is,
+// or with `cache` false, nowhere. 90 minutes is how late an advisory its
+// maintainer publishes can be seen. One gone or malformed is not kept.
 // Sorted by ecosystem and name. Versions are matched by npm's semver, from
 // the npm beside node, or by the semver peer where there is no npm.
 export function advisories(packages: Iterable<Package>, options?: AdvisoryOptions): Promise<Advisory[]>

package/git.d.ts

@@ -6,8 +6,8 @@ export interface GitCheckout {
   // `owner/name`, when the origin remote is a GitHub one. Never the
   // remote's URL, which can name a private host or carry a token.
   github?: string
-  // Where the directory sits in its checkout, `/`-separated; absent at
-  // the root.
+  // Where the directory sits in its checkout, `/`-separated; `''` at the
+  // root.
   directory?: string
   // `https://github.com/<github>`, whenever there is a `github`.
   url?: string

package/npm.d.ts

@@ -32,14 +32,45 @@ export const defaultCacheDir: string | undefined
 // cache: every read of it misses and every write is skipped. getTarball
 // reads ours all the same, set or not.
 //
-// The directory is trusted: a package's repo is answered from it as it
-// was written, with no request. Point it only at storage that nothing
-// less trusted than the caller can write, never a cache shared with or
-// restored from lower-trust jobs. Tarballs, crates (cargo.js), zips
+// The directory is trusted: a package's repo, as recorded or as a version
+// document kept here names it, and for 90 minutes a repository's published
+// advisories (advisories.js), are answered from it as they were written,
+// with no request. Point it only at storage that
+// nothing less trusted than the caller can write, never a cache shared
+// with or restored from lower-trust jobs. Tarballs, crates (cargo.js), zips
 // (soldeer.js) and tree tarballs (github.js) don't rely on this, since
-// they are checked against their integrity on every call.
+// they are checked against their integrity on every call; and the version
+// documents kept here (getMeta) never supply an integrity, only confirm
+// one a caller gives (verifyDist, and getMeta with a `dist`). A call's `cache` option, where it has
+// one, keeps what it caches elsewhere, or writes none of it here.
 export function setCacheDir(dir?: string | false): void
 
+// A store of the caller's, a database's say, for what would otherwise be
+// kept in setCacheDir's cache: `read` answers the value last written
+// under that `type` and `key`, or null or undefined for none. A `type` is
+// what is kept and a `key` which one: `npm/versions` and `npm/tarballs`
+// with `<name>@<version>` for a version's document and tarball; and for
+// advisories.js, `github/advisories` with the repository's `owner/name` in
+// lowercase for its listing, and `cargo/repos`, `composer/repos` and
+// `soldeer/repos` with a package's name for the repository it names.
+// Values are plain JSON data, which `read` may answer as written or as a
+// copy, but a tarball's are its bytes, a Uint8Array. Whatever it answers
+// is checked as a cached file is: one malformed, stale or kept differently
+// is a miss, and a tarball's bytes that do not match throw. A rejection
+// from either is the call's. It is trusted as setCacheDir's directory is.
+export interface CacheStore {
+  read(type: string, key: string): Promise<unknown>
+  write(type: string, key: string, value: unknown): Promise<void>
+}
+
+// Where a call keeps what it caches: in setCacheDir's cache where `cache`
+// is left out; in a store of the caller's in place of it; or with `false`,
+// nowhere, though setCacheDir's cache is read all the same. Other tools'
+// caches and ours are read for tarballs whatever it is (getTarball).
+export interface CacheOptions {
+  cache?: CacheStore | false
+}
+
 // A failed request: `status` is the HTTP status the registry answered
 // with (404 for a package or version it does not have).
 export class HttpError extends Error {
@@ -49,9 +80,9 @@ export class HttpError extends Error {
 
 // A package's GitHub repo as its registry metadata names it — getRepo
 // (package.js) over its `latest` document: `github` is `owner/name`, and
-// `directory` is where in the repo the package sits, absent for a
-// package at the repo root. Throws when the package cannot be fetched or
-// names no GitHub repo.
+// `directory` is where in the repo the package sits, `''` at the repo
+// root, absent where it declares none. Throws when the package cannot be
+// fetched or names no GitHub repo.
 export interface GitHubLink {
   github: string
   directory?: string
@@ -73,7 +104,8 @@ export function resolvePackageRepos(packageNames: Iterable<string>, options?: {
 // release that resolved repos by other rules included; a write answers
 // false where it could not write, and never throws. A write takes the
 // `github` a lookup gives, `owner/name` with no trailing `.git`, so what
-// it writes reads back.
+// it writes reads back; a `directory` left out is unknown, and `''` the
+// repo root.
 export function readPackageRepoCache(name: string): Promise<PackageRepo | null>
 export function writePackageRepoCache(name: string, github: string, directory?: string): Promise<boolean>
 
@@ -86,25 +118,83 @@ export interface Dist {
   integrity: string
 }
 
+// `gitHead` is the commit the document names, as npm reads it off the git
+// checkout it publishes from, where it is a full commit id: 40 lowercase
+// hex digits, or 64. `repository`, `homepage` and `bugs` are the
+// document's, as the package.json published had them, or npm filled them
+// in from one another: each a string as it is, or for `repository` and
+// `bugs` an object, of which only the strings named below are read. Each is
+// absent where the document has none, or has another shape, or an object
+// with none of those strings. All of it is the publisher's word, held to no
+// repository.
 export interface PackageMeta {
   name: string
   version: string
   dist: Dist
+  gitHead?: string
+  repository?: string | PackageRepository
+  homepage?: string
+  bugs?: string | PackageBugs
+}
+
+export interface PackageRepository {
+  type: string | undefined
+  url: string | undefined
+  directory: string | undefined
+}
+
+export interface PackageBugs {
+  url: string | undefined
+  email: string | undefined
+}
+
+// getMeta's options: `cache`, and the `dist` a caller already has of that
+// version, as a lockfile gives it, held to the rules above.
+export interface MetaOptions extends CacheOptions {
+  dist?: Dist
 }
 
 // The registry's version document, refused unless it is for that name and
 // version and its dist is held to the rules above.
-export function getMeta(name: string, version: string): Promise<PackageMeta>
+//
+// With `dist`, npm's own cache comes first, where getTarball looks for
+// npm's tarballs: where it keeps the full packument of that name, as npm 10
+// and later fetch it, and that version's entry there has the given dist's
+// tarball and integrity, that entry answers, with no request, and neither
+// `cache` nor setCacheDir's cache is read or written. An abbreviated
+// packument, as earlier npm fetches, is passed over, as is one whose
+// content does not match the sha512 npm filed it by. What it answers
+// beside the dist is taken from npm's cache as it is from setCacheDir's.
+// Else the document `cache` keeps answers, with no request, where its
+// tarball and integrity are the given dist's; else the registry's is asked
+// for, and throws where they are not.
+//
+// Without `dist`, the registry is asked on every call: a cache never
+// supplies a dist. Kept whole, as `cache` says, and for good once it
+// passes, as the registry never takes a version twice: in setCacheDir's
+// cache, compressed, where `cache` is left out. One refused is not kept.
+export function getMeta(name: string, version: string, options?: MetaOptions): Promise<PackageMeta>
+
+// When the registry says each version of a package it lists was published,
+// by version, as Date#toISOString writes it: from the whole package's
+// document, the only one with `time`, megabytes for some. A version whose
+// time is missing, not an ISO 8601 UTC time, or not one there is, as
+// 2018-02-29, is left out. Refused unless the document is for that name.
+export function getPublishTimes(name: string): Promise<Map<string, string>>
 
-// Fetches the registry's dist for that version and throws unless its
-// integrity is the one given, as for a dist read off a lockfile.
-export function verifyDist(name: string, version: string, dist: Dist): Promise<void>
+// Throws unless the integrity of that version is the one given, as for a
+// dist read off a lockfile: by the version document `cache` keeps, with no
+// request, else by the registry's, kept then as getMeta keeps it. One kept
+// with another integrity, refused, or for another name or version, is
+// passed over for the registry's, which throws where it has another.
+export function verifyDist(name: string, version: string, dist: Dist, options?: CacheOptions): Promise<void>
 
 // A published version's gzipped tarball, whole, in memory. Without `dist`,
-// the version document is fetched on every call, cached tarball or not;
-// with one, it is trusted as given and nothing but the tarball is asked
-// for. Either way the bytes are checked against `dist.integrity`, whether
-// they were downloaded (before they are cached) or read from a cache.
+// the version document is fetched on every call, cached tarball or not,
+// and kept as getMeta keeps it; with one, it is trusted as given and
+// nothing but the tarball is asked for. Either way the bytes are checked
+// against `dist.integrity`, whether they were downloaded (before they are
+// cached) or read from a cache.
 //
 // Other caches are read first, and never written: npm's (cacache under
 // npm_config_cache, else npm's default, `~/.npm` outside Windows; and npm
@@ -112,5 +202,7 @@ export function verifyDist(name: string, version: string, dist: Dist): Promise<v
 // `~/.audit/cache/tgz/<org>:<name>-<version>.tgz`, then defaultCacheDir
 // and cacheDirFor('stasis'), as setCacheDir files a tarball in either,
 // whether set or not. A file there that does not match is passed over.
-// Then setCacheDir's cache, where one that does not match throws.
-export function getTarball(name: string, version: string, dist?: Dist): Promise<Uint8Array>
+// Then setCacheDir's cache, or a `cache` store in its place, where bytes
+// that do not match throw. A tarball downloaded is kept there, unless
+// `cache` is false.
+export function getTarball(name: string, version: string, dist?: Dist, options?: CacheOptions): Promise<Uint8Array>

package/npm.js

@@ -1,4 +1,4 @@
 export { cacheDirFor, defaultCacheDir, setCacheDir } from './src/cache.js'
 export { HttpError } from './src/http.js'
 export { getGitHub, readPackageRepoCache, resolvePackageRepos, writePackageRepoCache } from './src/npm/repos.js'
-export { getMeta, getTarball, verifyDist } from './src/npm/tarball.js'
+export { getMeta, getPublishTimes, getTarball, verifyDist } from './src/npm/tarball.js'

package/package.d.ts

@@ -5,7 +5,10 @@ export interface PackageRepoLink {
   github?: string
   // Where in that repo the package sits, `/`-separated, as the tree path
   // itself (a homepage URL's decoded; a `repository.directory`'s `\` read
-  // as `/`); absent at its root, and whenever `github` is.
+  // as `/`, its empty and `.` parts dropped); `''` at its root, where
+  // `repository.directory` declares it (`./`, `/`, `/.`, `.`, `''`). Absent
+  // where unknown: neither field names a directory in it (one with a `..`
+  // part names none), and whenever `github` is absent.
   directory?: string
   // `https://github.com/<github>`, whenever there is a `github`.
   url?: string

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@preventive/upstream",
-  "version": "1.0.0-alpha.10",
+  "version": "1.0.0-alpha.12",
   "description": "Minimal npm, crates.io and Soldeer registry clients, a GitHub client, and a git checkout reader: the calls we use, nothing more",
   "type": "module",
   "exports": {
@@ -85,9 +85,11 @@
     "src/github/read.js",
     "src/github/write.js",
     "src/http.js",
+    "src/npm/cacache.js",
     "src/npm/registry.js",
     "src/npm/repos.js",
     "src/npm/tarball.js",
+    "src/npm/versions.js",
     "src/package.js",
     "src/pool.js",
     "src/remote.js",

package/src/advisories/github.js

@@ -1,6 +1,7 @@
 import assert from 'node:assert/strict'
 
 import { assertRepo, assertSoldeerName, assertSoldeerVersion, assertion, isGhsa, isRefName, isStrings } from '../args.js'
+import { readRecord, writeRecord } from '../cache.js'
 import { isGone } from '../github/client.js'
 import { recover } from '../http.js'
 import { pool } from '../pool.js'
@@ -8,6 +9,12 @@ import { advisoryUrl, covered, detailsOf, isText, mayBeInRange, metrics } from '
 import { soldeerRepos } from './repos.js'
 
 const REPOS_AT_ONCE = 4
+const DIR = 'github/advisories'
+// A maintainer publishes one whenever they are ready, and seeing it before
+// GitHub reviews it is what a repository is asked for: 90 minutes, not a month.
+const LISTING_TTL_MS = 90 * 60 * 1000
+// Stamped on each entry, and raised when a listing is kept differently.
+const VERSION = 1
 
 const isRepoAdvisory = (advisory) => advisory && typeof advisory === 'object' && isGhsa(advisory.ghsa_id)
   && advisory.state === 'published' && isText(advisory.summary)
@@ -15,7 +22,51 @@ const isRepoAdvisory = (advisory) => advisory && typeof advisory === 'object' &&
 
 export const assertClient = assertion('a GitHub client from createClient', (value) => typeof value?.listRepoAdvisories === 'function')
 
-const listAdvisories = (github, repo) => github.listRepoAdvisories({ repo }).catch(recover(isGone, []))
+// What rows are made from, and all that is kept of a listing: each advisory
+// not withdrawn, with its CVSS vector GitHub prefers, and each vulnerable
+// range with the package it names. A listing that is malformed is refused
+// whole, and never kept; one whose only fault is a description (isDigest)
+// is answered, but not kept either.
+function digest(repo, list) {
+  return list.flatMap((advisory) => {
+    assert.ok(isRepoAdvisory(advisory), `advisories: malformed advisory from ${repo}`)
+    if (advisory.withdrawn_at) return []
+    const cvss = [advisory.cvss_severities?.cvss_v3, advisory.cvss_severities?.cvss_v4, advisory.cvss].find((entry) => typeof entry?.vector_string === 'string')
+    const ranges = (advisory.vulnerabilities ?? []).map((vulnerability) => {
+      const range = vulnerability?.vulnerable_version_range ?? ''
+      assert.ok(typeof range === 'string', `advisories: malformed range in ${advisory.ghsa_id}`)
+      const { ecosystem, name } = vulnerability?.package ?? {}
+      return { range, ...(typeof ecosystem === 'string' && { ecosystem }), ...(typeof name === 'string' && { name }) }
+    })
+    return [{
+      ghsa: advisory.ghsa_id,
+      title: advisory.summary,
+      ...(advisory.description != null && advisory.description !== '' && { description: advisory.description }),
+      ...metrics({ severity: advisory.severity, score: cvss?.score, vector: cvss?.vector_string, cwe: advisory.cwe_ids }),
+      ranges,
+    }]
+  })
+}
+
+// A description is checked only for `details` (detailsOf), so a malformed
+// one leaves the listing usable, but not kept.
+const isDigest = (advisory) => advisory && typeof advisory === 'object' && isGhsa(advisory.ghsa) && isText(advisory.title)
+  && (advisory.description === undefined || isText(advisory.description))
+  && Array.isArray(advisory.ranges) && advisory.ranges.every((entry) => typeof entry?.range === 'string')
+
+// A repository's listing, digested, through the cache: GitHub's names are
+// case-insensitive, so one entry answers every spelling. A repository gone,
+// renamed or blocked has none, and that is not kept.
+async function listAdvisories(github, repo, cache) {
+  const name = repo.toLowerCase()
+  const entry = await readRecord(DIR, name, cache, LISTING_TTL_MS)
+  if (entry?.v === VERSION && Array.isArray(entry.advisories) && entry.advisories.every(isDigest)) return entry.advisories
+  const list = await github.listRepoAdvisories({ repo }).catch(recover(isGone, null))
+  if (list === null) return []
+  const advisories = digest(repo, list)
+  if (advisories.every(isDigest)) await writeRecord(DIR, name, { v: VERSION, advisories }, cache)
+  return advisories
+}
 
 // GitHub's `>= 1.0.0, < 1.2.6` is npm's with the commas dropped.
 // Maintainers write these unreviewed: one without a range, or with one
@@ -23,17 +74,16 @@ const listAdvisories = (github, repo) => github.listRepoAdvisories({ repo }).cat
 // may publish it long before the advisory database has one, and its
 // `description` is the text `details` asks for: the listing has it all.
 function fromRepository(repo, name, advisory, range, asked, { covers, details }) {
-  const cvss = [advisory.cvss_severities?.cvss_v3, advisory.cvss_severities?.cvss_v4, advisory.cvss].find((entry) => typeof entry?.vector_string === 'string')
   return {
     name,
     source: 'repository',
-    id: advisory.ghsa_id,
-    ghsa: advisory.ghsa_id,
-    url: advisoryUrl(advisory.ghsa_id, repo),
+    id: advisory.ghsa,
+    ghsa: advisory.ghsa,
+    url: advisoryUrl(advisory.ghsa, repo),
     aliases: [],
-    title: advisory.summary,
-    ...(details && detailsOf(advisory.description, advisory.ghsa_id)),
-    ...metrics({ severity: advisory.severity, score: cvss?.score, vector: cvss?.vector_string, cwe: advisory.cwe_ids }),
+    title: advisory.title,
+    ...(details && detailsOf(advisory.description, advisory.ghsa)),
+    ...metrics({ severity: advisory.severity, score: advisory.cvss, vector: advisory.cvssVector, cwe: advisory.cwe }),
     range,
     versions: covered(asked, range.replaceAll(',', ' '), covers),
   }
@@ -41,21 +91,18 @@ function fromRepository(repo, name, advisory, range, asked, { covers, details })
 
 // Each asked name's repository, `repoOf` it, is asked once, and its
 // advisories' vulnerable ranges become rows for the names that `takes` a
-// vulnerability, one per name, advisory and range, holding the asked
+// range's package, one per name, advisory and range, holding the asked
 // versions it `covers`, and with `details`, its text.
-async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers, details }) {
+async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers, details, cache }) {
   // GitHub's names are case-insensitive: one spelling asks for all.
   const namesOf = Map.groupBy([...asked.keys()].filter(repoOf), (name) => repoOf(name).toLowerCase())
-  const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ repo: repoOf(names[0]), names, list: await listAdvisories(github, repoOf(names[0])) }))
+  const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ repo: repoOf(names[0]), names, list: await listAdvisories(github, repoOf(names[0]), cache) }))
   const rows = new Map()
   for (const { repo, names, list } of listed) {
     for (const advisory of list) {
-      assert.ok(isRepoAdvisory(advisory), `advisories: malformed advisory from ${repo}`)
-      for (const vulnerability of advisory.withdrawn_at ? [] : advisory.vulnerabilities ?? []) {
-        const range = vulnerability?.vulnerable_version_range ?? ''
-        assert.ok(typeof range === 'string', `advisories: malformed range in ${advisory.ghsa_id}`)
-        for (const name of names.filter((candidate) => takes(candidate, vulnerability?.package))) {
-          rows.set(`${name} ${advisory.ghsa_id} ${range}`, fromRepository(repo, name, advisory, range, asked.get(name), { covers, details }))
+      for (const { range, ...pkg } of advisory.ranges) {
+        for (const name of names.filter((candidate) => takes(candidate, pkg))) {
+          rows.set(`${name} ${advisory.ghsa} ${range}`, fromRepository(repo, name, advisory, range, asked.get(name), { covers, details }))
         }
       }
     }
@@ -63,9 +110,10 @@ async function repositoryAdvisories(github, asked, { repoOf, takes = () => true,
   return [...rows.values()]
 }
 
-// Each asked name's repository: as given, `known`, or else looked up.
-async function reposOf(asked, known, lookUp) {
-  const found = await lookUp([...asked.keys()].filter((name) => !known.has(name)))
+// Each asked name's repository: as given, `known`, or else looked up,
+// through `cache`.
+async function reposOf(asked, known, lookUp, cache) {
+  const found = await lookUp([...asked.keys()].filter((name) => !known.has(name)), { asked, cache })
   return (name) => known.get(name) ?? found.get(name)
 }
 
@@ -74,12 +122,12 @@ async function reposOf(asked, known, lookUp) {
 // each package's repository, `known` or else looked up, adds what it
 // publishes for GitHub's `ecosystem` entries naming the package, for the
 // versions `rows` do not already report under that GHSA.
-export async function withRepositories(rows, asked, { github, repoAdvisories, known, details }, { ecosystem, lookUp, covers }) {
+export async function withRepositories(rows, asked, { github, repoAdvisories, known, details, cache }, { ecosystem, lookUp, covers }) {
   if (!repoAdvisories) return rows
   const reported = new Set(rows.flatMap((row) => [row.id, row.ghsa, ...row.aliases].filter(isGhsa).flatMap((id) => row.versions.map((version) => `${row.name} ${id} ${version}`))))
-  const repoOf = await reposOf(asked, known, lookUp)
-  const takes = (name, pkg) => pkg?.ecosystem === ecosystem && pkg.name === name
-  const added = await repositoryAdvisories(github, asked, { repoOf, takes, covers, details })
+  const repoOf = await reposOf(asked, known, lookUp, cache)
+  const takes = (name, pkg) => pkg.ecosystem === ecosystem && pkg.name === name
+  const added = await repositoryAdvisories(github, asked, { repoOf, takes, covers, details, cache })
   return [...rows, ...added.map((row) => ({ ...row, versions: row.versions.filter((version) => !reported.has(`${row.name} ${row.id} ${version}`)) }))]
 }
 
@@ -93,7 +141,7 @@ export const GITHUB = {
   repositoryOnly: true,
   assertName: assertRepo,
   assertVersion: assertion('a version or a branch name', isRefName),
-  advisories: (asked, { github, details }) => repositoryAdvisories(github, asked, { repoOf: (repo) => repo, covers: coversPlaceholder, details }),
+  advisories: (asked, { github, details, cache }) => repositoryAdvisories(github, asked, { repoOf: (repo) => repo, covers: coversPlaceholder, details, cache }),
 }
 
 // Soldeer packages, which no advisory database has: the repository each
@@ -104,5 +152,5 @@ export const SOLDEER = {
   repositoryOnly: true,
   assertName: assertSoldeerName,
   assertVersion: assertSoldeerVersion,
-  advisories: async (asked, { github, known, details }) => await repositoryAdvisories(github, asked, { repoOf: await reposOf(asked, known, soldeerRepos), details }),
+  advisories: async (asked, { github, known, details, cache }) => await repositoryAdvisories(github, asked, { repoOf: await reposOf(asked, known, soldeerRepos, cache), details, cache }),
 }

package/src/advisories/index.js

@@ -1,6 +1,7 @@
 import assert from 'node:assert/strict'
 
 import { assertArgs, assertBoolean, assertRepo, assertion, isIterable, optional, sameName } from '../args.js'
+import { assertCache } from '../cache.js'
 import { order } from './common.js'
 import { GITHUB, SOLDEER, assertClient } from './github.js'
 import { NPM } from './npm.js'
@@ -34,7 +35,7 @@ function collect(packages) {
 }
 
 export async function advisories(packages, options = {}) {
-  assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean) })
+  assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean), cache: optional(assertCache) })
   assert.ok(!options.repoAdvisories || options.github, 'advisories: repoAdvisories needs a github client')
   const byEcosystem = collect(packages)
   for (const ecosystem of byEcosystem.keys()) assert.ok(!ECOSYSTEMS[ecosystem].repositoryOnly || options.github, `advisories: ${ecosystem} packages need a github client`)
@@ -43,7 +44,7 @@ export async function advisories(packages, options = {}) {
     const names = [...named.keys()].toSorted()
     const asked = new Map(names.map((name) => [name, [...named.get(name).versions].toSorted(compare)]))
     const known = new Map(names.filter((name) => named.get(name).github).map((name) => [name, named.get(name).github]))
-    const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, known })
+    const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, known, cache: options.cache })
     return rows.map((row) => ({ ecosystem, ...row }))
   }))
   return found.flat().filter((row) => row.versions.length > 0).toSorted((a, b) => order(a.ecosystem, b.ecosystem) || order(a.name, b.name))

package/src/advisories/repos.js

@@ -4,7 +4,8 @@ import { show } from '../args.js'
 import { addRepos, readRepos } from '../cache.js'
 import { lookUpCrateRepos } from '../cargo/repos.js'
 import { PACKAGIST_REPO, SOLDEER_API, buildUrl, isNotFound, recover, request } from '../http.js'
-import { lookUpPackageRepo } from '../npm/repos.js'
+import { getVersionDocument } from '../npm/versions.js'
+import { getRepo } from '../package.js'
 import { pool } from '../pool.js'
 import { githubRepoOfUrl } from '../remote.js'
 
@@ -12,10 +13,10 @@ const PACKAGES_AT_ONCE = 8
 
 // Name → its GitHub repo, or null where the registry has no such package
 // or it names no GitHub repo; any other failure throws. Only a repo found
-// is cached, for a month.
-async function cachedRepos(dir, names, fetchMissing) {
-  const repos = await readRepos(dir, names)
-  await addRepos(dir, repos, await fetchMissing(names.filter((name) => !repos.has(name))))
+// is cached, for a month, in the cache set or `cache` (readRecord).
+async function cachedRepos(dir, names, fetchMissing, cache) {
+  const repos = await readRepos(dir, names, cache)
+  await addRepos(dir, repos, await fetchMissing(names.filter((name) => !repos.has(name))), cache)
   return repos
 }
 
@@ -42,7 +43,17 @@ async function fetchSoldeerRepo(name) {
   return githubRepoOfUrl(answer.data[0]?.github_url) ?? null
 }
 
-export const crateRepos = (names) => lookUpCrateRepos('advisories', names)
-export const composerRepos = (names) => cachedRepos('composer/repos', names, (missing) => fetchEach(missing, fetchComposerRepo))
-export const npmRepos = (names) => fetchEach(names, async (name) => (await lookUpPackageRepo(name))?.github ?? null)
-export const soldeerRepos = (names) => cachedRepos('soldeer/repos', names, (missing) => fetchEach(missing, fetchSoldeerRepo))
+// The repo the newest version asked of each package names, from that
+// version's document, kept for good (getVersionDocument): null where the
+// registry does not have that version, or it names no GitHub repo.
+async function npmRepo(name, version, cache) {
+  const json = await getVersionDocument('advisories', name, version, cache)
+  return (json && getRepo(json).github) ?? null
+}
+
+// Each takes the names to look up, the versions `asked` of each, and the
+// call's `cache`.
+export const crateRepos = (names, { cache }) => lookUpCrateRepos('advisories', names, { cache })
+export const composerRepos = (names, { cache }) => cachedRepos('composer/repos', names, (missing) => fetchEach(missing, fetchComposerRepo), cache)
+export const npmRepos = (names, { asked, cache }) => fetchEach(names, (name) => npmRepo(name, asked.get(name).at(-1), cache))
+export const soldeerRepos = (names, { cache }) => cachedRepos('soldeer/repos', names, (missing) => fetchEach(missing, fetchSoldeerRepo), cache)

package/src/args.js

@@ -7,7 +7,7 @@ export const matches = (regex) => (value) => typeof value === 'string' && regex.
 // Bidi controls can make a log line read as something it doesn't say.
 const isBidi = (char) => (char >= '\u202A' && char <= '\u202E') || (char >= '\u2066' && char <= '\u2069')
 // A prototype's properties would be read as options without being checked.
-const isPlainObject = (value) => value != null && [Object.prototype, null].includes(Object.getPrototypeOf(value))
+export const isPlainObject = (value) => value != null && [Object.prototype, null].includes(Object.getPrototypeOf(value))
 // Lone surrogates make encodeURIComponent throw.
 const isString = (value, max, allowed = '') => typeof value === 'string' && value.length <= max && value.isWellFormed()
   && ![...value].some((char) => isControl(char) && !allowed.includes(char))

package/src/cache.js

@@ -3,12 +3,22 @@ import { constants } from 'node:fs'
 import { mkdir, open, rename, rm, writeFile } from 'node:fs/promises'
 import { homedir } from 'node:os'
 import { dirname, isAbsolute, join, resolve } from 'node:path'
+import { promisify } from 'node:util'
+import { gunzip, gzip } from 'node:zlib'
 
-import { assertCacheName, assertDirectoryPath, isRepo } from './args.js'
+import { assertCacheName, assertDirectoryPath, assertion, isRepo } from './args.js'
 import { MAX_BYTES, decode } from './http.js'
+import { pool } from './pool.js'
 
-const DIRS = new Set(['npm/repos', 'npm/tarballs', 'cargo/repos', 'cargo/crates', 'composer/repos', 'soldeer/repos', 'soldeer/zips', 'github/trees'])
+const DIRS = new Set(['npm/repos', 'npm/tarballs', 'npm/versions', 'cargo/repos', 'cargo/crates', 'composer/repos', 'soldeer/repos', 'soldeer/zips', 'github/trees', 'github/advisories'])
 const RECORD_TTL_MS = 30 * 24 * 60 * 60 * 1000 // A link only moves on a transfer or rename, and GitHub redirects those.
+const RECORDS_AT_ONCE = 8
+// JSON filed gzipped, as `<name>.json.gz`: a listing carries each
+// advisory's full text, Markdown that compresses well, and a version
+// document all npm has of that version.
+const GZIP_DIRS = new Set(['github/advisories', 'npm/versions'])
+const compress = promisify(gzip)
+const decompress = promisify(gunzip)
 
 // An environment variable may hold anything: only an absolute path is one.
 const absolute = (path) => (typeof path === 'string' && isAbsolute(path) ? path : undefined)
@@ -95,10 +105,12 @@ export async function readCache(dir, key) {
   return path && await readRegularFile(path)
 }
 
-export async function readCacheJSON(dir, key) {
+// With `compressed`, the file is decompressed first, to no more than a file
+// read; one that is not gzip, fails its CRC, or runs past that, is a miss.
+export async function readCacheJSON(dir, key, { compressed = false } = {}) {
   const bytes = await readCache(dir, key)
   try {
-    return bytes && JSON.parse(decode(bytes, key))
+    return bytes && JSON.parse(decode(compressed ? await decompress(bytes, { maxOutputLength: MAX_BYTES }) : bytes, key))
   } catch {
     return null
   }
@@ -119,33 +131,74 @@ export async function writeCache(dir, key, data) {
   }
 }
 
-export async function writeCacheJSON(dir, key, value) {
-  return await writeCache(dir, key, JSON.stringify(value))
+export async function writeCacheJSON(dir, key, value, { compressed = false } = {}) {
+  const json = JSON.stringify(value)
+  return await writeCache(dir, key, compressed ? await compress(json) : json)
 }
 
-export async function readRecord(dir, name) {
-  const record = await readCacheJSON(dir, `${name}.json`)
+// A call's `cache` option: a store of the caller's, with read and write,
+// to keep what the call caches in place of the cache set; or false, for
+// the cache set to be read and nothing written. `cache` below is that.
+export const assertCache = assertion('false, or a store with read and write', (value) => value === false || (typeof value?.read === 'function' && typeof value?.write === 'function'))
+
+// What is kept as `dir` and `name`: in a `cache` store, else in the cache
+// set, as `fromDisk` and `toDisk` read and write it.
+async function read(dir, name, cache, fromDisk) {
+  assert.ok(DIRS.has(dir), `Unexpected cache entry: ${dir}`)
+  return cache ? await cache.read(dir, name) : await fromDisk()
+}
+
+async function write(dir, name, value, cache, toDisk) {
+  assert.ok(DIRS.has(dir), `Unexpected cache entry: ${dir}`)
+  if (cache === false) return false
+  if (!cache) return await toDisk()
+  await cache.write(dir, name, value)
+  return true
+}
+
+// JSON, as it is in a store, and in the cache set gzipped where `dir` is.
+const jsonFile = (dir, name) => (GZIP_DIRS.has(dir) ? { file: `${name}.json.gz`, compressed: true } : { file: `${name}.json` })
+
+export function readJSON(dir, name, cache) {
+  const { file, compressed } = jsonFile(dir, name)
+  return read(dir, name, cache, () => readCacheJSON(dir, file, { compressed }))
+}
+
+export function writeJSON(dir, name, value, cache) {
+  const { file, compressed } = jsonFile(dir, name)
+  return write(dir, name, value, cache, () => writeCacheJSON(dir, file, value, { compressed }))
+}
+
+// Bytes, in the cache set as `file`. A store may keep what it is given and
+// answer what it keeps, so bytes cross it as a copy each way; what it
+// answers that is not bytes is none.
+export async function readBytes(dir, name, file, cache) {
+  const bytes = await read(dir, name, cache, () => readCache(dir, file))
+  if (!(bytes instanceof Uint8Array)) return null
+  return cache ? new Uint8Array(bytes) : bytes
+}
+
+export const writeBytes = (dir, name, file, bytes, cache) => write(dir, name, cache ? new Uint8Array(bytes) : bytes, cache, () => writeCache(dir, file, bytes))
+
+// A record, kept as readJSON and writeJSON keep it, for `ttl`.
+export async function readRecord(dir, name, cache, ttl = RECORD_TTL_MS) {
+  const record = await readJSON(dir, name, cache)
   const age = typeof record?.at === 'number' ? Date.now() - record.at : Number.NaN
-  const fresh = age >= 0 && age <= RECORD_TTL_MS // An entry from the future is not fresh forever.
+  const fresh = age >= 0 && age <= ttl // An entry from the future is not fresh forever.
   return fresh && record.name === name ? record : null
 }
 
-export const writeRecord = (dir, name, value) => writeCacheJSON(dir, `${name}.json`, { at: Date.now(), name, ...value })
+export const writeRecord = (dir, name, value, cache) => writeJSON(dir, name, { at: Date.now(), name, ...value }, cache)
 
-// Name → the GitHub repo cached for it, for those of `names` that have one.
-export async function readRepos(dir, names) {
-  const repos = new Map()
-  for (const name of names) {
-    const entry = await readRecord(dir, name)
-    if (isRepo(entry?.github)) repos.set(name, entry.github)
-  }
-  return repos
+// Name → the GitHub repo kept for it (readRecord), for those of `names`
+// that have one.
+export async function readRepos(dir, names, cache) {
+  const entries = await pool(names, RECORDS_AT_ONCE, (name) => readRecord(dir, name, cache))
+  return new Map(names.flatMap((name, i) => (isRepo(entries[i]?.github) ? [[name, entries[i].github]] : [])))
 }
 
-// An answer into `repos`, and into the cache where it found a repo.
-export async function addRepos(dir, repos, answer) {
-  for (const [name, github] of answer) {
-    repos.set(name, github)
-    if (github) await writeRecord(dir, name, { github })
-  }
+// An answer into `repos`, and kept (writeRecord) where it found a repo.
+export async function addRepos(dir, repos, answer, cache) {
+  for (const [name, github] of answer) repos.set(name, github)
+  await pool([...answer].filter(([, github]) => github), RECORDS_AT_ONCE, ([name, github]) => writeRecord(dir, name, { github }, cache))
 }

package/src/cargo/repos.js

@@ -28,11 +28,11 @@ async function askCrates(method, names) {
 }
 
 // Name → its GitHub repo, or null where crates.io has no such crate or it
-// names no GitHub repo, through the cache: only a repo found is cached,
-// for a month. A failed request throws, or with `soft` leaves its names
-// out; `cachedOnly` asks nothing.
-export async function lookUpCrateRepos(method, names, { soft = false, cachedOnly = false } = {}) {
-  const repos = await readRepos(DIR, names)
+// names no GitHub repo, through the cache, or a `cache` store (readRecord):
+// only a repo found is kept, for a month. A failed request throws, or with
+// `soft` leaves its names out; `cachedOnly` asks nothing.
+export async function lookUpCrateRepos(method, names, { soft = false, cachedOnly = false, cache } = {}) {
+  const repos = await readRepos(DIR, names, cache)
   if (cachedOnly) return repos
   for (const [i, chunk] of chunks(names.filter((name) => !repos.has(name)), PER_REQUEST).entries()) {
     if (i > 0) await sleep(PACE_MS)
@@ -40,7 +40,7 @@ export async function lookUpCrateRepos(method, names, { soft = false, cachedOnly
       if (soft) return new Map()
       throw error
     })
-    await addRepos(DIR, repos, answer)
+    await addRepos(DIR, repos, answer, cache)
   }
   return repos
 }

package/src/download.js

@@ -1,7 +1,7 @@
 import assert from 'node:assert/strict'
 import { createHash } from 'node:crypto'
 
-import { ourCachePaths, readCache, readRegularFile, writeCache } from './cache.js'
+import { ourCachePaths, readBytes, readRegularFile, writeBytes } from './cache.js'
 import { request } from './http.js'
 import { gitTreeOfTarball } from './tree.js'
 
@@ -18,9 +18,10 @@ const DIGESTS = {
 // caller gives, never a disk; without it nothing is read. From `local`,
 // other tools' caches, and with `ours` our default cache and stasis's,
 // whether set or not (read, never written; a mismatch is passed over),
-// else the cache set (a mismatch throws), else what `locate` answers,
-// checked and then cached.
-export async function verifiedDownload({ method, dir, what, ext, algorithm, expected, local = [], ours = false, locate, options = {}, list }) {
+// else the cache set, or a `cache` store in its place (readBytes; a
+// mismatch throws), else what `locate` answers, checked and then kept as
+// writeBytes keeps it.
+export async function verifiedDownload({ method, dir, what, ext, algorithm, expected, local = [], ours = false, cache, locate, options = {}, list }) {
   assert.ok(Object.hasOwn(DIGESTS, algorithm) && typeof expected === 'string' && expected !== '', `${method}: nothing to check ${what} against`)
   const digest = (bytes) => DIGESTS[algorithm](bytes, { expected, list })
   const key = `${what}.${ext}`
@@ -33,10 +34,10 @@ export async function verifiedDownload({ method, dir, what, ext, algorithm, expe
     assert.ok(actual === expected, `${method}: integrity mismatch for ${what} from ${from}: expected ${expected}, got ${actual}`)
     return bytes
   }
-  const cached = await readCache(dir, key)
-  if (cached) return await check(cached, 'the cache')
+  const cached = await readBytes(dir, what, key, cache)
+  if (cached) return await check(cached, cache ? 'the store' : 'the cache')
   const url = await locate()
   const bytes = await check(await request(url, { ...options, as: 'bytes' }), url)
-  await writeCache(dir, key, bytes)
+  await writeBytes(dir, what, key, bytes, cache)
   return bytes
 }

package/src/git.js

@@ -54,5 +54,5 @@ export async function findGitCheckout(dir) {
   const directory = relative(dirs.root, start).split(sep).join('/')
   const github = await originGitHub(dirs.commonDir)
   const commit = await headCommit(dirs.gitDir, dirs.commonDir)
-  return { ...(github && { github }), ...(directory && { directory }), ...(github && { url: `https://github.com/${github}` }), ...(commit && { commit }) }
+  return { ...(github && { github }), directory, ...(github && { url: `https://github.com/${github}` }), ...(commit && { commit }) }
 }

package/src/npm/cacache.js

@@ -0,0 +1,44 @@
+import { Buffer } from 'node:buffer'
+import { createHash } from 'node:crypto'
+import { join } from 'node:path'
+
+import { readRegularFile } from '../cache.js'
+
+// npm's cache, cacache as npm 5 and later keep it: a response by the URL
+// it answered, under the key `make-fetch-happen:request-cache:<url>`. The
+// index has a file for each key's sha256, of lines `<sha1>\t<entry>`, each
+// entry's sha1 of its own JSON: the last whole one for the key counts, and
+// one with no integrity is one deleted. The content is filed by its
+// sha512, and checked against it.
+const KEY = 'make-fetch-happen:request-cache:'
+const digest = (algorithm, data, encoding) => createHash(algorithm).update(data).digest(encoding)
+const shard = (hex) => [hex.slice(0, 2), hex.slice(2, 4), hex.slice(4)]
+const decoder = new TextDecoder('utf-8')
+
+async function lastEntry(root, key) {
+  const bytes = await readRegularFile(join(root, 'index-v5', ...shard(digest('sha256', key, 'hex'))))
+  let entry = null
+  for (const line of bytes ? decoder.decode(bytes).split('\n') : []) {
+    const tab = line.indexOf('\t')
+    const json = line.slice(tab + 1)
+    if (tab < 0 || digest('sha1', json, 'hex') !== line.slice(0, tab)) continue
+    try {
+      const parsed = JSON.parse(json)
+      if (parsed?.key === key) entry = parsed
+    } catch {
+      // A line cut short, or written over: passed over, as npm passes it.
+    }
+  }
+  return entry
+}
+
+// The body npm keeps for `url`, beside what it kept of the response, or
+// null: none kept, one deleted, or content gone or not its sha512.
+export async function readNpmCached(root, url) {
+  const entry = await lastEntry(root, `${KEY}${url}`)
+  const integrity = typeof entry?.integrity === 'string' ? entry.integrity : ''
+  const sha512 = /(?:^|\s)sha512-([\d+/A-Za-z]+={0,2})(?=[\s?]|$)/u.exec(integrity)?.[1]
+  if (sha512 === undefined) return null
+  const body = await readRegularFile(join(root, 'content-v2', 'sha512', ...shard(Buffer.from(sha512, 'base64').toString('hex'))))
+  return body && digest('sha512', body, 'base64') === sha512 ? { body, metadata: entry.metadata } : null
+}

package/src/npm/registry.js

@@ -6,13 +6,16 @@ import { NPM_REGISTRY, buildUrl, request } from '../http.js'
 // Private packages are always scoped.
 export function withNpmToken(name, url, options = {}) {
   const method = options.method ?? 'GET'
-  assert.ok(method === 'GET' && `${new URL(url)}` === url && url.startsWith(`${NPM_REGISTRY}/${name}/`), `Unexpected request for ${name}: ${method} ${url}`)
+  const own = url === `${NPM_REGISTRY}/${name}` || url.startsWith(`${NPM_REGISTRY}/${name}/`)
+  assert.ok(method === 'GET' && `${new URL(url)}` === url && own, `Unexpected request for ${name}: ${method} ${url}`)
   const token = process.env.NPM_TOKEN
   return token && name.startsWith('@') ? { ...options, method, headers: { ...options.headers, Authorization: `Bearer ${token}` } } : options
 }
 
-export async function getDocument(method, name, spec) {
-  const url = buildUrl(NPM_REGISTRY, [...name.split('/'), spec])
+// A version's document, or with no `spec` the whole package's, every
+// version's and `time`.
+export async function getDocument(method, name, ...spec) {
+  const url = buildUrl(NPM_REGISTRY, [...name.split('/'), ...spec])
   const json = await request(url, withNpmToken(name, url, { as: 'json' }))
   assert.ok(json?.name === name, `${method}: the registry answered for ${show(json?.name)}, not ${name}`)
   return json

package/src/npm/repos.js

@@ -1,6 +1,6 @@
 import assert from 'node:assert/strict'
 
-import { assertNames, assertPackageName, assertion, isRepo } from '../args.js'
+import { assertNames, assertPackageName, assertion, isRepo, optional } from '../args.js'
 import { readRecord, writeRecord } from '../cache.js'
 import { isNotFound, recover } from '../http.js'
 import { assertRepoDirectory, getRepo, isRepoDirectory } from '../package.js'
@@ -12,8 +12,10 @@ const CONCURRENCY = 8
 // Stamped on each entry, and raised when getRepo would answer differently:
 // 2 is `repository` taking precedence over `bugs`; 3 the shorthand dropping
 // a `.git`, URLs read past whitespace, and `repository.directory`'s `\` read
-// as `/`.
-const VERSION = 3
+// as `/`; 4 a declared root answered as `''`, no directory declared left
+// out (unknown) rather than read as the root, and a declared directory's
+// empty and `.` parts dropped.
+const VERSION = 4
 
 // A slug a lookup gives: getRepo answers no repo ending in `.git`, the
 // suffix it drops (withoutDotGit; `.GIT` it keeps, and so does this).
@@ -33,23 +35,23 @@ export async function getGitHub(name) {
 
 // An entry under another VERSION was resolved by other rules, and may
 // name a repo getRepo no longer would (a stale tracker's, or a shorthand's
-// with its `.git`): a miss, as is one naming a repo or a directory no
-// lookup gives, or without `directory`, which would read as a package at
-// the repo root.
+// with its `.git`, or the root where a package declared no directory): a
+// miss, as is one naming a repo or a directory no lookup gives. One without
+// `directory` is a package whose place in its repo is unknown.
 export async function readPackageRepoCache(name) {
   assertPackageName('readPackageRepoCache', 'name', name)
   const entry = await readRecord(DIR, name)
-  if (entry?.v !== VERSION || !isLookedUpRepo(entry.github) || !isRepoDirectory(entry.directory)) return null
-  return { github: entry.github, ...(entry.directory && { directory: entry.directory }) }
+  if (entry?.v !== VERSION || !isLookedUpRepo(entry.github) || (entry.directory !== undefined && !isRepoDirectory(entry.directory))) return null
+  return { github: entry.github, ...(entry.directory !== undefined && { directory: entry.directory }) }
 }
 
 // Only resolved repos are cached: a 404, a rate limit and a package with
 // no repo link fail alike.
-export async function writePackageRepoCache(name, github, directory = '') {
+export async function writePackageRepoCache(name, github, directory) {
   assertPackageName('writePackageRepoCache', 'name', name)
   assertLookedUpRepo('writePackageRepoCache', 'github', github)
-  assertRepoDirectory('writePackageRepoCache', 'directory', directory)
-  return await writeRecord(DIR, name, { v: VERSION, github, directory })
+  optional(assertRepoDirectory)('writePackageRepoCache', 'directory', directory)
+  return await writeRecord(DIR, name, { v: VERSION, github, ...(directory !== undefined && { directory }) })
 }
 
 export async function resolvePackageRepos(packageNames, options = {}) {
@@ -60,11 +62,11 @@ export async function resolvePackageRepos(packageNames, options = {}) {
 
 // Through the cache. Null for a package the registry does not have, or
 // one naming no GitHub repo; any other failure throws.
-export async function lookUpPackageRepo(name) {
+async function lookUpPackageRepo(name) {
   const stored = await readPackageRepoCache(name)
   if (stored) return stored
   const { github, directory } = await fetchRepo('lookUpPackageRepo', name).catch(recover(isNotFound, {}))
   if (!github) return null
   await writePackageRepoCache(name, github, directory)
-  return { github, ...(directory && { directory }) }
+  return { github, ...(directory !== undefined && { directory }) }
 }

package/src/npm/tarball.js

@@ -1,39 +1,16 @@
-import assert from 'node:assert/strict'
 import { Buffer } from 'node:buffer'
 import { homedir } from 'node:os'
 import { join, resolve } from 'node:path'
 
-import { assertArgs, assertPackage, assertion, matches, show } from '../args.js'
+import { assertArgs, assertPackage, assertPackageName, isPlainObject, isSha, optional } from '../args.js'
+import { assertCache } from '../cache.js'
 import { verifiedDownload } from '../download.js'
-import { NPM_REGISTRY, buildUrl } from '../http.js'
+import { NPM_REGISTRY, decode } from '../http.js'
+import { readNpmCached } from './cacache.js'
 import { getDocument, withNpmToken } from './registry.js'
+import { checkedDist, getVersion, matchingDist } from './versions.js'
 
 const DIR = 'npm/tarballs' // No expiry: the registry never takes a version twice.
-// One sha512 and nothing else, as the registry writes it: a sha1, a second
-// hash or an option is refused rather than trusted. 64 bytes leave the
-// last character before `==` two bits, so only A, Q, g or w is canonical.
-const assertIntegrity = assertion('"sha512-" and a base64 sha512', matches(/^sha512-[\dA-Za-z+/]{85}[AQgw]==$/u))
-
-// `{ tarball, integrity }` and nothing else, the tarball exactly the
-// registry's own URL for that version. Each field is read once, and only
-// as the object's own, into a copy that is checked and used from then on:
-// the caller's object can change while a request is out, and a field on
-// its prototype is not one it has.
-function checkedDist(method, name, version, dist) {
-  assertArgs(method, dist, { tarball: null, integrity: null }, 'dist')
-  const [tarball, integrity] = ['tarball', 'integrity'].map((key) => (Object.hasOwn(dist, key) ? dist[key] : undefined))
-  assertIntegrity(method, 'dist.integrity', integrity)
-  const segments = name.split('/')
-  const expected = buildUrl(NPM_REGISTRY, [...segments, '-', `${segments.at(-1)}-${version}.tgz`])
-  assert.ok(tarball === expected, `${method}: dist.tarball must be ${expected}, got ${show(tarball)}`)
-  return { tarball, integrity }
-}
-
-async function getDist(method, name, version) {
-  const json = await getDocument(method, name, version)
-  assert.ok(json.version === version, `${method}: the registry answered for ${name}@${show(json.version)}, not ${name}@${version}`)
-  return checkedDist(method, name, version, { tarball: json.dist?.tarball, integrity: json.dist?.integrity })
-}
 
 // Where npm keeps its cache, short of an .npmrc moving it: npm_config_cache,
 // as npm sets it for what it runs, else npm's default. On Windows, npm 4
@@ -53,23 +30,108 @@ function localPaths(name, version, integrity) {
   return [...npm, join(homedir(), '.audit/cache/tgz', `${name.replace(/^@/u, '').replace('/', ':')}-${version}.tgz`)]
 }
 
-export async function getMeta(name, version) {
+// verifyDist's and getTarball's options: `cache` alone.
+const assertOptions = (method, options) => assertArgs(method, options, { cache: optional(assertCache) })
+
+const own = (object, key) => (Object.hasOwn(object, key) ? object[key] : undefined)
+const stringOr = (value) => (typeof value === 'string' ? value : undefined)
+
+// A string as it is, or an object's strings under `keys`, the rest undefined:
+// none where it is neither, or names none of them.
+function linkOf(value, keys) {
+  if (typeof value === 'string') return value
+  if (!isPlainObject(value)) return undefined
+  const link = Object.fromEntries(keys.map((key) => [key, stringOr(own(value, key))]))
+  return keys.some((key) => link[key] !== undefined) ? link : undefined
+}
+
+const mediaType = (value) => (typeof value === 'string' ? value.split(';')[0].trim().toLowerCase() : undefined)
+
+function jsonOf(bytes, from) {
+  try {
+    return JSON.parse(decode(bytes, from))
+  } catch {
+    return undefined
+  }
+}
+
+// A version as npm's own cache keeps it, where it keeps its name's full
+// packument, as npm 10 and later fetch it, whose entry for the version
+// matches `given` (matchingDist): that entry is the version's document. An
+// abbreviated packument, as earlier npm fetches, has none of the rest
+// getMeta answers, and is passed over.
+async function npmCachedVersion(name, version, given) {
+  const url = `${NPM_REGISTRY}/${name.replace('/', '%2f')}`
+  for (const root of npmCacheDirs()) {
+    const cached = await readNpmCached(join(root, '_cacache'), url)
+    if (!cached || mediaType(cached.metadata?.resHeaders?.['content-type']) !== 'application/json') continue
+    const packument = jsonOf(cached.body, url)
+    const json = packument?.name === name && isPlainObject(packument.versions) ? own(packument.versions, version) : undefined
+    const dist = matchingDist('getMeta', name, version, json, given)
+    if (dist) return { dist, json }
+  }
+  return undefined
+}
+
+// The version's document: with the `dist` the caller has, npm's own cache's
+// where it matches (npmCachedVersion), and neither `cache` nor the cache
+// set is read or written; else as getVersion has it. Beside its dist, the
+// document's gitHead where that is a full commit id, and its repository,
+// homepage and bugs, in the shapes a package.json gives them. Each but
+// dist is left out where there is none.
+export async function getMeta(name, version, options = {}) {
   assertPackage('getMeta', name, version)
-  return { name, version, dist: await getDist('getMeta', name, version) }
+  assertArgs('getMeta', options, { dist: null, cache: optional(assertCache) })
+  const given = options.dist === undefined ? undefined : checkedDist('getMeta', name, version, options.dist)
+  const { dist, json } = (given && await npmCachedVersion(name, version, given)) ?? await getVersion('getMeta', name, version, given, options.cache)
+  const about = {
+    gitHead: isSha(own(json, 'gitHead')) ? json.gitHead : undefined,
+    repository: linkOf(own(json, 'repository'), ['type', 'url', 'directory']),
+    homepage: stringOr(own(json, 'homepage')),
+    bugs: linkOf(own(json, 'bugs'), ['url', 'email']),
+  }
+  return { name, version, dist, ...Object.fromEntries(Object.entries(about).filter(([, value]) => value !== undefined)) }
+}
+
+// Only the whole package's document has `time`: megabytes for some.
+const PUBLISHED = /^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d)(?:\.(\d{1,3}))?Z$/u
+const record = (value) => (isPlainObject(value) ? value : {})
+
+// A time toISOString writes back as given, its milliseconds padded: none
+// Date.parse moves, as it moves 2018-02-29 to March.
+function readTime(value) {
+  const match = typeof value === 'string' ? PUBLISHED.exec(value) : null
+  if (match === null) return undefined
+  const iso = `${match[1]}.${(match[2] ?? '').padEnd(3, '0')}Z`
+  const parsed = Date.parse(iso)
+  return Number.isFinite(parsed) && new Date(parsed).toISOString() === iso ? iso : undefined
+}
+
+export async function getPublishTimes(name) {
+  assertPackageName('getPublishTimes', 'name', name)
+  const json = await getDocument('getPublishTimes', name)
+  const time = record(json.time)
+  const times = new Map()
+  for (const version of Object.keys(record(json.versions))) {
+    const at = readTime(Object.hasOwn(time, version) ? time[version] : undefined)
+    if (at !== undefined) times.set(version, at)
+  }
+  return times
 }
 
-export async function verifyDist(name, version, dist) {
+export async function verifyDist(name, version, dist, options = {}) {
   assertPackage('verifyDist', name, version)
-  const given = checkedDist('verifyDist', name, version, dist)
-  const { integrity } = await getDist('verifyDist', name, version)
-  assert.ok(given.integrity === integrity, `verifyDist: ${name}@${version} is ${integrity} on the registry, not ${given.integrity}`)
+  assertOptions('verifyDist', options)
+  await getVersion('verifyDist', name, version, checkedDist('verifyDist', name, version, dist), options.cache)
 }
 
 // Without `dist`, the version document is read every time, cache or not:
 // bytes from a cache are checked against the registry's integrity, never
 // against anything a cache itself holds.
-export async function getTarball(name, version, dist) {
+export async function getTarball(name, version, dist, options = {}) {
   assertPackage('getTarball', name, version)
-  const { tarball, integrity } = dist === undefined ? await getDist('getTarball', name, version) : checkedDist('getTarball', name, version, dist)
-  return await verifiedDownload({ method: 'getTarball', dir: DIR, what: `${name}@${version}`, ext: 'tgz', algorithm: 'sha512', expected: integrity, local: localPaths(name, version, integrity), ours: true, locate: () => tarball, options: withNpmToken(name, tarball) })
+  assertOptions('getTarball', options)
+  const { cache } = options
+  const { tarball, integrity } = dist === undefined ? (await getVersion('getTarball', name, version, undefined, cache)).dist : checkedDist('getTarball', name, version, dist)
+  return await verifiedDownload({ method: 'getTarball', dir: DIR, what: `${name}@${version}`, ext: 'tgz', algorithm: 'sha512', expected: integrity, local: localPaths(name, version, integrity), ours: true, cache, locate: () => tarball, options: withNpmToken(name, tarball) })
 }

package/src/npm/versions.js

@@ -0,0 +1,97 @@
+import assert from 'node:assert/strict'
+
+import { assertArgs, assertion, matches, show } from '../args.js'
+import { readJSON, writeJSON } from '../cache.js'
+import { NPM_REGISTRY, buildUrl, isNotFound, recover } from '../http.js'
+import { getDocument } from './registry.js'
+
+const DIR = 'npm/versions' // No expiry: the registry never takes a version twice, nor changes its dist.
+// One sha512 and nothing else, as the registry writes it: a sha1, a second
+// hash or an option is refused rather than trusted. 64 bytes leave the
+// last character before `==` two bits, so only A, Q, g or w is canonical.
+const assertIntegrity = assertion('"sha512-" and a base64 sha512', matches(/^sha512-[\dA-Za-z+/]{85}[AQgw]==$/u))
+
+// `{ tarball, integrity }` and nothing else, the tarball exactly the
+// registry's own URL for that version. Each field is read once, and only
+// as the object's own, into a copy that is checked and used from then on:
+// the caller's object can change while a request is out, and a field on
+// its prototype is not one it has.
+export function checkedDist(method, name, version, dist) {
+  assertArgs(method, dist, { tarball: null, integrity: null }, 'dist')
+  const [tarball, integrity] = ['tarball', 'integrity'].map((key) => (Object.hasOwn(dist, key) ? dist[key] : undefined))
+  assertIntegrity(method, 'dist.integrity', integrity)
+  const segments = name.split('/')
+  const expected = buildUrl(NPM_REGISTRY, [...segments, '-', `${segments.at(-1)}-${version}.tgz`])
+  assert.ok(tarball === expected, `${method}: dist.tarball must be ${expected}, got ${show(tarball)}`)
+  return { tarball, integrity }
+}
+
+const distOf = (method, name, version, json) => checkedDist(method, name, version, { tarball: json.dist?.tarball, integrity: json.dist?.integrity })
+
+// Whether a version document is one to keep, or to take as kept: for that
+// name and version, with a dist that passes.
+function isKeepable(method, name, version, json) {
+  if (json?.name !== name || json.version !== version) return false
+  try {
+    distOf(method, name, version, json)
+    return true
+  } catch {
+    return false
+  }
+}
+
+// A version's document as `cache` keeps it (readJSON), or none.
+async function keptDocument(method, name, version, cache) {
+  const json = await readJSON(DIR, `${name}@${version}`, cache)
+  return isKeepable(method, name, version, json) ? json : undefined
+}
+
+// The registry's document, kept whole in `cache` once it is keepable; one
+// whose dist is refused is answered all the same.
+async function keepDocument(method, name, version, json, cache) {
+  assert.ok(json.version === version, `${method}: the registry answered for ${name}@${show(json.version)}, not ${name}@${version}`)
+  if (isKeepable(method, name, version, json)) await writeJSON(DIR, `${name}@${version}`, json, cache)
+  return json
+}
+
+const fetchDocument = async (method, name, version, cache) => await keepDocument(method, name, version, await getDocument(method, name, version), cache)
+
+// A version's document, kept or else fetched, for what is not an
+// integrity: a package's repo (advisories.js), as trusted as setCacheDir's
+// repo records are. Null where the registry does not have that version:
+// its 404 alone, never a failure of the cache's.
+export async function getVersionDocument(method, name, version, cache) {
+  const kept = await keptDocument(method, name, version, cache)
+  if (kept) return kept
+  const json = await getDocument(method, name, version).catch(recover(isNotFound, null))
+  return json && await keepDocument(method, name, version, json, cache)
+}
+
+// The dist of a version document a cache answers, where it is that
+// version's, its dist passes, and its tarball and integrity are `given`'s
+// (checkedDist): a cache never supplies a dist, only confirms one.
+export function matchingDist(method, name, version, json, given) {
+  if (!isKeepable(method, name, version, json)) return undefined
+  const dist = distOf(method, name, version, json)
+  return dist.integrity === given.integrity && dist.tarball === given.tarball ? dist : undefined
+}
+
+// A version's document, and its dist checked. A kept document is read only
+// for a caller that already has a dist, `given`, and answers with no
+// request where it matches (matchingDist). Else, or without one, the
+// registry is asked, and its document, kept in place of any other, throws
+// where it does not.
+export async function getVersion(method, name, version, given, cache) {
+  if (given !== undefined) {
+    const kept = await readJSON(DIR, `${name}@${version}`, cache)
+    const dist = matchingDist(method, name, version, kept, given)
+    if (dist) return { dist, json: kept }
+  }
+  const json = await fetchDocument(method, name, version, cache)
+  const dist = distOf(method, name, version, json)
+  if (given !== undefined) {
+    assert.ok(dist.integrity === given.integrity, `${method}: ${name}@${version} is ${dist.integrity} on the registry, not ${given.integrity}`)
+    assert.ok(dist.tarball === given.tarball, `${method}: ${name}@${version} is at ${dist.tarball} on the registry, not ${given.tarball}`)
+  }
+  return { dist, json }
+}

package/src/package.js

@@ -38,8 +38,16 @@ function repoSubdirectory(value) {
 
 // `repository.directory` is a path its author may write with Windows' `\`.
 // A tree path is `/`-separated: a `\` is a name's own only in a homepage's
-// (`%5C`), which is the path itself.
-const declaredDirectory = (value) => repoSubdirectory(str(value).replaceAll('\\', '/'))
+// (`%5C`), which is the path itself. Its empty and `.` parts are dropped
+// (`./a//b/.` is `a/b`), and one of nothing else (`./`, `/`, `/.`, `.`, `''`)
+// declares the repo's root, `''`; one with a `..` part names none, whatever
+// it would come to. A package.json that declares none says nothing of where
+// in the repo it sits.
+function declaredDirectory(value) {
+  if (typeof value !== 'string') return undefined
+  const parts = value.replaceAll('\\', '/').split('/').filter((part) => part !== '' && part !== '.')
+  return parts.length === 0 ? '' : repoSubdirectory(parts.join('/'))
+}
 
 export const isRepoDirectory = (value) => typeof value === 'string' && (value === '' || repoSubdirectory(value) === value)
 export const assertRepoDirectory = assertion('a path inside the repository', isRepoDirectory)
@@ -54,7 +62,7 @@ export function getRepo(pkg) {
   // `bugs` tracker can be left pointing at a former owner or misspelt.
   const github = [declared, repoIn(bugsRegex, urlOf(bugs)), homepageRepo].find(isRepo)
   if (github === undefined) return {}
-  const directory = (sameName(declared, github) && declaredDirectory(repository?.directory))
-    || (sameName(homepageRepo, github) && repoSubdirectory(decodePath(homepage.directory)))
-  return { github, ...(directory && { directory }), url: `https://github.com/${github}` }
+  const directory = (sameName(declared, github) ? declaredDirectory(repository?.directory) : undefined)
+    ?? (sameName(homepageRepo, github) ? repoSubdirectory(decodePath(homepage.directory)) : undefined)
+  return { github, ...(directory !== undefined && { directory }), url: `https://github.com/${github}` }
 }