@preventive/upstream 1.0.0-alpha.9 → 1.0.1

package/src/github/read.js

@@ -1,9 +1,10 @@
 import assert from 'node:assert/strict'
 import { Buffer } from 'node:buffer'
 
-import { assertArgs, assertGhsa, assertLogin, assertNumber, assertPath, assertRef, assertRepo, assertSha, assertTagName, assertTreeId, isSha, isSha1, isTagName, optional, sameName, show } from '../args.js'
+import { assertArgs, assertBoolean, assertGhsa, assertLogin, assertNumber, assertPath, assertRef, assertRepo, assertSha, assertTagName, assertTreeId, isRepo, isSha, isSha1, isTagName, optional, sameName, show } from '../args.js'
 import { verifiedDownload } from '../download.js'
-import { decode, encodeSegment, requestWithHeaders } from '../http.js'
+import { HttpError, decode, encodeSegment, requestWithHeaders } from '../http.js'
+import { FORMATS, lockfileOf, projectFormats } from '../lockfiles.js'
 import { gitTreeOfListing, objectId } from '../tree.js'
 import { api, bindMethods, call, clientHeaders, isGone, repoApi } from './client.js'
 
@@ -48,10 +49,29 @@ async function listUserRepos(headers, options) {
   return await paginate('listUserRepos', headers, (paging) => api(['user', 'repos'], { ...paging, sort: 'full_name' }), options?.maxPages)
 }
 
+// Where GitHub's API sends a repository renamed or transferred: to that
+// repository by its id.
+const REPO_BY_ID = /^https:\/\/api\.github\.com\/repositories\/([1-9]\d{0,14})$/u
+const REDIRECTS = new Set([301, 302, 307, 308])
+
+// With `followRedirect`, GitHub's redirect for a repository renamed or
+// transferred is followed, once, on its API alone, and the answer held to
+// the id it names rather than to the name asked; any other redirect, or a
+// second, is refused as without it.
 async function getRepo(headers, options) {
-  assertArgs('getRepo', options, { repo: assertRepo })
-  const info = await call(headers, repoApi(options.repo))
-  assert.ok(sameName(info?.full_name, options.repo), `getRepo: answered for ${show(info?.full_name)}, not ${options.repo}`)
+  assertArgs('getRepo', options, { repo: assertRepo, followRedirect: optional(assertBoolean) })
+  const { repo, followRedirect } = options
+  let info
+  try {
+    info = await call(headers, repoApi(repo))
+  } catch (err) {
+    const id = followRedirect && err instanceof HttpError && REDIRECTS.has(err.status) ? REPO_BY_ID.exec(err.headers.get('location') ?? '')?.[1] : undefined
+    if (id === undefined) throw err
+    const moved = await call(headers, api(['repositories', id]))
+    assert.ok(moved?.id === Number(id) && isRepo(moved.full_name), `getRepo: ${repo} redirected to repository ${id}, answered for ${show(moved?.id)}, ${show(moved?.full_name)}`)
+    return moved
+  }
+  assert.ok(sameName(info?.full_name, repo), `getRepo: answered for ${show(info?.full_name)}, not ${repo}`)
   return info
 }
 
@@ -175,6 +195,71 @@ async function listRepoDir(headers, options) {
   return await listTree('listRepoDir', headers, options.repo, tree)
 }
 
+// A tree and every tree under it in one listing, their entries by paths
+// from it, or null where GitHub cut it short, as it does past 100,000
+// entries or 7 MB. Each tree in it, the top one too, has to hash from the
+// entries listed in it, so none is left out, changed or added: an entry is
+// filed under the path before its last `/`, a tree's own with a `/` after
+// it, so `/x` or `a//x` is in no tree.
+async function listTreeDeep(method, headers, repo, tree) {
+  const listing = await call(headers, repoApi(repo, ['git', 'trees', tree], { recursive: 1 }))
+  if (listing?.truncated === true) return null
+  const entries = Array.isArray(listing?.tree) && listing.tree.every((entry) => typeof entry?.path === 'string') ? listing.tree : []
+  const dirs = new Map([['', { sha: tree, listed: [] }], ...entries.filter((entry) => entry.type === 'tree').map((entry) => [`${entry.path}/`, { sha: entry.sha, listed: [] }])])
+  const filed = entries.every(({ path, mode, type, sha }) => {
+    const at = path.lastIndexOf('/') + 1
+    const dir = dirs.get(path.slice(0, at))
+    dir?.listed.push({ path: path.slice(at), mode, type, sha })
+    return dir !== undefined
+  })
+  assert.ok(filed && [...dirs.values()].every(({ sha, listed }) => gitTreeOfListing(listed) === sha), `${method}: GitHub's listing of tree ${tree} in ${repo}, with the trees under it, is not that tree`)
+  return entries
+}
+
+// Each file in `tree`, which is at `base` in the repo, by its path, its
+// name and its blob's id: from the tree's listing with every tree under
+// it, or, where GitHub cuts that short, from its own and then each
+// subtree's in turn, as GitHub advises for a large tree. A node_modules,
+// of packages installed, is not the repo's own.
+async function* filesIn(method, headers, repo, tree, base) {
+  const deep = await listTreeDeep(method, headers, repo, tree)
+  for (const { path, mode, type, sha } of deep ?? await listTree(method, headers, repo, tree)) {
+    const names = path.split('/')
+    if (names.includes('node_modules')) continue
+    if (type === 'blob' && mode !== '120000') yield { path: `${base}${path}`, name: names.at(-1), sha }
+    else if (deep === null && type === 'tree') yield* filesIn(method, headers, repo, sha, `${base}${path}/`)
+  }
+}
+
+const byPath = (a, b) => (a.path < b.path ? -1 : 1)
+
+async function findRepoLockfiles(headers, options) {
+  assertArgs('findRepoLockfiles', options, { repo: assertRepo, sha: assertSha, directory: optional(assertPath) })
+  const { repo, sha, directory } = options
+  const tree = await treeAt('findRepoLockfiles', headers, options)
+  const lockfiles = []
+  const projects = new Map()
+  const files = new Map()
+  const locked = []
+  for await (const { path, name, sha: blob } of filesIn('findRepoLockfiles', headers, repo, tree, directory === undefined ? '' : `${directory}/`)) {
+    files.set(path, blob)
+    const lockfile = lockfileOf(name)
+    const formats = projectFormats(name)
+    if (lockfile === undefined && formats.length === 0) continue
+    assertPath('findRepoLockfiles', `${repo}@${sha}'s ${lockfile === undefined ? 'project file' : 'lockfile'}`, path)
+    if (lockfile !== undefined) lockfiles.push({ path, format: lockfile.format, sha: blob })
+    if (lockfile?.locks !== undefined) locked.push({ path: `${path.slice(0, -name.length)}${lockfile.locks}`, format: lockfile.format })
+    if (formats.length > 0) projects.set(path, { path, formats, sha: blob })
+  }
+  // A file a lockfile beside it locks is a project file of that format too.
+  for (const { path, format } of locked) {
+    if (!files.has(path)) continue
+    const formats = projects.get(path)?.formats ?? []
+    projects.set(path, { path, formats: Object.keys(FORMATS).filter((known) => known === format || formats.includes(known)), sha: files.get(path) })
+  }
+  return { lockfiles: lockfiles.toSorted(byPath), projects: [...projects.values()].toSorted(byPath) }
+}
+
 async function getPullRequest(headers, options) {
   assertArgs('getPullRequest', options, { repo: assertRepo, number: assertNumber })
   const { repo, number } = options
@@ -219,5 +304,5 @@ async function listRepoAdvisories(headers, options) {
   return (await Array.fromAsync(cursorPages('listRepoAdvisories', headers, pageUrl))).flat()
 }
 
-export const readMethods = { getCurrentUser, listUserRepos, getRepo, getRepoHead, getRepoTag, listRepoTags, getRepoFile, getRepoTarball, getRepoTreeTarball, getRepoTreeId, listRepoDir, getPullRequest, getCollaboratorPermission, getAdvisory, listRepoAdvisories }
+export const readMethods = { getCurrentUser, listUserRepos, getRepo, getRepoHead, getRepoTag, listRepoTags, getRepoFile, getRepoTarball, getRepoTreeTarball, getRepoTreeId, listRepoDir, findRepoLockfiles, getPullRequest, getCollaboratorPermission, getAdvisory, listRepoAdvisories }
 export const createClient = (options) => bindMethods(clientHeaders('createClient', options, true), readMethods)

package/src/http.js

@@ -32,11 +32,13 @@ const LIMITS = {
 }
 const decoder = new TextDecoder('utf-8', { fatal: true })
 
+// `headers` are the response's, where there was one to read them from.
 export class HttpError extends Error {
-  constructor(status, message) {
+  constructor(status, message, headers) {
     super(message)
     this.name = 'HttpError'
     this.status = status
+    if (headers !== undefined) this.headers = headers
   }
 }
 
@@ -119,7 +121,7 @@ export async function requestWithHeaders(url, options) {
   if (!res.ok) {
     // A stream decode holds back a character cut at the limit rather than refusing it.
     const text = await readBody(res, 4096, { truncate: true }).then((bytes) => new TextDecoder('utf-8', { fatal: true }).decode(bytes, { stream: true })).catch(() => '')
-    throw new HttpError(res.status, `${options.method ?? 'GET'} ${url} ${res.status}: ${printable(text)}`)
+    throw new HttpError(res.status, `${options.method ?? 'GET'} ${url} ${res.status}: ${printable(text)}`, res.headers)
   }
   return { body: parse(await readBody(res, LIMITS[options.as].bytes), url, options.as), headers: res.headers }
 }

package/src/lockfiles.js

@@ -0,0 +1,113 @@
+// Each tool that writes a lockfile, by the format of it: the names it
+// writes one under, and those of the project files it locks, each a name
+// as git has it, `cargo.lock` not Cargo's, or a pattern. A lockfile's
+// pattern names in a `locks` group the file beside it that it locks, where
+// no name of that file's own tells it is one.
+export const FORMATS = {
+  // pnpm-lock.<branch>.yaml with git-branch-lockfile; shrinkwrap.yaml before
+  // pnpm 3. package.yaml and package.json5 in place of package.json, and
+  // .pnpmfile's hooks, which change what is locked.
+  pnpm: {
+    lockfiles: ['pnpm-lock.yaml', /^pnpm-lock\..+\.yaml$/u, 'shrinkwrap.yaml'],
+    projects: ['package.json', 'package.yaml', 'package.json5', 'pnpm-workspace.yaml', '.pnpmfile.cjs', '.pnpmfile.mjs'],
+  },
+  // npm-shrinkwrap.json is npm's own format, which npm reads before a
+  // package-lock.json beside it.
+  npm: { lockfiles: ['package-lock.json', 'npm-shrinkwrap.json'], projects: ['package.json'] },
+  // Of Yarn 1 or a later Yarn, which only the text tells apart. .yarnrc.yml
+  // has Yarn 4's catalogs.
+  yarn: { lockfiles: ['yarn.lock'], projects: ['package.json', '.yarnrc.yml'] },
+  // bun.lockb is Bun's binary one, before Bun 1.2.
+  bun: { lockfiles: ['bun.lock', 'bun.lockb'], projects: ['package.json'] },
+  deno: { lockfiles: ['deno.lock'], projects: ['deno.json', 'deno.jsonc', 'package.json'] },
+  // <script>.py.lock, of `uv lock --script`, which locks the script.
+  uv: { lockfiles: ['uv.lock', /^(?<locks>.+\.py)\.lock$/u], projects: ['pyproject.toml'] },
+  poetry: { lockfiles: ['poetry.lock'], projects: ['pyproject.toml'] },
+  // PEP 751's.
+  pylock: { lockfiles: ['pylock.toml', /^pylock\.[^.]+\.toml$/u], projects: ['pyproject.toml'] },
+  pipenv: { lockfiles: ['Pipfile.lock'], projects: ['Pipfile'] },
+  pdm: { lockfiles: ['pdm.lock'], projects: ['pyproject.toml'] },
+  pixi: { lockfiles: ['pixi.lock'], projects: ['pixi.toml', 'pyproject.toml'] },
+  // A unified lockfile of another name has to end in `.conda-lock.yml`; the
+  // explicit and env ones are conda-<platform>.lock and .lock.yml. A
+  // conda-build recipe, meta.yaml, is a source too.
+  'conda-lock': {
+    lockfiles: ['conda-lock.yml', 'conda-lock.yaml', /^.+\.conda-lock\.yml$/u, /^conda-[\da-z]+-[\da-z]+\.lock(?:\.yml)?$/u],
+    projects: ['environment.yml', 'environment.yaml', 'meta.yaml', 'pyproject.toml'],
+  },
+  // A gemspec, whose dependencies a Gemfile's `gemspec` takes.
+  bundler: { lockfiles: ['Gemfile.lock', 'gems.locked'], projects: ['Gemfile', 'gems.rb', /^.+\.gemspec$/u] },
+  composer: { lockfiles: ['composer.lock'], projects: ['composer.json'] },
+  cargo: { lockfiles: ['Cargo.lock'], projects: ['Cargo.toml'] },
+  soldeer: { lockfiles: ['soldeer.lock'], projects: ['soldeer.toml', 'foundry.toml'] },
+  foundry: { lockfiles: ['foundry.lock'], projects: ['foundry.toml'] },
+  // Each name CocoaPods reads a Podfile under, and a podspec, whose
+  // dependencies a Podfile's `podspec` takes.
+  cocoapods: { lockfiles: ['Podfile.lock'], projects: ['Podfile', 'Podfile.rb', 'CocoaPods.podfile', 'CocoaPods.podfile.yaml', /^.+\.podspec(?:\.json)?$/u] },
+  swiftpm: { lockfiles: ['Package.resolved'], projects: ['Package.swift', /^Package@swift-\d+(?:\.\d+){0,2}\.swift$/u] },
+  carthage: { lockfiles: ['Cartfile.resolved'], projects: ['Cartfile', 'Cartfile.private'] },
+  // go.sum has the checksums; go.mod, the versions, is locked as it is. A
+  // workspace's go.work.sum has those its modules' go.sum do not.
+  go: { lockfiles: ['go.sum', 'go.work.sum'], projects: ['go.mod', 'go.work'] },
+  dep: { lockfiles: ['Gopkg.lock'], projects: ['Gopkg.toml'] },
+  // packages.<project>.lock.json, for one of projects that share a
+  // directory; Directory.Packages.props has the versions of Central
+  // Package Management, which the project files then leave out.
+  nuget: { lockfiles: ['packages.lock.json', /^packages\..+\.lock\.json$/u], projects: [/^.+\.(?:cs|fs|vb)proj$/u, 'Directory.Packages.props'] },
+  paket: { lockfiles: ['paket.lock'], projects: ['paket.dependencies'] },
+  // Any *.lockfile: Gradle's own, the legacy one of each configuration, or
+  // one a build names. Version catalogs, libs.versions.toml and the like.
+  gradle: { lockfiles: [/^.+\.lockfile$/u], projects: ['build.gradle', 'build.gradle.kts', 'settings.gradle', 'settings.gradle.kts', /^.+\.versions\.toml$/u] },
+  // pubspec_overrides.yaml overrides pubspec.yaml's dependency_overrides,
+  // workspace and resolution.
+  pub: { lockfiles: ['pubspec.lock'], projects: ['pubspec.yaml', 'pubspec_overrides.yaml'] },
+  mix: { lockfiles: ['mix.lock'], projects: ['mix.exs'] },
+  // rebar.config.script, where there is one, makes the configuration.
+  rebar: { lockfiles: ['rebar.lock'], projects: ['rebar.config', 'rebar.config.script'] },
+  julia: { lockfiles: ['Manifest.toml', 'JuliaManifest.toml', /^(?:Julia)?Manifest-v\d+\.\d+\.toml$/u], projects: ['Project.toml', 'JuliaProject.toml'] },
+  // Each of a configuration's name, stack.yaml or another, and `.lock`.
+  stack: { lockfiles: [/^(?<locks>.+\.ya?ml)\.lock$/u], projects: ['stack.yaml'] },
+  // Each of a project file's name, cabal.project or another, and `.freeze`.
+  cabal: { lockfiles: [/^(?<locks>.+)\.freeze$/u], projects: ['cabal.project', /^.+\.cabal$/u] },
+  shards: { lockfiles: ['shard.lock'], projects: ['shard.yml', 'shard.override.yml'] },
+  opam: { lockfiles: ['opam.locked', /^.+\.opam\.locked$/u], projects: ['opam', /^.+\.opam$/u] },
+  nix: { lockfiles: ['flake.lock'], projects: ['flake.nix'] },
+  conan: { lockfiles: ['conan.lock'], projects: ['conanfile.txt', 'conanfile.py'] },
+  // <name>.MODULE.bazel, which MODULE.bazel's include() takes.
+  bazel: { lockfiles: ['MODULE.bazel.lock'], projects: ['MODULE.bazel', /^[^.].*\.MODULE\.bazel$/u] },
+  helm: { lockfiles: ['Chart.lock'], projects: ['Chart.yaml'] },
+  // Of no project file, but the library it is made from.
+  renv: { lockfiles: ['renv.lock'], projects: [] },
+  // Of the configuration in the directory, each of its .tf files, a child
+  // module's too, of which there is none; OpenTofu's .tofu files, and a
+  // Stack's components.
+  terraform: { lockfiles: ['.terraform.lock.hcl'], projects: [/^.+\.(?:tf|tofu)(?:\.json)?$/u, /^.+\.tfcomponent\.hcl$/u] },
+}
+
+const TOOLS = Object.entries(FORMATS)
+const KNOWN = TOOLS.flatMap(([, { lockfiles, projects }]) => [...lockfiles, ...projects])
+
+// A name of none of them, as most in a repository are, is passed over at
+// once, by a set of the names and one pattern of all the patterns.
+const NAMES = new Set(KNOWN.filter((known) => typeof known === 'string'))
+const PATTERN = new RegExp(KNOWN.filter((known) => typeof known !== 'string').map(({ source }) => `(?:${source})`).join('|'), 'u')
+const isKnown = (name) => NAMES.has(name) || PATTERN.test(name)
+
+// Whether `name` is one of `names`, each a name or a pattern.
+const isOf = (names, name) => names.some((known) => (typeof known === 'string' ? known === name : known.test(name)))
+
+// The lockfile a file of this name is, `{ format, locks }`, `locks` the name
+// of the file beside it that it locks where its pattern names one; or
+// undefined.
+export function lockfileOf(name) {
+  for (const [format, { lockfiles }] of isKnown(name) ? TOOLS : []) {
+    for (const known of lockfiles) {
+      const match = typeof known === 'string' ? known === name && {} : known.exec(name)
+      if (match) return { format, locks: match.groups?.locks }
+    }
+  }
+}
+
+// The formats of the lockfiles that lock a project of a file of this name,
+// in FORMATS' order, none where it is no project file.
+export const projectFormats = (name) => (isKnown(name) ? TOOLS.filter(([, { projects }]) => isOf(projects, name)).map(([format]) => format) : [])

package/src/npm/cacache.js

@@ -0,0 +1,44 @@
+import { Buffer } from 'node:buffer'
+import { createHash } from 'node:crypto'
+import { join } from 'node:path'
+
+import { readRegularFile } from '../cache.js'
+
+// npm's cache, cacache as npm 5 and later keep it: a response by the URL
+// it answered, under the key `make-fetch-happen:request-cache:<url>`. The
+// index has a file for each key's sha256, of lines `<sha1>\t<entry>`, each
+// entry's sha1 of its own JSON: the last whole one for the key counts, and
+// one with no integrity is one deleted. The content is filed by its
+// sha512, and checked against it.
+const KEY = 'make-fetch-happen:request-cache:'
+const digest = (algorithm, data, encoding) => createHash(algorithm).update(data).digest(encoding)
+const shard = (hex) => [hex.slice(0, 2), hex.slice(2, 4), hex.slice(4)]
+const decoder = new TextDecoder('utf-8')
+
+async function lastEntry(root, key) {
+  const bytes = await readRegularFile(join(root, 'index-v5', ...shard(digest('sha256', key, 'hex'))))
+  let entry = null
+  for (const line of bytes ? decoder.decode(bytes).split('\n') : []) {
+    const tab = line.indexOf('\t')
+    const json = line.slice(tab + 1)
+    if (tab < 0 || digest('sha1', json, 'hex') !== line.slice(0, tab)) continue
+    try {
+      const parsed = JSON.parse(json)
+      if (parsed?.key === key) entry = parsed
+    } catch {
+      // A line cut short, or written over: passed over, as npm passes it.
+    }
+  }
+  return entry
+}
+
+// The body npm keeps for `url`, beside what it kept of the response, or
+// null: none kept, one deleted, or content gone or not its sha512.
+export async function readNpmCached(root, url) {
+  const entry = await lastEntry(root, `${KEY}${url}`)
+  const integrity = typeof entry?.integrity === 'string' ? entry.integrity : ''
+  const sha512 = /(?:^|\s)sha512-([\d+/A-Za-z]+={0,2})(?=[\s?]|$)/u.exec(integrity)?.[1]
+  if (sha512 === undefined) return null
+  const body = await readRegularFile(join(root, 'content-v2', 'sha512', ...shard(Buffer.from(sha512, 'base64').toString('hex'))))
+  return body && digest('sha512', body, 'base64') === sha512 ? { body, metadata: entry.metadata } : null
+}

package/src/npm/registry.js

@@ -3,16 +3,22 @@ import assert from 'node:assert/strict'
 import { show } from '../args.js'
 import { NPM_REGISTRY, buildUrl, request } from '../http.js'
 
+// The first of these that is set and not empty.
+const npmToken = () => process.env.PREVENTIVE_MEASURES_NPM_TOKEN || process.env.STASIS_NPM_TOKEN || process.env.NPM_TOKEN
+
 // Private packages are always scoped.
 export function withNpmToken(name, url, options = {}) {
   const method = options.method ?? 'GET'
-  assert.ok(method === 'GET' && `${new URL(url)}` === url && url.startsWith(`${NPM_REGISTRY}/${name}/`), `Unexpected request for ${name}: ${method} ${url}`)
-  const token = process.env.NPM_TOKEN
+  const own = url === `${NPM_REGISTRY}/${name}` || url.startsWith(`${NPM_REGISTRY}/${name}/`)
+  assert.ok(method === 'GET' && `${new URL(url)}` === url && own, `Unexpected request for ${name}: ${method} ${url}`)
+  const token = npmToken()
   return token && name.startsWith('@') ? { ...options, method, headers: { ...options.headers, Authorization: `Bearer ${token}` } } : options
 }
 
-export async function getDocument(method, name, spec) {
-  const url = buildUrl(NPM_REGISTRY, [...name.split('/'), spec])
+// A version's document, or with no `spec` the whole package's, every
+// version's and `time`.
+export async function getDocument(method, name, ...spec) {
+  const url = buildUrl(NPM_REGISTRY, [...name.split('/'), ...spec])
   const json = await request(url, withNpmToken(name, url, { as: 'json' }))
   assert.ok(json?.name === name, `${method}: the registry answered for ${show(json?.name)}, not ${name}`)
   return json

package/src/npm/repos.js

@@ -1,6 +1,6 @@
 import assert from 'node:assert/strict'
 
-import { assertNames, assertPackageName, assertRepo, isRepo } from '../args.js'
+import { assertNames, assertPackageName, assertion, isRepo, optional } from '../args.js'
 import { readRecord, writeRecord } from '../cache.js'
 import { isNotFound, recover } from '../http.js'
 import { assertRepoDirectory, getRepo, isRepoDirectory } from '../package.js'
@@ -10,8 +10,18 @@ import { getDocument } from './registry.js'
 const DIR = 'npm/repos'
 const CONCURRENCY = 8
 // Stamped on each entry, and raised when getRepo would answer differently:
-// 2 is `repository` taking precedence over `bugs`.
-const VERSION = 2
+// 2 is `repository` taking precedence over `bugs`; 3 the shorthand dropping
+// a `.git`, URLs read past whitespace, and `repository.directory`'s `\` read
+// as `/`; 4 a declared root answered as `''`, no directory declared left
+// out (unknown) rather than read as the root, and a declared directory's
+// empty and `.` parts dropped.
+const VERSION = 4
+
+// A slug a lookup gives: getRepo answers no repo ending in `.git`, the
+// suffix it drops (withoutDotGit; `.GIT` it keeps, and so does this).
+const isLookedUpRepo = (github) => isRepo(github) && !github.endsWith('.git')
+// A write takes no slug a read would discard: whatever it writes reads back.
+const assertLookedUpRepo = assertion('"owner/name" with no trailing ".git"', isLookedUpRepo)
 
 // `latest`, not the full packument, which is megabytes of version history.
 const fetchRepo = async (method, name) => getRepo(await getDocument(method, name, 'latest'))
@@ -24,22 +34,24 @@ export async function getGitHub(name) {
 }
 
 // An entry under another VERSION was resolved by other rules, and may
-// name the repo a stale tracker does: a miss, as is one without
-// `directory`, which would read as a package at the repo root.
+// name a repo getRepo no longer would (a stale tracker's, or a shorthand's
+// with its `.git`, or the root where a package declared no directory): a
+// miss, as is one naming a repo or a directory no lookup gives. One without
+// `directory` is a package whose place in its repo is unknown.
 export async function readPackageRepoCache(name) {
   assertPackageName('readPackageRepoCache', 'name', name)
   const entry = await readRecord(DIR, name)
-  if (entry?.v !== VERSION || !isRepo(entry.github) || !isRepoDirectory(entry.directory)) return null
-  return { github: entry.github, ...(entry.directory && { directory: entry.directory }) }
+  if (entry?.v !== VERSION || !isLookedUpRepo(entry.github) || (entry.directory !== undefined && !isRepoDirectory(entry.directory))) return null
+  return { github: entry.github, ...(entry.directory !== undefined && { directory: entry.directory }) }
 }
 
 // Only resolved repos are cached: a 404, a rate limit and a package with
 // no repo link fail alike.
-export async function writePackageRepoCache(name, github, directory = '') {
+export async function writePackageRepoCache(name, github, directory) {
   assertPackageName('writePackageRepoCache', 'name', name)
-  assertRepo('writePackageRepoCache', 'github', github)
-  assertRepoDirectory('writePackageRepoCache', 'directory', directory)
-  return await writeRecord(DIR, name, { v: VERSION, github, directory })
+  assertLookedUpRepo('writePackageRepoCache', 'github', github)
+  optional(assertRepoDirectory)('writePackageRepoCache', 'directory', directory)
+  return await writeRecord(DIR, name, { v: VERSION, github, ...(directory !== undefined && { directory }) })
 }
 
 export async function resolvePackageRepos(packageNames, options = {}) {
@@ -50,11 +62,11 @@ export async function resolvePackageRepos(packageNames, options = {}) {
 
 // Through the cache. Null for a package the registry does not have, or
 // one naming no GitHub repo; any other failure throws.
-export async function lookUpPackageRepo(name) {
+async function lookUpPackageRepo(name) {
   const stored = await readPackageRepoCache(name)
   if (stored) return stored
   const { github, directory } = await fetchRepo('lookUpPackageRepo', name).catch(recover(isNotFound, {}))
   if (!github) return null
   await writePackageRepoCache(name, github, directory)
-  return { github, ...(directory && { directory }) }
+  return { github, ...(directory !== undefined && { directory }) }
 }

package/src/npm/tarball.js

@@ -1,39 +1,16 @@
-import assert from 'node:assert/strict'
 import { Buffer } from 'node:buffer'
 import { homedir } from 'node:os'
 import { join, resolve } from 'node:path'
 
-import { assertArgs, assertPackage, assertion, matches, show } from '../args.js'
+import { assertArgs, assertPackage, assertPackageName, isPlainObject, isSha, optional } from '../args.js'
+import { assertCache } from '../cache.js'
 import { verifiedDownload } from '../download.js'
-import { NPM_REGISTRY, buildUrl } from '../http.js'
+import { NPM_REGISTRY, decode } from '../http.js'
+import { readNpmCached } from './cacache.js'
 import { getDocument, withNpmToken } from './registry.js'
+import { checkedDist, getVersion, matchingDist } from './versions.js'
 
 const DIR = 'npm/tarballs' // No expiry: the registry never takes a version twice.
-// One sha512 and nothing else, as the registry writes it: a sha1, a second
-// hash or an option is refused rather than trusted. 64 bytes leave the
-// last character before `==` two bits, so only A, Q, g or w is canonical.
-const assertIntegrity = assertion('"sha512-" and a base64 sha512', matches(/^sha512-[\dA-Za-z+/]{85}[AQgw]==$/u))
-
-// `{ tarball, integrity }` and nothing else, the tarball exactly the
-// registry's own URL for that version. Each field is read once, and only
-// as the object's own, into a copy that is checked and used from then on:
-// the caller's object can change while a request is out, and a field on
-// its prototype is not one it has.
-function checkedDist(method, name, version, dist) {
-  assertArgs(method, dist, { tarball: null, integrity: null }, 'dist')
-  const [tarball, integrity] = ['tarball', 'integrity'].map((key) => (Object.hasOwn(dist, key) ? dist[key] : undefined))
-  assertIntegrity(method, 'dist.integrity', integrity)
-  const segments = name.split('/')
-  const expected = buildUrl(NPM_REGISTRY, [...segments, '-', `${segments.at(-1)}-${version}.tgz`])
-  assert.ok(tarball === expected, `${method}: dist.tarball must be ${expected}, got ${show(tarball)}`)
-  return { tarball, integrity }
-}
-
-async function getDist(method, name, version) {
-  const json = await getDocument(method, name, version)
-  assert.ok(json.version === version, `${method}: the registry answered for ${name}@${show(json.version)}, not ${name}@${version}`)
-  return checkedDist(method, name, version, { tarball: json.dist?.tarball, integrity: json.dist?.integrity })
-}
 
 // Where npm keeps its cache, short of an .npmrc moving it: npm_config_cache,
 // as npm sets it for what it runs, else npm's default. On Windows, npm 4
@@ -53,23 +30,108 @@ function localPaths(name, version, integrity) {
   return [...npm, join(homedir(), '.audit/cache/tgz', `${name.replace(/^@/u, '').replace('/', ':')}-${version}.tgz`)]
 }
 
-export async function getMeta(name, version) {
+// verifyDist's and getTarball's options: `cache` alone.
+const assertOptions = (method, options) => assertArgs(method, options, { cache: optional(assertCache) })
+
+const own = (object, key) => (Object.hasOwn(object, key) ? object[key] : undefined)
+const stringOr = (value) => (typeof value === 'string' ? value : undefined)
+
+// A string as it is, or an object's strings under `keys`, the rest undefined:
+// none where it is neither, or names none of them.
+function linkOf(value, keys) {
+  if (typeof value === 'string') return value
+  if (!isPlainObject(value)) return undefined
+  const link = Object.fromEntries(keys.map((key) => [key, stringOr(own(value, key))]))
+  return keys.some((key) => link[key] !== undefined) ? link : undefined
+}
+
+const mediaType = (value) => (typeof value === 'string' ? value.split(';')[0].trim().toLowerCase() : undefined)
+
+function jsonOf(bytes, from) {
+  try {
+    return JSON.parse(decode(bytes, from))
+  } catch {
+    return undefined
+  }
+}
+
+// A version as npm's own cache keeps it, where it keeps its name's full
+// packument, as npm 10 and later fetch it, whose entry for the version
+// matches `given` (matchingDist): that entry is the version's document. An
+// abbreviated packument, as earlier npm fetches, has none of the rest
+// getMeta answers, and is passed over.
+async function npmCachedVersion(name, version, given) {
+  const url = `${NPM_REGISTRY}/${name.replace('/', '%2f')}`
+  for (const root of npmCacheDirs()) {
+    const cached = await readNpmCached(join(root, '_cacache'), url)
+    if (!cached || mediaType(cached.metadata?.resHeaders?.['content-type']) !== 'application/json') continue
+    const packument = jsonOf(cached.body, url)
+    const json = packument?.name === name && isPlainObject(packument.versions) ? own(packument.versions, version) : undefined
+    const dist = matchingDist('getMeta', name, version, json, given)
+    if (dist) return { dist, json }
+  }
+  return undefined
+}
+
+// The version's document: with the `dist` the caller has, npm's own cache's
+// where it matches (npmCachedVersion), and neither `cache` nor the cache
+// set is read or written; else as getVersion has it. Beside its dist, the
+// document's gitHead where that is a full commit id, and its repository,
+// homepage and bugs, in the shapes a package.json gives them. Each but
+// dist is left out where there is none.
+export async function getMeta(name, version, options = {}) {
   assertPackage('getMeta', name, version)
-  return { name, version, dist: await getDist('getMeta', name, version) }
+  assertArgs('getMeta', options, { dist: null, cache: optional(assertCache) })
+  const given = options.dist === undefined ? undefined : checkedDist('getMeta', name, version, options.dist)
+  const { dist, json } = (given && await npmCachedVersion(name, version, given)) ?? await getVersion('getMeta', name, version, given, options.cache)
+  const about = {
+    gitHead: isSha(own(json, 'gitHead')) ? json.gitHead : undefined,
+    repository: linkOf(own(json, 'repository'), ['type', 'url', 'directory']),
+    homepage: stringOr(own(json, 'homepage')),
+    bugs: linkOf(own(json, 'bugs'), ['url', 'email']),
+  }
+  return { name, version, dist, ...Object.fromEntries(Object.entries(about).filter(([, value]) => value !== undefined)) }
+}
+
+// Only the whole package's document has `time`: megabytes for some.
+const PUBLISHED = /^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d)(?:\.(\d{1,3}))?Z$/u
+const record = (value) => (isPlainObject(value) ? value : {})
+
+// A time toISOString writes back as given, its milliseconds padded: none
+// Date.parse moves, as it moves 2018-02-29 to March.
+function readTime(value) {
+  const match = typeof value === 'string' ? PUBLISHED.exec(value) : null
+  if (match === null) return undefined
+  const iso = `${match[1]}.${(match[2] ?? '').padEnd(3, '0')}Z`
+  const parsed = Date.parse(iso)
+  return Number.isFinite(parsed) && new Date(parsed).toISOString() === iso ? iso : undefined
+}
+
+export async function getPublishTimes(name) {
+  assertPackageName('getPublishTimes', 'name', name)
+  const json = await getDocument('getPublishTimes', name)
+  const time = record(json.time)
+  const times = new Map()
+  for (const version of Object.keys(record(json.versions))) {
+    const at = readTime(Object.hasOwn(time, version) ? time[version] : undefined)
+    if (at !== undefined) times.set(version, at)
+  }
+  return times
 }
 
-export async function verifyDist(name, version, dist) {
+export async function verifyDist(name, version, dist, options = {}) {
   assertPackage('verifyDist', name, version)
-  const given = checkedDist('verifyDist', name, version, dist)
-  const { integrity } = await getDist('verifyDist', name, version)
-  assert.ok(given.integrity === integrity, `verifyDist: ${name}@${version} is ${integrity} on the registry, not ${given.integrity}`)
+  assertOptions('verifyDist', options)
+  await getVersion('verifyDist', name, version, checkedDist('verifyDist', name, version, dist), options.cache)
 }
 
 // Without `dist`, the version document is read every time, cache or not:
 // bytes from a cache are checked against the registry's integrity, never
 // against anything a cache itself holds.
-export async function getTarball(name, version, dist) {
+export async function getTarball(name, version, dist, options = {}) {
   assertPackage('getTarball', name, version)
-  const { tarball, integrity } = dist === undefined ? await getDist('getTarball', name, version) : checkedDist('getTarball', name, version, dist)
-  return await verifiedDownload({ method: 'getTarball', dir: DIR, what: `${name}@${version}`, ext: 'tgz', algorithm: 'sha512', expected: integrity, local: localPaths(name, version, integrity), ours: true, locate: () => tarball, options: withNpmToken(name, tarball) })
+  assertOptions('getTarball', options)
+  const { cache } = options
+  const { tarball, integrity } = dist === undefined ? (await getVersion('getTarball', name, version, undefined, cache)).dist : checkedDist('getTarball', name, version, dist)
+  return await verifiedDownload({ method: 'getTarball', dir: DIR, what: `${name}@${version}`, ext: 'tgz', algorithm: 'sha512', expected: integrity, local: localPaths(name, version, integrity), ours: true, cache, locate: () => tarball, options: withNpmToken(name, tarball) })
 }