@preventive/upstream 1.0.0-alpha.0 → 1.0.0-alpha.10

package/src/github/read.js

@@ -1,9 +1,10 @@
 import assert from 'node:assert/strict'
 import { Buffer } from 'node:buffer'
-import { createHash } from 'node:crypto'
 
-import { assertArgs, assertGhsa, assertLogin, assertNumber, assertPath, assertRef, assertRepo, assertSha, isSha, optional, sameName, show } from '../args.js'
-import { decode, encodeSegment } from '../http.js'
+import { assertArgs, assertGhsa, assertLogin, assertNumber, assertPath, assertRef, assertRepo, assertSha, assertTagName, assertTreeId, isSha, isSha1, isTagName, optional, sameName, show } from '../args.js'
+import { verifiedDownload } from '../download.js'
+import { decode, encodeSegment, requestWithHeaders } from '../http.js'
+import { gitTreeOfListing, objectId } from '../tree.js'
 import { api, bindMethods, call, clientHeaders, isGone, repoApi } from './client.js'
 
 const PER_PAGE = 100
@@ -20,6 +21,24 @@ async function* pages(method, headers, pageUrl, maxPages = MAX_PAGES) {
   }
 }
 
+// A list GitHub pages by a cursor instead: the `after` of each page's
+// Link header `next` link, the one thing read off the link, so the next
+// page is asked of the list `pageUrl` names, wherever the link points. A
+// next link proves a page more.
+async function* cursorPages(method, headers, pageUrl, maxPages = MAX_PAGES) {
+  const paging = { per_page: PER_PAGE }
+  for (let page = 1; ; page++) {
+    const answer = await requestWithHeaders(pageUrl(paging), { as: 'json', headers })
+    assert.ok(Array.isArray(answer.body), `${method}: expected an array for page ${page}`)
+    yield answer.body
+    const next = /<([^>]*)>\s*;\s*rel="next"/u.exec(answer.headers.get('link') ?? '')?.[1]
+    if (next === undefined) return
+    paging.after = URL.parse(next)?.searchParams.get('after')
+    assert.ok(paging.after, `${method}: page ${page} links the next with no cursor`)
+    assert.ok(page < maxPages, `${method}: more than ${maxPages} pages`)
+  }
+}
+
 const paginate = async (...args) => (await Array.fromAsync(pages(...args))).flat()
 const getCurrentUser = (headers) => call(headers, api(['user']))
 
@@ -46,6 +65,32 @@ export async function getRepoHead(headers, options) {
   return { branch: ref, oid: data.object.sha }
 }
 
+async function getRepoTag(headers, options) {
+  assertArgs('getRepoTag', options, { repo: assertRepo, tag: assertTagName })
+  const { repo, tag } = options
+  const ref = await call(headers, repoApi(repo, ['git', 'ref', 'tags', encodeSegment(tag)]))
+  assert.ok(ref?.ref === `refs/tags/${tag}`, `getRepoTag: GitHub answered for ${show(ref?.ref)}, not ${show(`refs/tags/${tag}`)}`)
+  let { object } = ref
+  for (let hops = 0; object?.type === 'tag' && isSha(object.sha) && hops < 8; hops++) {
+    const annotated = await call(headers, repoApi(repo, ['git', 'tags', object.sha]))
+    assert.ok(annotated?.sha === object.sha, `getRepoTag: GitHub answered for another tag object than ${object.sha}`)
+    object = annotated.object
+  }
+  assert.ok(object?.type === 'commit' && isSha(object.sha), `getRepoTag: ${repo} has no commit for tag ${show(tag)}`)
+  return { tag, oid: object.sha }
+}
+
+// GitHub lists each tag with the commit it names, an annotated one's too.
+async function listRepoTags(headers, options) {
+  assertArgs('listRepoTags', options, { repo: assertRepo, maxPages: optional(assertNumber) })
+  const { repo, maxPages } = options
+  const tags = await paginate('listRepoTags', headers, (paging) => repoApi(repo, ['tags'], paging), maxPages)
+  return tags.map((entry) => {
+    assert.ok(isTagName(entry?.name) && isSha(entry.commit?.sha), `listRepoTags: GitHub listed ${show(entry?.name)} in ${repo}, which is no tag name with a commit`)
+    return { tag: entry.name, oid: entry.commit.sha }
+  })
+}
+
 // GitHub's object for the path first: a directory, symlink or submodule is
 // refused rather than read as a file's text. Up to 1 MB the object holds
 // the content; past it, `encoding: none`, and the blob it names is read,
@@ -61,16 +106,73 @@ async function getRepoFile(headers, options) {
   const bytes = file.encoding === 'none'
     ? await call({ ...headers, Accept: 'application/vnd.github.raw' }, repoApi(repo, ['git', 'blobs', file.sha]), { as: 'bytes' })
     : Buffer.from(file.content, 'base64')
-  const blob = createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex')
+  const blob = objectId('blob', bytes).toString('hex')
   assert.ok(bytes.length === file.size && blob === file.sha, `getRepoFile: ${show(path)} came back as blob ${blob}, not ${file.sha}`)
   return decode(bytes, url)
 }
 
-// The one request that follows a redirect, to codeload.github.com. A full
-// sha only, so the bytes are that commit's, not wherever a ref points now.
+// Follows the redirect to codeload.github.com. A tree id names its content,
+// so the bytes are held to it, downloaded or cached, and cached by it alone,
+// for good. What a tarball cannot show, a submodule's commit, a subtree
+// with nothing in it or the LF blob of a file marked `eol=crlf`, comes
+// from GitHub's listings of the trees, which the id checks as well: a
+// directory at a time, as a recursive listing of a large tree is cut short.
+async function treeTarball(method, headers, repo, tree) {
+  const listings = new Map()
+  const list = (sha) => {
+    if (!listings.has(sha)) listings.set(sha, call(headers, repoApi(repo, ['git', 'trees', sha])).then((listing) => (Array.isArray(listing?.tree) ? listing.tree : [])))
+    return listings.get(sha)
+  }
+  const locate = () => repoApi(repo, ['tarball', tree])
+  return await verifiedDownload({ method, dir: 'github/trees', what: tree, ext: 'tgz', algorithm: 'tree', expected: tree, locate, options: { headers, redirect: 'follow' }, list })
+}
+
+async function commitTree(method, headers, repo, sha) {
+  const commit = await call(headers, repoApi(repo, ['git', 'commits', sha]))
+  assert.ok(commit?.sha === sha && isSha1(commit.tree?.sha), `${method}: GitHub names no tree for ${repo}@${sha}`)
+  return commit.tree.sha
+}
+
+// A full sha only, so the bytes are that commit's, not wherever a ref points
+// now. Its tree's tarball, not its own, in which `git archive` rewrites the
+// files marked `export-subst`.
 async function getRepoTarball(headers, options) {
   assertArgs('getRepoTarball', options, { repo: assertRepo, sha: assertSha })
-  return await call(headers, repoApi(options.repo, ['tarball', options.sha]), { as: 'bytes', redirect: 'follow' })
+  const { repo, sha } = options
+  const tree = await commitTree('getRepoTarball', headers, repo, sha)
+  return await treeTarball('getRepoTarball', headers, repo, tree)
+}
+
+async function getRepoTreeTarball(headers, options) {
+  assertArgs('getRepoTreeTarball', options, { repo: assertRepo, tree: assertTreeId })
+  return await treeTarball('getRepoTreeTarball', headers, options.repo, options.tree)
+}
+
+// Not recursive: GitHub cuts short a recursive listing of a large tree.
+async function listTree(method, headers, repo, tree) {
+  const entries = (await call(headers, repoApi(repo, ['git', 'trees', tree])))?.tree
+  assert.ok(Array.isArray(entries) && gitTreeOfListing(entries) === tree, `${method}: GitHub's listing of tree ${tree} in ${repo} is not that tree`)
+  return entries.map(({ path, mode, type, sha }) => ({ path, mode, type, sha }))
+}
+
+async function treeAt(method, headers, options) {
+  assertArgs(method, options, { repo: assertRepo, sha: assertSha, directory: optional(assertPath), path: optional(assertPath) })
+  assert.ok(options.directory === undefined || options.path === undefined, `${method}: directory and path are one option, pass directory alone`)
+  const { repo, sha, directory = options.path } = options
+  let tree = await commitTree(method, headers, repo, sha)
+  for (const name of directory?.split('/') ?? []) {
+    const entry = (await listTree(method, headers, repo, tree)).find((candidate) => candidate.path === name)
+    assert.ok(entry?.type === 'tree', `${method}: ${repo}@${sha} has no directory at ${show(directory)}`)
+    tree = entry.sha
+  }
+  return tree
+}
+
+const getRepoTreeId = (headers, options) => treeAt('getRepoTreeId', headers, options)
+
+async function listRepoDir(headers, options) {
+  const tree = await treeAt('listRepoDir', headers, options)
+  return await listTree('listRepoDir', headers, options.repo, tree)
 }
 
 async function getPullRequest(headers, options) {
@@ -110,15 +212,12 @@ async function getAdvisory(headers, options) {
   return advisory
 }
 
-// One page: GitHub pages this list by a cursor in the Link header, which
-// `call` does not read, so a full page is refused rather than cut short.
 async function listRepoAdvisories(headers, options) {
   assertArgs('listRepoAdvisories', options, { repo: assertRepo })
-  const list = await call(headers, repoApi(options.repo, ['security-advisories'], { state: 'published', per_page: PER_PAGE }))
-  assert.ok(Array.isArray(list), `listRepoAdvisories: expected an array for ${options.repo}`)
-  assert.ok(list.length < PER_PAGE, `listRepoAdvisories: ${options.repo} has ${PER_PAGE} or more published advisories, more than a page`)
-  return list
+  const { repo } = options
+  const pageUrl = (paging) => repoApi(repo, ['security-advisories'], { state: 'published', ...paging })
+  return (await Array.fromAsync(cursorPages('listRepoAdvisories', headers, pageUrl))).flat()
 }
 
-export const readMethods = { getCurrentUser, listUserRepos, getRepo, getRepoHead, getRepoFile, getRepoTarball, getPullRequest, getCollaboratorPermission, getAdvisory, listRepoAdvisories }
+export const readMethods = { getCurrentUser, listUserRepos, getRepo, getRepoHead, getRepoTag, listRepoTags, getRepoFile, getRepoTarball, getRepoTreeTarball, getRepoTreeId, listRepoDir, getPullRequest, getCollaboratorPermission, getAdvisory, listRepoAdvisories }
 export const createClient = (options) => bindMethods(clientHeaders('createClient', options, true), readMethods)

package/src/github/write.js

@@ -66,19 +66,13 @@ export function parseGraphQLResponse(status, text) {
     throw new HttpError(status, `GitHub GraphQL ${status}: ${printable(text.slice(0, 4096)) || '(empty body)'}`)
   }
   let json
-  try { json = JSON.parse(text) } catch (err) {
-    throw new Error(`GitHub GraphQL ${status}: malformed JSON response (${err.message}): ${printable(text.slice(0, 200))}`, { cause: err })
+  try { json = JSON.parse(text) } catch (cause) {
+    throw new Error(`GitHub GraphQL ${status}: malformed JSON response (${cause.message}): ${printable(text.slice(0, 200))}`, { cause })
   }
   if (json.errors) throw new Error(`GitHub GraphQL: ${printable(JSON.stringify(json.errors))}`)
   return json.data
 }
 
-// A signed commit; the branch must already exist.
-async function createCommitOnBranch(headers, input) {
-  const res = await send(api(['graphql']), { method: 'POST', headers, body: { query: CREATE_COMMIT_MUTATION, variables: { input } }, as: 'json' })
-  return parseGraphQLResponse(res.status, decode(await readBody(res, 1024 * 1024), 'GitHub GraphQL')).createCommitOnBranch.commit
-}
-
 async function forkRepo(headers, options) {
   assertArgs('forkRepo', options, { repo: assertRepo, name: optional(assertRepoName), organization: optional(assertLogin), defaultBranchOnly: optional(assertBoolean) })
   const { repo, name, organization, defaultBranchOnly } = options
@@ -93,13 +87,16 @@ async function createBranch(headers, options) {
   return await call(headers, repoApi(repo, ['git', 'refs']), { method: 'POST', body: { ref: `refs/heads/${branch}`, sha } })
 }
 
+// A signed commit; the branch must already exist.
 async function createCommit(headers, options) {
   assertArgs('createCommit', options, { repo: assertRepo, branch: assertRef, message: null, additions: null, deletions: null, expectedHeadOid: optional(assertSha) })
   const { repo, branch } = options
   const changes = fileChanges(options.additions, options.deletions)
   const message = commitMessage(options.message)
   const expectedHeadOid = options.expectedHeadOid ?? (await getRepoHead(headers, { repo, branch })).oid
-  return await createCommitOnBranch(headers, { branch: { repositoryNameWithOwner: repo, branchName: branch }, message, fileChanges: changes, expectedHeadOid })
+  const input = { branch: { repositoryNameWithOwner: repo, branchName: branch }, message, fileChanges: changes, expectedHeadOid }
+  const res = await send(api(['graphql']), { method: 'POST', headers, body: { query: CREATE_COMMIT_MUTATION, variables: { input } }, as: 'json' })
+  return parseGraphQLResponse(res.status, decode(await readBody(res, 1024 * 1024), 'GitHub GraphQL')).createCommitOnBranch.commit
 }
 
 async function createPR(headers, options) {

package/src/http.js

@@ -11,9 +11,13 @@ export const GITHUB_API = 'https://api.github.com'
 export const OSV_API = 'https://api.osv.dev'
 export const CRATES_API = 'https://crates.io'
 export const PACKAGIST_REPO = 'https://repo.packagist.org'
+export const CRATES_INDEX = 'https://index.crates.io'
+export const CRATES_STATIC = 'https://static.crates.io'
+export const SOLDEER_API = 'https://api.soldeer.xyz'
+export const SOLDEER_REVISIONS = 'https://soldeer-revisions.s3.amazonaws.com'
 export const MAX_BYTES = 512 * 1024 * 1024
 
-const ORIGINS = new Set([NPM_REGISTRY, GITHUB_API, OSV_API, CRATES_API, PACKAGIST_REPO])
+const ORIGINS = new Set([NPM_REGISTRY, GITHUB_API, OSV_API, CRATES_API, PACKAGIST_REPO, CRATES_INDEX, CRATES_STATIC, SOLDEER_API, SOLDEER_REVISIONS])
 const isSegment = matches(/^(?!\.\.?$)(?:[\w.~@-]|%[\dA-F]{2})+$/u)
 const isQueryKey = matches(/^[a-z_]+(?:\[\])?$/u)
 const isQueryValue = (value) => (typeof value === 'string' && value !== '') || (Number.isSafeInteger(value) && value >= 0)
@@ -36,11 +40,17 @@ export class HttpError extends Error {
   }
 }
 
+export const isNotFound = (err) => err instanceof HttpError && err.status === 404
+export const recover = (expected, value) => (err) => {
+  if (!expected(err)) throw err
+  return value
+}
+
 export function decode(bytes, from) {
   try {
     return decoder.decode(bytes)
-  } catch (err) {
-    throw new Error(`Malformed UTF-8 from ${from}`, { cause: err })
+  } catch (cause) {
+    throw new Error(`Malformed UTF-8 from ${from}`, { cause })
   }
 }
 
@@ -91,22 +101,27 @@ export async function readBody(res, limit, { truncate = false } = {}) {
   return new Uint8Array(await new Blob(chunks).arrayBuffer(), 0, Math.min(size, limit))
 }
 
+function parse(bytes, url, as) {
+  if (as === 'bytes') return bytes
+  const text = decode(bytes, url)
+  if (as === 'text') return text
+  try {
+    return JSON.parse(text)
+  } catch (cause) {
+    throw new Error(`Malformed JSON from ${url}: ${printable(text.slice(0, 200))}`, { cause })
+  }
+}
+
 // Read as the caller says, not by content type, which proxies drop or
-// rewrite.
-export async function request(url, options) {
+// rewrite; answered with the response's headers.
+export async function requestWithHeaders(url, options) {
   const res = await send(url, options)
   if (!res.ok) {
     // A stream decode holds back a character cut at the limit rather than refusing it.
     const text = await readBody(res, 4096, { truncate: true }).then((bytes) => new TextDecoder('utf-8', { fatal: true }).decode(bytes, { stream: true })).catch(() => '')
     throw new HttpError(res.status, `${options.method ?? 'GET'} ${url} ${res.status}: ${printable(text)}`)
   }
-  const bytes = await readBody(res, LIMITS[options.as].bytes)
-  if (options.as === 'bytes') return bytes
-  const text = decode(bytes, url)
-  if (options.as === 'text') return text
-  try {
-    return JSON.parse(text)
-  } catch (err) {
-    throw new Error(`Malformed JSON from ${url}: ${printable(text.slice(0, 200))}`, { cause: err })
-  }
+  return { body: parse(await readBody(res, LIMITS[options.as].bytes), url, options.as), headers: res.headers }
 }
+
+export const request = async (url, options) => (await requestWithHeaders(url, options)).body

package/src/npm/registry.js

@@ -0,0 +1,19 @@
+import assert from 'node:assert/strict'
+
+import { show } from '../args.js'
+import { NPM_REGISTRY, buildUrl, request } from '../http.js'
+
+// Private packages are always scoped.
+export function withNpmToken(name, url, options = {}) {
+  const method = options.method ?? 'GET'
+  assert.ok(method === 'GET' && `${new URL(url)}` === url && url.startsWith(`${NPM_REGISTRY}/${name}/`), `Unexpected request for ${name}: ${method} ${url}`)
+  const token = process.env.NPM_TOKEN
+  return token && name.startsWith('@') ? { ...options, method, headers: { ...options.headers, Authorization: `Bearer ${token}` } } : options
+}
+
+export async function getDocument(method, name, spec) {
+  const url = buildUrl(NPM_REGISTRY, [...name.split('/'), spec])
+  const json = await request(url, withNpmToken(name, url, { as: 'json' }))
+  assert.ok(json?.name === name, `${method}: the registry answered for ${show(json?.name)}, not ${name}`)
+  return json
+}

package/src/npm/repos.js

@@ -1,21 +1,28 @@
 import assert from 'node:assert/strict'
 
-import { assertArgs, assertBoolean, assertPackageName, assertRepo, isRepo, optional, show } from '../args.js'
-import { readCacheJSON, writeCacheJSON } from '../cache.js'
-import { HttpError, NPM_REGISTRY, buildUrl, request } from '../http.js'
+import { assertNames, assertPackageName, assertion, isRepo } from '../args.js'
+import { readRecord, writeRecord } from '../cache.js'
+import { isNotFound, recover } from '../http.js'
 import { assertRepoDirectory, getRepo, isRepoDirectory } from '../package.js'
 import { pool } from '../pool.js'
+import { getDocument } from './registry.js'
 
 const DIR = 'npm/repos'
-const TTL_MS = 30 * 24 * 60 * 60 * 1000 // A link only moves on a transfer or rename, and GitHub redirects those.
 const CONCURRENCY = 8
+// Stamped on each entry, and raised when getRepo would answer differently:
+// 2 is `repository` taking precedence over `bugs`; 3 the shorthand dropping
+// a `.git`, URLs read past whitespace, and `repository.directory`'s `\` read
+// as `/`.
+const VERSION = 3
+
+// A slug a lookup gives: getRepo answers no repo ending in `.git`, the
+// suffix it drops (withoutDotGit; `.GIT` it keeps, and so does this).
+const isLookedUpRepo = (github) => isRepo(github) && !github.endsWith('.git')
+// A write takes no slug a read would discard: whatever it writes reads back.
+const assertLookedUpRepo = assertion('"owner/name" with no trailing ".git"', isLookedUpRepo)
 
 // `latest`, not the full packument, which is megabytes of version history.
-async function fetchRepo(method, name) {
-  const json = await request(buildUrl(NPM_REGISTRY, [...name.split('/'), 'latest']), { as: 'json' })
-  assert.ok(json?.name === name, `${method}: the registry answered for ${show(json?.name)}, not ${name}`)
-  return getRepo(json)
-}
+const fetchRepo = async (method, name) => getRepo(await getDocument(method, name, 'latest'))
 
 export async function getGitHub(name) {
   assertPackageName('getGitHub', 'name', name)
@@ -24,14 +31,15 @@ export async function getGitHub(name) {
   return link
 }
 
-// An entry without `directory` predates the field: a miss, not a package
-// at the repo root.
+// An entry under another VERSION was resolved by other rules, and may
+// name a repo getRepo no longer would (a stale tracker's, or a shorthand's
+// with its `.git`): a miss, as is one naming a repo or a directory no
+// lookup gives, or without `directory`, which would read as a package at
+// the repo root.
 export async function readPackageRepoCache(name) {
   assertPackageName('readPackageRepoCache', 'name', name)
-  const entry = await readCacheJSON(DIR, `${name}.json`)
-  const age = typeof entry?.at === 'number' ? Date.now() - entry.at : Number.NaN
-  const fresh = age >= 0 && age <= TTL_MS // An entry from the future is not fresh forever.
-  if (!fresh || entry.name !== name || !isRepo(entry.github) || !isRepoDirectory(entry.directory)) return null
+  const entry = await readRecord(DIR, name)
+  if (entry?.v !== VERSION || !isLookedUpRepo(entry.github) || !isRepoDirectory(entry.directory)) return null
   return { github: entry.github, ...(entry.directory && { directory: entry.directory }) }
 }
 
@@ -39,16 +47,13 @@ export async function readPackageRepoCache(name) {
 // no repo link fail alike.
 export async function writePackageRepoCache(name, github, directory = '') {
   assertPackageName('writePackageRepoCache', 'name', name)
-  assertRepo('writePackageRepoCache', 'github', github)
+  assertLookedUpRepo('writePackageRepoCache', 'github', github)
   assertRepoDirectory('writePackageRepoCache', 'directory', directory)
-  return await writeCacheJSON(DIR, `${name}.json`, { at: Date.now(), name, github, directory })
+  return await writeRecord(DIR, name, { v: VERSION, github, directory })
 }
 
 export async function resolvePackageRepos(packageNames, options = {}) {
-  assert.ok(typeof packageNames?.[Symbol.iterator] === 'function' && typeof packageNames !== 'string', 'resolvePackageRepos: packageNames must be an iterable of names')
-  assertArgs('resolvePackageRepos', options, { cachedOnly: optional(assertBoolean) })
-  const names = [...new Set(packageNames)]
-  for (const name of names) assertPackageName('resolvePackageRepos', 'name', name)
+  const names = assertNames('resolvePackageRepos', 'packageNames', packageNames, options, assertPackageName)
   const found = await pool(names, CONCURRENCY, (name) => (options.cachedOnly ? readPackageRepoCache(name) : lookUpPackageRepo(name)).catch(() => null))
   return new Map(names.flatMap((name, i) => (found[i] ? [[name, found[i]]] : [])))
 }
@@ -58,10 +63,7 @@ export async function resolvePackageRepos(packageNames, options = {}) {
 export async function lookUpPackageRepo(name) {
   const stored = await readPackageRepoCache(name)
   if (stored) return stored
-  const { github, directory } = await fetchRepo('lookUpPackageRepo', name).catch((err) => {
-    if (err instanceof HttpError && err.status === 404) return {}
-    throw err
-  })
+  const { github, directory } = await fetchRepo('lookUpPackageRepo', name).catch(recover(isNotFound, {}))
   if (!github) return null
   await writePackageRepoCache(name, github, directory)
   return { github, ...(directory && { directory }) }

package/src/npm/tarball.js

@@ -1,30 +1,18 @@
 import assert from 'node:assert/strict'
 import { Buffer } from 'node:buffer'
-import { createHash } from 'node:crypto'
-import { readFile, stat } from 'node:fs/promises'
 import { homedir } from 'node:os'
 import { join, resolve } from 'node:path'
 
-import { assertArgs, assertPackageName, assertPackageVersion, assertion, matches, show } from '../args.js'
-import { readCache, writeCache } from '../cache.js'
-import { MAX_BYTES, NPM_REGISTRY, buildUrl, request } from '../http.js'
+import { assertArgs, assertPackage, assertion, matches, show } from '../args.js'
+import { verifiedDownload } from '../download.js'
+import { NPM_REGISTRY, buildUrl } from '../http.js'
+import { getDocument, withNpmToken } from './registry.js'
 
 const DIR = 'npm/tarballs' // No expiry: the registry never takes a version twice.
 // One sha512 and nothing else, as the registry writes it: a sha1, a second
 // hash or an option is refused rather than trusted. 64 bytes leave the
 // last character before `==` two bits, so only A, Q, g or w is canonical.
 const assertIntegrity = assertion('"sha512-" and a base64 sha512', matches(/^sha512-[\dA-Za-z+/]{85}[AQgw]==$/u))
-const sha512 = (bytes) => `sha512-${createHash('sha512').update(bytes).digest('base64')}`
-
-function assertPackage(method, name, version) {
-  assertPackageName(method, 'name', name)
-  assertPackageVersion(method, 'version', version)
-}
-
-function tarballUrl(name, version) {
-  const segments = name.split('/')
-  return buildUrl(NPM_REGISTRY, [...segments, '-', `${segments.at(-1)}-${version}.tgz`])
-}
 
 // `{ tarball, integrity }` and nothing else, the tarball exactly the
 // registry's own URL for that version. Each field is read once, and only
@@ -35,22 +23,18 @@ function checkedDist(method, name, version, dist) {
   assertArgs(method, dist, { tarball: null, integrity: null }, 'dist')
   const [tarball, integrity] = ['tarball', 'integrity'].map((key) => (Object.hasOwn(dist, key) ? dist[key] : undefined))
   assertIntegrity(method, 'dist.integrity', integrity)
-  const expected = tarballUrl(name, version)
+  const segments = name.split('/')
+  const expected = buildUrl(NPM_REGISTRY, [...segments, '-', `${segments.at(-1)}-${version}.tgz`])
   assert.ok(tarball === expected, `${method}: dist.tarball must be ${expected}, got ${show(tarball)}`)
   return { tarball, integrity }
 }
 
 async function getDist(method, name, version) {
-  const json = await request(buildUrl(NPM_REGISTRY, [...name.split('/'), version]), { as: 'json' })
-  assert.ok(json?.name === name && json.version === version, `${method}: the registry answered for ${show(json?.name)}@${show(json?.version)}, not ${name}@${version}`)
+  const json = await getDocument(method, name, version)
+  assert.ok(json.version === version, `${method}: the registry answered for ${name}@${show(json.version)}, not ${name}@${version}`)
   return checkedDist(method, name, version, { tarball: json.dist?.tarball, integrity: json.dist?.integrity })
 }
 
-function assertBytes(bytes, integrity, what) {
-  const actual = sha512(bytes)
-  assert.ok(actual === integrity, `getTarball: integrity mismatch for ${what}: expected ${integrity}, got ${actual}`)
-}
-
 // Where npm keeps its cache, short of an .npmrc moving it: npm_config_cache,
 // as npm sets it for what it runs, else npm's default. On Windows, npm 4
 // and before defaulted to %APPDATA%, later ones to %LOCALAPPDATA%.
@@ -61,24 +45,12 @@ function npmCacheDirs() {
   return [process.env.LOCALAPPDATA, process.env.APPDATA].map((dir) => join(dir || homedir(), 'npm-cache'))
 }
 
-// Only a regular file: a FIFO or a device would block or never end.
-async function readRegularFile(path) {
-  const stats = await stat(path).catch(() => null)
-  if (!stats?.isFile() || stats.size > MAX_BYTES) return null
-  return await readFile(path).then((bytes) => new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength), () => null)
-}
-
 // npm 5+ (cacache) files a tarball by its sha512, npm 4 and before as
 // <name>/<version>/package.tgz; ~/.audit as <org>:<name>-<version>.tgz.
-async function readLocalCaches(name, version, integrity) {
+function localPaths(name, version, integrity) {
   const hex = Buffer.from(integrity.slice('sha512-'.length), 'base64').toString('hex')
   const npm = npmCacheDirs().flatMap((root) => [join(root, '_cacache/content-v2/sha512', hex.slice(0, 2), hex.slice(2, 4), hex.slice(4)), join(root, name, version, 'package.tgz')])
-  const audit = join(homedir(), '.audit/cache/tgz', `${name.replace(/^@/u, '').replace('/', ':')}-${version}.tgz`)
-  for (const path of [...npm, audit]) {
-    const bytes = await readRegularFile(path)
-    if (bytes && sha512(bytes) === integrity) return bytes
-  }
-  return null
+  return [...npm, join(homedir(), '.audit/cache/tgz', `${name.replace(/^@/u, '').replace('/', ':')}-${version}.tgz`)]
 }
 
 export async function getMeta(name, version) {
@@ -95,21 +67,9 @@ export async function verifyDist(name, version, dist) {
 
 // Without `dist`, the version document is read every time, cache or not:
 // bytes from a cache are checked against the registry's integrity, never
-// against anything a cache itself holds. A mismatch in ours throws rather
-// than fetching over it.
+// against anything a cache itself holds.
 export async function getTarball(name, version, dist) {
   assertPackage('getTarball', name, version)
-  dist = dist === undefined ? await getDist('getTarball', name, version) : checkedDist('getTarball', name, version, dist)
-  const local = await readLocalCaches(name, version, dist.integrity)
-  if (local) return local
-  const key = `${name}@${version}.tgz`
-  const cached = await readCache(DIR, key)
-  if (cached) {
-    assertBytes(cached, dist.integrity, `${name}@${version} from the cache`)
-    return cached
-  }
-  const bytes = await request(dist.tarball, { as: 'bytes' })
-  assertBytes(bytes, dist.integrity, `${name}@${version} from ${dist.tarball}`)
-  await writeCache(DIR, key, bytes)
-  return bytes
+  const { tarball, integrity } = dist === undefined ? await getDist('getTarball', name, version) : checkedDist('getTarball', name, version, dist)
+  return await verifiedDownload({ method: 'getTarball', dir: DIR, what: `${name}@${version}`, ext: 'tgz', algorithm: 'sha512', expected: integrity, local: localPaths(name, version, integrity), ours: true, locate: () => tarball, options: withNpmToken(name, tarball) })
 }

package/src/package.js

@@ -1,7 +1,7 @@
 import assert from 'node:assert/strict'
 
 import { assertion, isRepo, isRepoPath, sameName } from './args.js'
-import { githubRepoOfUrl } from './remote.js'
+import { githubRepoOfUrl, withoutDotGit } from './remote.js'
 
 const bugsRegex = /^(?i:https?:\/\/github\.com)(?::\d{1,5})?\/(?<repo>[\w-]+\/[\w.-]+)\/issues\/?$/u
 // npm's `owner/name` shorthand means GitHub. No dots in the owner, so a
@@ -11,9 +11,12 @@ const shorthandRegex = /^(?:github:)?(?<repo>[\w-]+\/[\w.-]+)$/u
 // The ref is one segment: a branch with a `/` reads as part of the directory.
 const homepageRegex = /^(?:https?:\/\/)?(?:www\.)?github\.com(?::\d{1,5})?\/(?<repo>[\w-]+\/[\w.-]+?)(?:\.git)?(?:\/(?:tree\/[^/]+(?:\/(?<directory>.*))?)?)?$/iu
 const str = (value) => (typeof value === 'string' ? value : '')
-const urlOf = (field) => str(field?.url ?? field)
+// Read past the whitespace around it, as `homepage` is.
+const urlOf = (field) => str(field?.url ?? field).trim()
 const homepageUrl = (homepage) => str(homepage).trim().split(/[?#]/u)[0] // npm appends `#readme`.
-const repoIn = (regex, text) => regex.exec(text)?.groups.repo
+// Every trailing `.git` dropped, as the URLs' are: no answer ends in one
+// (the npm repo cache refuses one).
+const repoIn = (regex, text) => withoutDotGit(regex.exec(text)?.groups.repo)
 // A URL's path spells a tree path percent-encoded; one that does not decode names none.
 function decodePath(path = '') {
   try {
@@ -33,6 +36,11 @@ function repoSubdirectory(value) {
   return isRepoPath(path) ? path : undefined // No empty, `.`, `..` or `.git` part: a path inside the repo.
 }
 
+// `repository.directory` is a path its author may write with Windows' `\`.
+// A tree path is `/`-separated: a `\` is a name's own only in a homepage's
+// (`%5C`), which is the path itself.
+const declaredDirectory = (value) => repoSubdirectory(str(value).replaceAll('\\', '/'))
+
 export const isRepoDirectory = (value) => typeof value === 'string' && (value === '' || repoSubdirectory(value) === value)
 export const assertRepoDirectory = assertion('a path inside the repository', isRepoDirectory)
 
@@ -41,9 +49,12 @@ export function getRepo(pkg) {
   const { bugs, repository } = pkg
   const declared = repositoryRepo(urlOf(repository))
   const homepage = homepageRegex.exec(homepageUrl(pkg.homepage))?.groups
-  const github = [repoIn(bugsRegex, urlOf(bugs)), declared, homepage?.repo].find(isRepo)
+  const homepageRepo = withoutDotGit(homepage?.repo)
+  // `repository` first: it is the field a publisher maintains, where a
+  // `bugs` tracker can be left pointing at a former owner or misspelt.
+  const github = [declared, repoIn(bugsRegex, urlOf(bugs)), homepageRepo].find(isRepo)
   if (github === undefined) return {}
-  const directory = (sameName(declared, github) && repoSubdirectory(repository?.directory))
-    || (sameName(homepage?.repo, github) && repoSubdirectory(decodePath(homepage.directory)))
+  const directory = (sameName(declared, github) && declaredDirectory(repository?.directory))
+    || (sameName(homepageRepo, github) && repoSubdirectory(decodePath(homepage.directory)))
   return { github, ...(directory && { directory }), url: `https://github.com/${github}` }
 }

package/src/pool.js

@@ -10,3 +10,5 @@ export async function pool(items, limit, fn) {
   }))
   return results
 }
+
+export const chunks = (items, size) => Array.from({ length: Math.ceil(items.length / size) }, (_, i) => items.slice(i * size, (i + 1) * size))

package/src/remote.js

@@ -7,9 +7,16 @@ import { isRepo } from './args.js'
 const gitUrlRegex = /^(?i:(?:git\+)?(?:https?|git|ssh)):\/\/(?:[\w.~%!$&'()*+,;=:-]*@)?(?i:github\.com)(?::\d{1,5})?\/(?<repo>[\w-]+\/[\w.-]+?)(?:\.git)?\/?$/u
 const scpRegex = /^(?i:git\+ssh:\/\/)?git@(?i:github\.com):(?<repo>[\w-]+\/[\w.-]+?)(?:\.git)?$/u
 
+// A repo with every trailing `.git` dropped, so none ends in one: in a loop,
+// as a regex would backtrack over a long run of them in a crafted URL.
+export function withoutDotGit(repo) {
+  while (repo?.endsWith('.git')) repo = repo.slice(0, -4)
+  return repo
+}
+
 export function githubRepoOfUrl(url) {
   if (typeof url !== 'string') return undefined
   const onGitHub = URL.parse(url.replace(/^git\+/iu, ''))?.hostname.toLowerCase() === 'github.com'
-  const repo = scpRegex.exec(url)?.groups.repo ?? (onGitHub ? gitUrlRegex.exec(url)?.groups.repo : undefined)
+  const repo = withoutDotGit(scpRegex.exec(url)?.groups.repo ?? (onGitHub ? gitUrlRegex.exec(url)?.groups.repo : undefined))
   return isRepo(repo) ? repo : undefined
 }

package/src/semver-peer.cjs

@@ -0,0 +1,8 @@
+'use strict'
+
+// The semver peer dependency by a plain require(), which a bundler follows
+// and bundles: CommonJS for that alone. In an ESM bundle that leaves semver
+// out, this require() is the bundler's stub, which throws.
+module.exports = function requirePeer() {
+  return require('semver')
+}