@preventive/upstream 1.0.0-alpha.1 → 1.0.0-alpha.10

package/src/attributes.js

@@ -0,0 +1,81 @@
+import { Buffer } from 'node:buffer'
+
+// Whether git writes a file out with CRLF, from the .gitattributes in its
+// tree alone, as `git archive` and a checkout with no config read them. Only
+// `*`, `*.ext`, name and path rules are read: any other rule that sets an
+// attribute line endings depend on refuses its file, and all below it.
+
+const RELEVANT = new Set(['text', 'crlf', 'eol', 'binary', 'ident', 'filter', 'working-tree-encoding'])
+const BINARY = [['diff', false], ['merge', false], ['text', false]]
+const GLOB = /[*?[\\]/u
+const PRINTING = new Set([0x08, 0x09, 0x0c, 0x1b]) // BS, HT, FF, ESC
+
+function matcher(pattern) {
+  const name = (path) => path.slice(path.lastIndexOf('/') + 1)
+  if (pattern.endsWith('/')) return () => false // a directory
+  if (!GLOB.test(pattern)) return pattern.includes('/') ? (path) => path === pattern.replace(/^\//u, '') : (path) => name(path) === pattern
+  const end = pattern.slice(1)
+  return pattern[0] === '*' && !GLOB.test(end) && !end.includes('/') ? (path) => name(path).endsWith(end) : null
+}
+
+// Read to the first NUL, as git reads one from a tree. A line with a name
+// git takes for none, or a negative pattern, git leaves out.
+function parseAttributes(bytes) {
+  const nul = bytes.indexOf(0)
+  const rules = []
+  for (const line of Buffer.from(bytes.buffer, bytes.byteOffset, nul === -1 ? bytes.length : nul).toString('latin1').split('\n')) {
+    const [pattern, ...tokens] = line.split(/[ \t\r]+/u).filter(Boolean)
+    if (pattern?.startsWith('"')) return null
+    if (pattern === undefined || pattern.startsWith('#') || pattern.startsWith('!')) continue
+    const states = tokens.map((token) => /^([-!]?)([\w.][\w.-]*)(?:=(.*))?$/u.exec(token))
+    if (states.includes(null) || !states.some(([, , name]) => RELEVANT.has(name))) continue
+    const match = matcher(pattern)
+    if (!match) return null
+    rules.push({ match, states: states.map(([, sign, name, value = true]) => [name, sign === '-' ? false : sign === '!' ? null : value]) })
+  }
+  return rules
+}
+
+// `base` is the directory's path and a `/`; `bytes` is undefined where it
+// has no .gitattributes, null where that is no file.
+export function withAttributes(above, base, bytes) {
+  if (above === null || bytes === null) return null
+  const rules = bytes === undefined ? [] : parseAttributes(bytes)
+  return rules && [...above, { base, rules }]
+}
+
+// convert.c's text test for `text=auto`, on the blob these bytes are written from.
+function isAutoText(written) {
+  let printable = 0
+  let other = written.at(-1) === 0x1a ? -1 : 0
+  for (let i = 0; i < written.length; i++) {
+    const c = written[i]
+    if (c === 0x0d && written[i + 1] !== 0x0a) return false
+    if (c === 0x0d || c === 0x0a) continue
+    if (c === 0) return false
+    if (c >= 0x20 ? c !== 0x7f : PRINTING.has(c)) printable++
+    else other++
+  }
+  return printable >> 7 >= other
+}
+
+// attr.c's order: the deepest file and its last line first. Anything else
+// rewriting the file (ident, filter, working-tree-encoding) refuses it.
+export function writtenWithCrlf(files, path, written) {
+  if (files === null) return false
+  const states = new Map()
+  const fill = (list) => {
+    for (const [name, value] of list.toReversed()) {
+      if (states.has(name)) continue
+      states.set(name, value)
+      if (name === 'binary' && value === true) fill(BINARY)
+    }
+  }
+  for (const { base, rules } of files.toReversed()) {
+    for (const { match, states: list } of rules.toReversed()) if (match(path.slice(base.length))) fill(list)
+  }
+  if (states.get('ident') === true || typeof states.get('filter') === 'string' || states.get('working-tree-encoding')) return false
+  const action = [states.get('text'), states.get('crlf')].find((value) => typeof value === 'boolean' || value === 'input' || value === 'auto')
+  if (action === false || states.get('eol') !== 'crlf') return false
+  return action !== 'auto' || isAutoText(written)
+}

package/src/cache.js

@@ -1,18 +1,50 @@
 import assert from 'node:assert/strict'
 import { constants } from 'node:fs'
 import { mkdir, open, rename, rm, writeFile } from 'node:fs/promises'
-import { dirname, join, resolve } from 'node:path'
+import { homedir } from 'node:os'
+import { dirname, isAbsolute, join, resolve } from 'node:path'
 
-import { assertDirectoryPath } from './args.js'
+import { assertCacheName, assertDirectoryPath, isRepo } from './args.js'
 import { MAX_BYTES, decode } from './http.js'
 
-const DIRS = new Set(['npm/repos', 'npm/tarballs', 'cargo/repos', 'cargo/crates', 'composer/repos', 'soldeer/zips'])
+const DIRS = new Set(['npm/repos', 'npm/tarballs', 'cargo/repos', 'cargo/crates', 'composer/repos', 'soldeer/repos', 'soldeer/zips', 'github/trees'])
 const RECORD_TTL_MS = 30 * 24 * 60 * 60 * 1000 // A link only moves on a transfer or rename, and GitHub redirects those.
 
+// An environment variable may hold anything: only an absolute path is one.
+const absolute = (path) => (typeof path === 'string' && isAbsolute(path) ? path : undefined)
+const within = (base, ...names) => (base === undefined ? undefined : join(base, ...names))
+
+// os.homedir() takes $HOME as it is, relative or empty, and throws where
+// neither it nor the user's passwd entry has one.
+function home() {
+  try {
+    return absolute(homedir())
+  } catch {
+    return undefined
+  }
+}
+
+// Only Linux and the BSDs follow the XDG Base Directory spec, so macOS and
+// Windows take no XDG_CACHE_HOME, as Go's os.UserCacheDir and Rust's dirs
+// take none.
+export function cacheDirFor(name) {
+  assertCacheName('cacheDirFor', 'name', name)
+  if (process.platform === 'darwin') return within(home(), 'Library', 'Caches', name)
+  if (process.platform === 'win32') return within(absolute(process.env.LOCALAPPDATA) ?? within(home(), 'AppData', 'Local'), name, 'Cache')
+  return within(absolute(process.env.XDG_CACHE_HOME) ?? within(home(), '.cache'), name)
+}
+
+export const defaultCacheDir = cacheDirFor('PreventiveMeasures')
+
 let root
 let tmpSeq = 0
 
-export function setCacheDir(dir) {
+export function setCacheDir(dir = defaultCacheDir) {
+  if (dir === false) {
+    root = undefined
+    return
+  }
+  assert.ok(dir !== undefined, 'setCacheDir: there is no default cache directory, as no absolute home directory is known: give one')
   assertDirectoryPath('setCacheDir', 'dir', dir)
   root = resolve(dir)
 }
@@ -24,12 +56,18 @@ export function setCacheDir(dir) {
 // would still meet, so a capital is written `!` and the letter, and a `!`
 // as `!!`. A Windows device name (`con.json` is the console) has its first
 // letter escaped, which encodeURIComponent never does.
-function cachePath(dir, key) {
+function cachePath(dir, key, base = root) {
   assert.ok(DIRS.has(dir) && key && typeof key === 'string', `Unexpected cache entry: ${dir}`)
   const name = encodeURIComponent(key.replace(/[!A-Z]/gu, (char) => `!${char.toLowerCase()}`)).replaceAll('%40', '@').replaceAll('%2F', '+')
-  return root === undefined ? null : join(root, dir, name.replace(/^(?=(?:con|prn|aux|nul|com\d|lpt\d)(?:\.|$))./u, (char) => `%${char.codePointAt(0).toString(16).toUpperCase()}`))
+  return base === undefined ? null : join(base, dir, name.replace(/^(?=(?:con|prn|aux|nul|com\d|lpt\d)(?:\.|$))./u, (char) => `%${char.codePointAt(0).toString(16).toUpperCase()}`))
 }
 
+// Caches of ours, read whether set or not for what is checked whatever its
+// source: the default one, and stasis's, which it sets with setCacheDir.
+const OURS = [defaultCacheDir, cacheDirFor('stasis')].filter((base) => base !== undefined)
+
+export const ourCachePaths = (dir, key) => OURS.map((base) => cachePath(dir, key, base))
+
 // Only a regular file, checked on the file as opened, so one swapped in
 // after a check can't pass: a FIFO or a device would block or never end.
 // Read to the size it had when opened, and no further.
@@ -93,3 +131,21 @@ export async function readRecord(dir, name) {
 }
 
 export const writeRecord = (dir, name, value) => writeCacheJSON(dir, `${name}.json`, { at: Date.now(), name, ...value })
+
+// Name → the GitHub repo cached for it, for those of `names` that have one.
+export async function readRepos(dir, names) {
+  const repos = new Map()
+  for (const name of names) {
+    const entry = await readRecord(dir, name)
+    if (isRepo(entry?.github)) repos.set(name, entry.github)
+  }
+  return repos
+}
+
+// An answer into `repos`, and into the cache where it found a repo.
+export async function addRepos(dir, repos, answer) {
+  for (const [name, github] of answer) {
+    repos.set(name, github)
+    if (github) await writeRecord(dir, name, { github })
+  }
+}

package/src/{cargo.js → cargo/crate.js}

@@ -3,16 +3,16 @@ import { readdir } from 'node:fs/promises'
 import { homedir } from 'node:os'
 import { join, resolve } from 'node:path'
 
-import { assertCrateName, assertCrateVersion, assertSha256, printable, show } from './args.js'
-import { verifiedDownload } from './download.js'
-import { CRATES_INDEX, CRATES_STATIC, buildUrl, encodeSegment, request } from './http.js'
+import { assertCrate, assertSha256, printable, show } from '../args.js'
+import { verifiedDownload } from '../download.js'
+import { CRATES_INDEX, CRATES_STATIC, buildUrl, encodeSegment, request } from '../http.js'
 
 const DIR = 'cargo/crates' // No expiry: crates.io never takes a version twice.
 
 // The sparse index, what cargo reads: a file per crate, by its lowercased
 // name under 1/, 2/, 3/<first letter>/ or <1st-2nd>/<3rd-4th>/, with a
 // JSON line per version, `cksum` the .crate's sha256.
-async function getChecksum(name, version) {
+async function getChecksum(method, name, version) {
   const lower = name.toLowerCase()
   const prefix = lower.length < 3 ? [String(lower.length)] : lower.length === 3 ? ['3', lower[0]] : [lower.slice(0, 2), lower.slice(2, 4)]
   const text = await request(buildUrl(CRATES_INDEX, [...prefix, lower]), { as: 'text' })
@@ -20,13 +20,13 @@ async function getChecksum(name, version) {
     try {
       return JSON.parse(line)
     } catch {
-      assert.fail(`getCrate: the index for ${name} has a line that is not JSON: ${printable(line.slice(0, 200))}`)
+      assert.fail(`${method}: the index for ${name} has a line that is not JSON: ${printable(line.slice(0, 200))}`)
     }
   }
   const entry = text.split('\n').filter(Boolean).map(parse).find((candidate) => candidate?.vers === version)
-  assert.ok(entry, `getCrate: the index has no ${name}@${version}`)
-  assert.ok(entry.name === name, `getCrate: the index answered for ${show(entry.name)}, not ${name}`)
-  assertSha256('getCrate', 'the index cksum', entry.cksum)
+  assert.ok(entry, `${method}: the index has no ${name}@${version}`)
+  assert.ok(entry.name === name, `${method}: the index answered for ${show(entry.name)}, not ${name}`)
+  assertSha256(method, 'the index cksum', entry.cksum)
   return entry.cksum
 }
 
@@ -37,11 +37,17 @@ async function localPaths(name, version) {
   return (await readdir(root).catch(() => [])).map((registry) => join(root, registry, `${name}-${version}.crate`))
 }
 
+export async function verifyChecksum(name, version, checksum) {
+  assertCrate('verifyChecksum', name, version)
+  assertSha256('verifyChecksum', 'checksum', checksum)
+  const cksum = await getChecksum('verifyChecksum', name, version)
+  assert.ok(cksum === checksum, `verifyChecksum: ${name}@${version} is ${cksum} on crates.io, not ${checksum}`)
+}
+
 export async function getCrate(name, version, checksum) {
-  assertCrateName('getCrate', 'name', name)
-  assertCrateVersion('getCrate', 'version', version)
+  assertCrate('getCrate', name, version)
   if (checksum !== undefined) assertSha256('getCrate', 'checksum', checksum)
-  const expected = checksum ?? await getChecksum(name, version)
+  const expected = checksum ?? await getChecksum('getCrate', name, version)
   const locate = () => buildUrl(CRATES_STATIC, ['crates', name, `${name}-${encodeSegment(version)}.crate`])
   return await verifiedDownload({ method: 'getCrate', dir: DIR, what: `${name}@${version}`, ext: 'crate', algorithm: 'sha256', expected, local: await localPaths(name, version), locate })
 }

package/src/cargo/repos.js

@@ -0,0 +1,52 @@
+import assert from 'node:assert/strict'
+import { setTimeout as sleep } from 'node:timers/promises'
+
+import { assertCrateName, assertNames, show } from '../args.js'
+import { addRepos, readRepos } from '../cache.js'
+import { CRATES_API, buildUrl, request } from '../http.js'
+import { chunks } from '../pool.js'
+import { githubRepoOfUrl } from '../remote.js'
+
+const DIR = 'cargo/repos'
+const PER_REQUEST = 100
+const PACE_MS = 1000 // crates.io asks for a request a second at most, and a user agent.
+const USER_AGENT = '@preventive/upstream (https://github.com/PreventiveMeasures/libraries)'
+
+// One request for up to a hundred crates: each name → the GitHub repo its
+// `repository` names, or null where it names none. A crate crates.io
+// leaves out of its answer is one it does not have, null too.
+async function askCrates(method, names) {
+  const url = buildUrl(CRATES_API, ['api', 'v1', 'crates'], { 'ids[]': names, per_page: PER_REQUEST })
+  const answer = await request(url, { as: 'json', headers: { 'User-Agent': USER_AGENT } })
+  assert.ok(Array.isArray(answer?.crates), `${method}: expected a list of crates from crates.io`)
+  const found = new Map(names.map((name) => [name, null]))
+  for (const crate of answer.crates) {
+    assert.ok(names.includes(crate?.id), `${method}: crates.io answered for ${show(crate?.id)}, which was not asked`)
+    found.set(crate.id, githubRepoOfUrl(crate.repository) ?? null)
+  }
+  return found
+}
+
+// Name → its GitHub repo, or null where crates.io has no such crate or it
+// names no GitHub repo, through the cache: only a repo found is cached,
+// for a month. A failed request throws, or with `soft` leaves its names
+// out; `cachedOnly` asks nothing.
+export async function lookUpCrateRepos(method, names, { soft = false, cachedOnly = false } = {}) {
+  const repos = await readRepos(DIR, names)
+  if (cachedOnly) return repos
+  for (const [i, chunk] of chunks(names.filter((name) => !repos.has(name)), PER_REQUEST).entries()) {
+    if (i > 0) await sleep(PACE_MS)
+    const answer = await askCrates(method, chunk).catch((error) => {
+      if (soft) return new Map()
+      throw error
+    })
+    await addRepos(DIR, repos, answer)
+  }
+  return repos
+}
+
+export async function resolveCrateRepos(crateNames, options = {}) {
+  const names = assertNames('resolveCrateRepos', 'crateNames', crateNames, options, assertCrateName)
+  const repos = await lookUpCrateRepos('resolveCrateRepos', names, { soft: true, cachedOnly: options.cachedOnly })
+  return new Map(names.flatMap((name) => (repos.get(name) ? [[name, { github: repos.get(name) }]] : [])))
+}

package/src/download.js

@@ -1,38 +1,42 @@
 import assert from 'node:assert/strict'
 import { createHash } from 'node:crypto'
 
-import { readCache, readRegularFile, writeCache } from './cache.js'
+import { ourCachePaths, readCache, readRegularFile, writeCache } from './cache.js'
 import { request } from './http.js'
+import { gitTreeOfTarball } from './tree.js'
 
 // A hash as each registry writes it: npm's integrity, one sha512 in base64
-// after `sha512-`; crates.io's and Soldeer's checksum, a sha256 in hex.
+// after `sha512-`; crates.io's and Soldeer's checksum, a sha256 in hex; a
+// git tree id for a tarball of that tree.
 const DIGESTS = {
   sha512: (bytes) => `sha512-${createHash('sha512').update(bytes).digest('base64')}`,
   sha256: (bytes) => createHash('sha256').update(bytes).digest('hex'),
+  tree: gitTreeOfTarball,
 }
 
 // Bytes whose `algorithm` hash is `expected`, which the registry or the
 // caller gives, never a disk; without it nothing is read. From `local`,
-// other tools' caches (read, never written; a mismatch is passed over),
-// else ours (a mismatch throws), else what `locate` answers, checked and
-// then cached.
-export async function verifiedDownload({ method, dir, what, ext, algorithm, expected, local = [], locate }) {
+// other tools' caches, and with `ours` our default cache and stasis's,
+// whether set or not (read, never written; a mismatch is passed over),
+// else the cache set (a mismatch throws), else what `locate` answers,
+// checked and then cached.
+export async function verifiedDownload({ method, dir, what, ext, algorithm, expected, local = [], ours = false, locate, options = {}, list }) {
   assert.ok(Object.hasOwn(DIGESTS, algorithm) && typeof expected === 'string' && expected !== '', `${method}: nothing to check ${what} against`)
-  const digest = DIGESTS[algorithm]
-  for (const path of local) {
+  const digest = (bytes) => DIGESTS[algorithm](bytes, { expected, list })
+  const key = `${what}.${ext}`
+  for (const path of ours ? [...local, ...ourCachePaths(dir, key)] : local) {
     const bytes = await readRegularFile(path)
-    if (bytes && digest(bytes) === expected) return bytes
+    if (bytes && await digest(bytes) === expected) return bytes
   }
-  const check = (bytes, from) => {
-    const actual = digest(bytes)
+  const check = async (bytes, from) => {
+    const actual = await digest(bytes)
     assert.ok(actual === expected, `${method}: integrity mismatch for ${what} from ${from}: expected ${expected}, got ${actual}`)
     return bytes
   }
-  const key = `${what}.${ext}`
   const cached = await readCache(dir, key)
-  if (cached) return check(cached, 'the cache')
+  if (cached) return await check(cached, 'the cache')
   const url = await locate()
-  const bytes = check(await request(url, { as: 'bytes' }), url)
+  const bytes = await check(await request(url, { ...options, as: 'bytes' }), url)
   await writeCache(dir, key, bytes)
   return bytes
 }

package/src/github/read.js

@@ -1,9 +1,10 @@
 import assert from 'node:assert/strict'
 import { Buffer } from 'node:buffer'
-import { createHash } from 'node:crypto'
 
-import { assertArgs, assertGhsa, assertLogin, assertNumber, assertPath, assertRef, assertRepo, assertSha, isSha, optional, sameName, show } from '../args.js'
-import { decode, encodeSegment } from '../http.js'
+import { assertArgs, assertGhsa, assertLogin, assertNumber, assertPath, assertRef, assertRepo, assertSha, assertTagName, assertTreeId, isSha, isSha1, isTagName, optional, sameName, show } from '../args.js'
+import { verifiedDownload } from '../download.js'
+import { decode, encodeSegment, requestWithHeaders } from '../http.js'
+import { gitTreeOfListing, objectId } from '../tree.js'
 import { api, bindMethods, call, clientHeaders, isGone, repoApi } from './client.js'
 
 const PER_PAGE = 100
@@ -20,6 +21,24 @@ async function* pages(method, headers, pageUrl, maxPages = MAX_PAGES) {
   }
 }
 
+// A list GitHub pages by a cursor instead: the `after` of each page's
+// Link header `next` link, the one thing read off the link, so the next
+// page is asked of the list `pageUrl` names, wherever the link points. A
+// next link proves a page more.
+async function* cursorPages(method, headers, pageUrl, maxPages = MAX_PAGES) {
+  const paging = { per_page: PER_PAGE }
+  for (let page = 1; ; page++) {
+    const answer = await requestWithHeaders(pageUrl(paging), { as: 'json', headers })
+    assert.ok(Array.isArray(answer.body), `${method}: expected an array for page ${page}`)
+    yield answer.body
+    const next = /<([^>]*)>\s*;\s*rel="next"/u.exec(answer.headers.get('link') ?? '')?.[1]
+    if (next === undefined) return
+    paging.after = URL.parse(next)?.searchParams.get('after')
+    assert.ok(paging.after, `${method}: page ${page} links the next with no cursor`)
+    assert.ok(page < maxPages, `${method}: more than ${maxPages} pages`)
+  }
+}
+
 const paginate = async (...args) => (await Array.fromAsync(pages(...args))).flat()
 const getCurrentUser = (headers) => call(headers, api(['user']))
 
@@ -46,6 +65,32 @@ export async function getRepoHead(headers, options) {
   return { branch: ref, oid: data.object.sha }
 }
 
+async function getRepoTag(headers, options) {
+  assertArgs('getRepoTag', options, { repo: assertRepo, tag: assertTagName })
+  const { repo, tag } = options
+  const ref = await call(headers, repoApi(repo, ['git', 'ref', 'tags', encodeSegment(tag)]))
+  assert.ok(ref?.ref === `refs/tags/${tag}`, `getRepoTag: GitHub answered for ${show(ref?.ref)}, not ${show(`refs/tags/${tag}`)}`)
+  let { object } = ref
+  for (let hops = 0; object?.type === 'tag' && isSha(object.sha) && hops < 8; hops++) {
+    const annotated = await call(headers, repoApi(repo, ['git', 'tags', object.sha]))
+    assert.ok(annotated?.sha === object.sha, `getRepoTag: GitHub answered for another tag object than ${object.sha}`)
+    object = annotated.object
+  }
+  assert.ok(object?.type === 'commit' && isSha(object.sha), `getRepoTag: ${repo} has no commit for tag ${show(tag)}`)
+  return { tag, oid: object.sha }
+}
+
+// GitHub lists each tag with the commit it names, an annotated one's too.
+async function listRepoTags(headers, options) {
+  assertArgs('listRepoTags', options, { repo: assertRepo, maxPages: optional(assertNumber) })
+  const { repo, maxPages } = options
+  const tags = await paginate('listRepoTags', headers, (paging) => repoApi(repo, ['tags'], paging), maxPages)
+  return tags.map((entry) => {
+    assert.ok(isTagName(entry?.name) && isSha(entry.commit?.sha), `listRepoTags: GitHub listed ${show(entry?.name)} in ${repo}, which is no tag name with a commit`)
+    return { tag: entry.name, oid: entry.commit.sha }
+  })
+}
+
 // GitHub's object for the path first: a directory, symlink or submodule is
 // refused rather than read as a file's text. Up to 1 MB the object holds
 // the content; past it, `encoding: none`, and the blob it names is read,
@@ -61,16 +106,73 @@ async function getRepoFile(headers, options) {
   const bytes = file.encoding === 'none'
     ? await call({ ...headers, Accept: 'application/vnd.github.raw' }, repoApi(repo, ['git', 'blobs', file.sha]), { as: 'bytes' })
     : Buffer.from(file.content, 'base64')
-  const blob = createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex')
+  const blob = objectId('blob', bytes).toString('hex')
   assert.ok(bytes.length === file.size && blob === file.sha, `getRepoFile: ${show(path)} came back as blob ${blob}, not ${file.sha}`)
   return decode(bytes, url)
 }
 
-// The one request that follows a redirect, to codeload.github.com. A full
-// sha only, so the bytes are that commit's, not wherever a ref points now.
+// Follows the redirect to codeload.github.com. A tree id names its content,
+// so the bytes are held to it, downloaded or cached, and cached by it alone,
+// for good. What a tarball cannot show, a submodule's commit, a subtree
+// with nothing in it or the LF blob of a file marked `eol=crlf`, comes
+// from GitHub's listings of the trees, which the id checks as well: a
+// directory at a time, as a recursive listing of a large tree is cut short.
+async function treeTarball(method, headers, repo, tree) {
+  const listings = new Map()
+  const list = (sha) => {
+    if (!listings.has(sha)) listings.set(sha, call(headers, repoApi(repo, ['git', 'trees', sha])).then((listing) => (Array.isArray(listing?.tree) ? listing.tree : [])))
+    return listings.get(sha)
+  }
+  const locate = () => repoApi(repo, ['tarball', tree])
+  return await verifiedDownload({ method, dir: 'github/trees', what: tree, ext: 'tgz', algorithm: 'tree', expected: tree, locate, options: { headers, redirect: 'follow' }, list })
+}
+
+async function commitTree(method, headers, repo, sha) {
+  const commit = await call(headers, repoApi(repo, ['git', 'commits', sha]))
+  assert.ok(commit?.sha === sha && isSha1(commit.tree?.sha), `${method}: GitHub names no tree for ${repo}@${sha}`)
+  return commit.tree.sha
+}
+
+// A full sha only, so the bytes are that commit's, not wherever a ref points
+// now. Its tree's tarball, not its own, in which `git archive` rewrites the
+// files marked `export-subst`.
 async function getRepoTarball(headers, options) {
   assertArgs('getRepoTarball', options, { repo: assertRepo, sha: assertSha })
-  return await call(headers, repoApi(options.repo, ['tarball', options.sha]), { as: 'bytes', redirect: 'follow' })
+  const { repo, sha } = options
+  const tree = await commitTree('getRepoTarball', headers, repo, sha)
+  return await treeTarball('getRepoTarball', headers, repo, tree)
+}
+
+async function getRepoTreeTarball(headers, options) {
+  assertArgs('getRepoTreeTarball', options, { repo: assertRepo, tree: assertTreeId })
+  return await treeTarball('getRepoTreeTarball', headers, options.repo, options.tree)
+}
+
+// Not recursive: GitHub cuts short a recursive listing of a large tree.
+async function listTree(method, headers, repo, tree) {
+  const entries = (await call(headers, repoApi(repo, ['git', 'trees', tree])))?.tree
+  assert.ok(Array.isArray(entries) && gitTreeOfListing(entries) === tree, `${method}: GitHub's listing of tree ${tree} in ${repo} is not that tree`)
+  return entries.map(({ path, mode, type, sha }) => ({ path, mode, type, sha }))
+}
+
+async function treeAt(method, headers, options) {
+  assertArgs(method, options, { repo: assertRepo, sha: assertSha, directory: optional(assertPath), path: optional(assertPath) })
+  assert.ok(options.directory === undefined || options.path === undefined, `${method}: directory and path are one option, pass directory alone`)
+  const { repo, sha, directory = options.path } = options
+  let tree = await commitTree(method, headers, repo, sha)
+  for (const name of directory?.split('/') ?? []) {
+    const entry = (await listTree(method, headers, repo, tree)).find((candidate) => candidate.path === name)
+    assert.ok(entry?.type === 'tree', `${method}: ${repo}@${sha} has no directory at ${show(directory)}`)
+    tree = entry.sha
+  }
+  return tree
+}
+
+const getRepoTreeId = (headers, options) => treeAt('getRepoTreeId', headers, options)
+
+async function listRepoDir(headers, options) {
+  const tree = await treeAt('listRepoDir', headers, options)
+  return await listTree('listRepoDir', headers, options.repo, tree)
 }
 
 async function getPullRequest(headers, options) {
@@ -110,15 +212,12 @@ async function getAdvisory(headers, options) {
   return advisory
 }
 
-// One page: GitHub pages this list by a cursor in the Link header, which
-// `call` does not read, so a full page is refused rather than cut short.
 async function listRepoAdvisories(headers, options) {
   assertArgs('listRepoAdvisories', options, { repo: assertRepo })
-  const list = await call(headers, repoApi(options.repo, ['security-advisories'], { state: 'published', per_page: PER_PAGE }))
-  assert.ok(Array.isArray(list), `listRepoAdvisories: expected an array for ${options.repo}`)
-  assert.ok(list.length < PER_PAGE, `listRepoAdvisories: ${options.repo} has ${PER_PAGE} or more published advisories, more than a page`)
-  return list
+  const { repo } = options
+  const pageUrl = (paging) => repoApi(repo, ['security-advisories'], { state: 'published', ...paging })
+  return (await Array.fromAsync(cursorPages('listRepoAdvisories', headers, pageUrl))).flat()
 }
 
-export const readMethods = { getCurrentUser, listUserRepos, getRepo, getRepoHead, getRepoFile, getRepoTarball, getPullRequest, getCollaboratorPermission, getAdvisory, listRepoAdvisories }
+export const readMethods = { getCurrentUser, listUserRepos, getRepo, getRepoHead, getRepoTag, listRepoTags, getRepoFile, getRepoTarball, getRepoTreeTarball, getRepoTreeId, listRepoDir, getPullRequest, getCollaboratorPermission, getAdvisory, listRepoAdvisories }
 export const createClient = (options) => bindMethods(clientHeaders('createClient', options, true), readMethods)

package/src/http.js

@@ -101,22 +101,27 @@ export async function readBody(res, limit, { truncate = false } = {}) {
   return new Uint8Array(await new Blob(chunks).arrayBuffer(), 0, Math.min(size, limit))
 }
 
+function parse(bytes, url, as) {
+  if (as === 'bytes') return bytes
+  const text = decode(bytes, url)
+  if (as === 'text') return text
+  try {
+    return JSON.parse(text)
+  } catch (cause) {
+    throw new Error(`Malformed JSON from ${url}: ${printable(text.slice(0, 200))}`, { cause })
+  }
+}
+
 // Read as the caller says, not by content type, which proxies drop or
-// rewrite.
-export async function request(url, options) {
+// rewrite; answered with the response's headers.
+export async function requestWithHeaders(url, options) {
   const res = await send(url, options)
   if (!res.ok) {
     // A stream decode holds back a character cut at the limit rather than refusing it.
     const text = await readBody(res, 4096, { truncate: true }).then((bytes) => new TextDecoder('utf-8', { fatal: true }).decode(bytes, { stream: true })).catch(() => '')
     throw new HttpError(res.status, `${options.method ?? 'GET'} ${url} ${res.status}: ${printable(text)}`)
   }
-  const bytes = await readBody(res, LIMITS[options.as].bytes)
-  if (options.as === 'bytes') return bytes
-  const text = decode(bytes, url)
-  if (options.as === 'text') return text
-  try {
-    return JSON.parse(text)
-  } catch (cause) {
-    throw new Error(`Malformed JSON from ${url}: ${printable(text.slice(0, 200))}`, { cause })
-  }
+  return { body: parse(await readBody(res, LIMITS[options.as].bytes), url, options.as), headers: res.headers }
 }
+
+export const request = async (url, options) => (await requestWithHeaders(url, options)).body

package/src/npm/registry.js

@@ -0,0 +1,19 @@
+import assert from 'node:assert/strict'
+
+import { show } from '../args.js'
+import { NPM_REGISTRY, buildUrl, request } from '../http.js'
+
+// Private packages are always scoped.
+export function withNpmToken(name, url, options = {}) {
+  const method = options.method ?? 'GET'
+  assert.ok(method === 'GET' && `${new URL(url)}` === url && url.startsWith(`${NPM_REGISTRY}/${name}/`), `Unexpected request for ${name}: ${method} ${url}`)
+  const token = process.env.NPM_TOKEN
+  return token && name.startsWith('@') ? { ...options, method, headers: { ...options.headers, Authorization: `Bearer ${token}` } } : options
+}
+
+export async function getDocument(method, name, spec) {
+  const url = buildUrl(NPM_REGISTRY, [...name.split('/'), spec])
+  const json = await request(url, withNpmToken(name, url, { as: 'json' }))
+  assert.ok(json?.name === name, `${method}: the registry answered for ${show(json?.name)}, not ${name}`)
+  return json
+}