@preventive/upstream 1.0.0-alpha.0 → 1.0.0-alpha.10

package/src/advisories/repos.js

@@ -1,54 +1,24 @@
 import assert from 'node:assert/strict'
-import { setTimeout as sleep } from 'node:timers/promises'
 
-import { isRepo, show } from '../args.js'
-import { readCacheJSON, writeCacheJSON } from '../cache.js'
-import { CRATES_API, HttpError, PACKAGIST_REPO, buildUrl, request } from '../http.js'
+import { show } from '../args.js'
+import { addRepos, readRepos } from '../cache.js'
+import { lookUpCrateRepos } from '../cargo/repos.js'
+import { PACKAGIST_REPO, SOLDEER_API, buildUrl, isNotFound, recover, request } from '../http.js'
 import { lookUpPackageRepo } from '../npm/repos.js'
 import { pool } from '../pool.js'
 import { githubRepoOfUrl } from '../remote.js'
 
-const TTL_MS = 30 * 24 * 60 * 60 * 1000
-const CRATES_PER_REQUEST = 100
-const CRATES_PACE_MS = 1000 // crates.io asks for a request a second at most, and a user agent.
-const USER_AGENT = '@preventive/upstream (https://github.com/PreventiveMeasures/libraries)'
 const PACKAGES_AT_ONCE = 8
 
 // Name → its GitHub repo, or null where the registry has no such package
 // or it names no GitHub repo; any other failure throws. Only a repo found
 // is cached, for a month.
 async function cachedRepos(dir, names, fetchMissing) {
-  const repos = new Map()
-  for (const name of names) {
-    const entry = await readCacheJSON(dir, `${name}.json`)
-    const age = typeof entry?.at === 'number' ? Date.now() - entry.at : Number.NaN
-    if (age >= 0 && age <= TTL_MS && entry.name === name && isRepo(entry.github)) repos.set(name, entry.github)
-  }
-  const missing = names.filter((name) => !repos.has(name))
-  for (const [name, github] of await fetchMissing(missing)) {
-    repos.set(name, github)
-    if (github) await writeCacheJSON(dir, `${name}.json`, { at: Date.now(), name, github })
-  }
+  const repos = await readRepos(dir, names)
+  await addRepos(dir, repos, await fetchMissing(names.filter((name) => !repos.has(name))))
   return repos
 }
 
-// A crate crates.io leaves out of its answer is one it does not have.
-async function fetchCrates(names) {
-  const found = new Map(names.map((name) => [name, null]))
-  for (let i = 0; i < names.length; i += CRATES_PER_REQUEST) {
-    if (i > 0) await sleep(CRATES_PACE_MS)
-    const chunk = names.slice(i, i + CRATES_PER_REQUEST)
-    const url = buildUrl(CRATES_API, ['api', 'v1', 'crates'], { 'ids[]': chunk, per_page: CRATES_PER_REQUEST })
-    const answer = await request(url, { as: 'json', headers: { 'User-Agent': USER_AGENT } })
-    assert.ok(Array.isArray(answer?.crates), 'advisories: expected a list of crates from crates.io')
-    for (const crate of answer.crates) {
-      assert.ok(chunk.includes(crate?.id), `advisories: crates.io answered for ${show(crate?.id)}, which was not asked`)
-      found.set(crate.id, githubRepoOfUrl(crate.repository) ?? null)
-    }
-  }
-  return found
-}
-
 async function fetchEach(names, fetch) {
   const found = await pool(names, PACKAGES_AT_ONCE, fetch)
   return new Map(names.map((name, i) => [name, found[i]]))
@@ -57,16 +27,22 @@ async function fetchEach(names, fetch) {
 // The first version in Packagist's file is its latest, in full.
 async function fetchComposerRepo(name) {
   const [vendor, pkg] = name.split('/')
-  const answer = await request(buildUrl(PACKAGIST_REPO, ['p2', vendor, `${pkg}.json`]), { as: 'json' }).catch((err) => {
-    if (err instanceof HttpError && err.status === 404) return null
-    throw err
-  })
+  const answer = await request(buildUrl(PACKAGIST_REPO, ['p2', vendor, `${pkg}.json`]), { as: 'json' }).catch(recover(isNotFound, null))
   if (answer === null) return null
-  const versions = Object.hasOwn(answer?.packages ?? {}, name) ? answer.packages[name] : undefined
+  const versions = Object.hasOwn(answer.packages ?? {}, name) ? answer.packages[name] : undefined
   assert.ok(Array.isArray(versions), `advisories: Packagist answered without ${name}`)
   return githubRepoOfUrl(versions[0]?.source?.url) ?? null
 }
 
-export const crateRepos = (names) => cachedRepos('cargo/repos', names, fetchCrates)
+// Soldeer answers a project it does not have with an empty list.
+async function fetchSoldeerRepo(name) {
+  const answer = await request(buildUrl(SOLDEER_API, ['api', 'v1', 'project'], { project_name: name }), { as: 'json' })
+  assert.ok(Array.isArray(answer?.data), `advisories: expected a list of projects from Soldeer for ${name}`)
+  for (const project of answer.data) assert.ok(project?.name === name, `advisories: Soldeer answered for ${show(project?.name)}, not ${name}`)
+  return githubRepoOfUrl(answer.data[0]?.github_url) ?? null
+}
+
+export const crateRepos = (names) => lookUpCrateRepos('advisories', names)
 export const composerRepos = (names) => cachedRepos('composer/repos', names, (missing) => fetchEach(missing, fetchComposerRepo))
 export const npmRepos = (names) => fetchEach(names, async (name) => (await lookUpPackageRepo(name))?.github ?? null)
+export const soldeerRepos = (names) => cachedRepos('soldeer/repos', names, (missing) => fetchEach(missing, fetchSoldeerRepo))

package/src/args.js

@@ -15,7 +15,9 @@ const isLogin = matches(/^(?=.{1,39}$)[a-z\d](?:-?[a-z\d])*$/iu)
 const isRepoName = matches(/^(?!\.\.?$)[\w.-]{1,100}$/u)
 // `.git` is never in a repo's tree, and no commit should write into it.
 export const isRepoPath = (value) => isString(value, 4096) && value.split('/').every((part) => !['', '.', '..', '.git'].includes(part.toLowerCase()))
-const BAD_REF = /^$|^@$|^-|[ ~^:?*[\\]|\.\.|@\{|^\/|\/$|\/\/|(?:^|\/)\.|\.lock(?:\/|$)|\.$/u // git check-ref-format, a rule per alternative
+// git check-ref-format, a rule per alternative. A tag's name may be `@` or
+// start with `-`; a branch's may not.
+const BAD_REF = /^$|[ ~^:?*[\\]|\.\.|@\{|^\/|\/$|\/\/|(?:^|\/)\.|\.lock(?:\/|$)|\.$/u
 const isToken = matches(/^[!-~]+$/u)
 // npm's rules for existing names: capitals allowed (JSONStream), the
 // legacy `~'!()*` not.
@@ -37,8 +39,10 @@ export function isRepo(value) {
 }
 
 export const isStrings = (value) => Array.isArray(value) && value.every((item) => typeof item === 'string')
+export const isIterable = (value) => typeof value?.[Symbol.iterator] === 'function' && typeof value !== 'string'
 export const sameName = (a, b) => typeof a === 'string' && a.toLowerCase() === b.toLowerCase()
-export const isRefName = (value) => isString(value, 255) && !BAD_REF.test(value)
+export const isTagName = (value) => isString(value, 255) && !BAD_REF.test(value)
+export const isRefName = (value) => isTagName(value) && !/^(?:@$|-)/u.test(value)
 export const isSha = matches(/^(?:[\da-f]{40}|[\da-f]{64})$/u)
 export const isGhsa = matches(/^GHSA(?:-[\da-hj-km-np-tv-z]{4}){3}$/u)
 export const assertion = (must, predicate) => (method, what, value) => assert.ok(predicate(value), `${method}: ${what} must be ${must}, got ${show(value)}`)
@@ -52,17 +56,33 @@ export function assertArgs(method, options, spec, name) {
   for (const [key, check] of Object.entries(spec)) check?.(method, label(key), options[key])
 }
 
+// The names a resolve*Repos call is given, each checked and given once,
+// and its options.
+export function assertNames(method, what, names, options, assertName) {
+  assert.ok(isIterable(names), `${method}: ${what} must be an iterable of names`)
+  assertArgs(method, options, { cachedOnly: optional(assertBoolean) })
+  const unique = [...new Set(names)]
+  for (const name of unique) assertName(method, 'name', name)
+  return unique
+}
+
 export const assertBoolean = assertion('a boolean', (value) => typeof value === 'boolean')
 export const assertNumber = assertion('a positive integer', (value) => Number.isSafeInteger(value) && value > 0)
 export const assertLine = assertion('a non-empty single line', (value) => isString(value, 1024) && value.trim() !== '')
 // 65,536 is GitHub's own limit on a body.
 export const assertText = assertion('text with no control characters', (value) => isString(value, 65_536, '\t\n\r'))
 export const assertDirectoryPath = assertion('a directory path', (value) => isString(value, 4096) && value !== '')
+// A short ASCII name, which no filesystem refuses, reads as another or takes
+// for a device: none of those is six characters of these alone.
+export const assertCacheName = assertion('6 to 32 of a-z, A-Z, 0-9, _ and -', matches(/^[a-zA-Z0-9_-]{6,32}$/u))
 export const assertLogin = assertion('a GitHub login', isLogin)
 export const assertRepoName = assertion('a repository name', isRepoName)
 export const assertRepo = assertion('"owner/name"', isRepo)
 export const assertRef = assertion('a branch or tag name', isRefName)
+export const assertTagName = assertion('a tag name', isTagName)
 export const assertSha = assertion('a full commit sha', isSha)
+export const isSha1 = matches(/^[\da-f]{40}$/u)
+export const assertTreeId = assertion('a full tree id', isSha1)
 export const assertPath = assertion('a path inside a repository', isRepoPath)
 export const assertToken = assertion('a token', isToken)
 export const assertTokenOrNull = assertion('a token, or null for anonymous access', (value) => value === null || isToken(value))
@@ -70,3 +90,21 @@ export const assertUserAgent = assertion('a printable user agent', matches(/^[ -
 export const assertPackageName = assertion('an npm package name', isPackageName)
 export const assertPackageVersion = assertion('an exact semver version', isExactVersion)
 export const assertGhsa = assertion('a GHSA id', isGhsa)
+export const assertCrateName = assertion('a crate name', matches(/^[A-Za-z][\w-]{0,63}$/u))
+export const assertCrateVersion = assertion('a semver version', matches(/^(?=.{5,256}$)(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:0|[1-9]\d*|\d*[A-Za-z-][\dA-Za-z-]*)(?:\.(?:0|[1-9]\d*|\d*[A-Za-z-][\dA-Za-z-]*))*)?(?:\+[\dA-Za-z-]+(?:\.[\dA-Za-z-]+)*)?$/u))
+// Soldeer's own rule for a name: 3 to 100 of lowercase letters, digits and
+// `-`, starting with one of them or `@`, ending with a letter or digit.
+export const assertSoldeerName = assertion('a Soldeer package name', matches(/^(?=.{3,100}$)[@\da-z][\da-z-]*[\da-z]$/u))
+// Soldeer takes any version but an empty one. Its registry has semver
+// (`5.7.0-rc.0`, `1.0.2-solc-0.8-simulate`), bare numbers and commit hashes.
+export const assertSoldeerVersion = assertion('letters, digits, `.`, `_`, `+` and `-`', matches(/^(?=.{1,128}$)[\dA-Za-z][\w.+-]*$/u))
+// A package's name and version, as a registry's getters take them.
+const assertNameVersion = (assertName, assertVersion) => (method, name, version) => {
+  assertName(method, 'name', name)
+  assertVersion(method, 'version', version)
+}
+export const assertCrate = assertNameVersion(assertCrateName, assertCrateVersion)
+export const assertPackage = assertNameVersion(assertPackageName, assertPackageVersion)
+export const assertSoldeerPackage = assertNameVersion(assertSoldeerName, assertSoldeerVersion)
+// As Cargo.lock and soldeer.lock write a checksum.
+export const assertSha256 = assertion('a sha256 in lowercase hex', matches(/^[\da-f]{64}$/u))

package/src/attributes.js

@@ -0,0 +1,81 @@
+import { Buffer } from 'node:buffer'
+
+// Whether git writes a file out with CRLF, from the .gitattributes in its
+// tree alone, as `git archive` and a checkout with no config read them. Only
+// `*`, `*.ext`, name and path rules are read: any other rule that sets an
+// attribute line endings depend on refuses its file, and all below it.
+
+const RELEVANT = new Set(['text', 'crlf', 'eol', 'binary', 'ident', 'filter', 'working-tree-encoding'])
+const BINARY = [['diff', false], ['merge', false], ['text', false]]
+const GLOB = /[*?[\\]/u
+const PRINTING = new Set([0x08, 0x09, 0x0c, 0x1b]) // BS, HT, FF, ESC
+
+function matcher(pattern) {
+  const name = (path) => path.slice(path.lastIndexOf('/') + 1)
+  if (pattern.endsWith('/')) return () => false // a directory
+  if (!GLOB.test(pattern)) return pattern.includes('/') ? (path) => path === pattern.replace(/^\//u, '') : (path) => name(path) === pattern
+  const end = pattern.slice(1)
+  return pattern[0] === '*' && !GLOB.test(end) && !end.includes('/') ? (path) => name(path).endsWith(end) : null
+}
+
+// Read to the first NUL, as git reads one from a tree. A line with a name
+// git takes for none, or a negative pattern, git leaves out.
+function parseAttributes(bytes) {
+  const nul = bytes.indexOf(0)
+  const rules = []
+  for (const line of Buffer.from(bytes.buffer, bytes.byteOffset, nul === -1 ? bytes.length : nul).toString('latin1').split('\n')) {
+    const [pattern, ...tokens] = line.split(/[ \t\r]+/u).filter(Boolean)
+    if (pattern?.startsWith('"')) return null
+    if (pattern === undefined || pattern.startsWith('#') || pattern.startsWith('!')) continue
+    const states = tokens.map((token) => /^([-!]?)([\w.][\w.-]*)(?:=(.*))?$/u.exec(token))
+    if (states.includes(null) || !states.some(([, , name]) => RELEVANT.has(name))) continue
+    const match = matcher(pattern)
+    if (!match) return null
+    rules.push({ match, states: states.map(([, sign, name, value = true]) => [name, sign === '-' ? false : sign === '!' ? null : value]) })
+  }
+  return rules
+}
+
+// `base` is the directory's path and a `/`; `bytes` is undefined where it
+// has no .gitattributes, null where that is no file.
+export function withAttributes(above, base, bytes) {
+  if (above === null || bytes === null) return null
+  const rules = bytes === undefined ? [] : parseAttributes(bytes)
+  return rules && [...above, { base, rules }]
+}
+
+// convert.c's text test for `text=auto`, on the blob these bytes are written from.
+function isAutoText(written) {
+  let printable = 0
+  let other = written.at(-1) === 0x1a ? -1 : 0
+  for (let i = 0; i < written.length; i++) {
+    const c = written[i]
+    if (c === 0x0d && written[i + 1] !== 0x0a) return false
+    if (c === 0x0d || c === 0x0a) continue
+    if (c === 0) return false
+    if (c >= 0x20 ? c !== 0x7f : PRINTING.has(c)) printable++
+    else other++
+  }
+  return printable >> 7 >= other
+}
+
+// attr.c's order: the deepest file and its last line first. Anything else
+// rewriting the file (ident, filter, working-tree-encoding) refuses it.
+export function writtenWithCrlf(files, path, written) {
+  if (files === null) return false
+  const states = new Map()
+  const fill = (list) => {
+    for (const [name, value] of list.toReversed()) {
+      if (states.has(name)) continue
+      states.set(name, value)
+      if (name === 'binary' && value === true) fill(BINARY)
+    }
+  }
+  for (const { base, rules } of files.toReversed()) {
+    for (const { match, states: list } of rules.toReversed()) if (match(path.slice(base.length))) fill(list)
+  }
+  if (states.get('ident') === true || typeof states.get('filter') === 'string' || states.get('working-tree-encoding')) return false
+  const action = [states.get('text'), states.get('crlf')].find((value) => typeof value === 'boolean' || value === 'input' || value === 'auto')
+  if (action === false || states.get('eol') !== 'crlf') return false
+  return action !== 'auto' || isAutoText(written)
+}

package/src/cache.js

@@ -1,16 +1,50 @@
 import assert from 'node:assert/strict'
-import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'
-import { dirname, join, resolve } from 'node:path'
+import { constants } from 'node:fs'
+import { mkdir, open, rename, rm, writeFile } from 'node:fs/promises'
+import { homedir } from 'node:os'
+import { dirname, isAbsolute, join, resolve } from 'node:path'
 
-import { assertDirectoryPath } from './args.js'
-import { decode } from './http.js'
+import { assertCacheName, assertDirectoryPath, isRepo } from './args.js'
+import { MAX_BYTES, decode } from './http.js'
 
-const DIRS = new Set(['npm/repos', 'npm/tarballs', 'cargo/repos', 'composer/repos'])
+const DIRS = new Set(['npm/repos', 'npm/tarballs', 'cargo/repos', 'cargo/crates', 'composer/repos', 'soldeer/repos', 'soldeer/zips', 'github/trees'])
+const RECORD_TTL_MS = 30 * 24 * 60 * 60 * 1000 // A link only moves on a transfer or rename, and GitHub redirects those.
+
+// An environment variable may hold anything: only an absolute path is one.
+const absolute = (path) => (typeof path === 'string' && isAbsolute(path) ? path : undefined)
+const within = (base, ...names) => (base === undefined ? undefined : join(base, ...names))
+
+// os.homedir() takes $HOME as it is, relative or empty, and throws where
+// neither it nor the user's passwd entry has one.
+function home() {
+  try {
+    return absolute(homedir())
+  } catch {
+    return undefined
+  }
+}
+
+// Only Linux and the BSDs follow the XDG Base Directory spec, so macOS and
+// Windows take no XDG_CACHE_HOME, as Go's os.UserCacheDir and Rust's dirs
+// take none.
+export function cacheDirFor(name) {
+  assertCacheName('cacheDirFor', 'name', name)
+  if (process.platform === 'darwin') return within(home(), 'Library', 'Caches', name)
+  if (process.platform === 'win32') return within(absolute(process.env.LOCALAPPDATA) ?? within(home(), 'AppData', 'Local'), name, 'Cache')
+  return within(absolute(process.env.XDG_CACHE_HOME) ?? within(home(), '.cache'), name)
+}
+
+export const defaultCacheDir = cacheDirFor('PreventiveMeasures')
 
 let root
 let tmpSeq = 0
 
-export function setCacheDir(dir) {
+export function setCacheDir(dir = defaultCacheDir) {
+  if (dir === false) {
+    root = undefined
+    return
+  }
+  assert.ok(dir !== undefined, 'setCacheDir: there is no default cache directory, as no absolute home directory is known: give one')
   assertDirectoryPath('setCacheDir', 'dir', dir)
   root = resolve(dir)
 }
@@ -18,14 +52,47 @@ export function setCacheDir(dir) {
 // URI-encoded, so a `/` or `..` in a key stays inside one file name, then
 // `@` kept and `/` written `+`, as pnpm names its store: `@babel+core@7.29.7`.
 // encodeURIComponent writes neither `@` nor `+` bare, so no two keys meet.
-function cachePath(dir, key) {
+// Where case is not told apart (macOS, Windows), JSONStream and jsonstream
+// would still meet, so a capital is written `!` and the letter, and a `!`
+// as `!!`. A Windows device name (`con.json` is the console) has its first
+// letter escaped, which encodeURIComponent never does.
+function cachePath(dir, key, base = root) {
   assert.ok(DIRS.has(dir) && key && typeof key === 'string', `Unexpected cache entry: ${dir}`)
-  return root === undefined ? null : join(root, dir, encodeURIComponent(key).replaceAll('%40', '@').replaceAll('%2F', '+'))
+  const name = encodeURIComponent(key.replace(/[!A-Z]/gu, (char) => `!${char.toLowerCase()}`)).replaceAll('%40', '@').replaceAll('%2F', '+')
+  return base === undefined ? null : join(base, dir, name.replace(/^(?=(?:con|prn|aux|nul|com\d|lpt\d)(?:\.|$))./u, (char) => `%${char.codePointAt(0).toString(16).toUpperCase()}`))
+}
+
+// Caches of ours, read whether set or not for what is checked whatever its
+// source: the default one, and stasis's, which it sets with setCacheDir.
+const OURS = [defaultCacheDir, cacheDirFor('stasis')].filter((base) => base !== undefined)
+
+export const ourCachePaths = (dir, key) => OURS.map((base) => cachePath(dir, key, base))
+
+// Only a regular file, checked on the file as opened, so one swapped in
+// after a check can't pass: a FIFO or a device would block or never end.
+// Read to the size it had when opened, and no further.
+export async function readRegularFile(path) {
+  const handle = await open(path, constants.O_RDONLY | (constants.O_NONBLOCK ?? 0)).catch(() => null)
+  try {
+    const stats = await handle?.stat()
+    if (!stats?.isFile() || stats.size > MAX_BYTES) return null
+    const bytes = new Uint8Array(stats.size)
+    for (let at = 0; at < bytes.length;) {
+      const { bytesRead } = await handle.read(bytes, at, bytes.length - at, at)
+      if (bytesRead === 0) return null
+      at += bytesRead
+    }
+    return bytes
+  } catch {
+    return null
+  } finally {
+    await handle?.close()
+  }
 }
 
 export async function readCache(dir, key) {
   const path = cachePath(dir, key)
-  return path && await readFile(path).then((bytes) => new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength), () => null)
+  return path && await readRegularFile(path)
 }
 
 export async function readCacheJSON(dir, key) {
@@ -55,3 +122,30 @@ export async function writeCache(dir, key, data) {
 export async function writeCacheJSON(dir, key, value) {
   return await writeCache(dir, key, JSON.stringify(value))
 }
+
+export async function readRecord(dir, name) {
+  const record = await readCacheJSON(dir, `${name}.json`)
+  const age = typeof record?.at === 'number' ? Date.now() - record.at : Number.NaN
+  const fresh = age >= 0 && age <= RECORD_TTL_MS // An entry from the future is not fresh forever.
+  return fresh && record.name === name ? record : null
+}
+
+export const writeRecord = (dir, name, value) => writeCacheJSON(dir, `${name}.json`, { at: Date.now(), name, ...value })
+
+// Name → the GitHub repo cached for it, for those of `names` that have one.
+export async function readRepos(dir, names) {
+  const repos = new Map()
+  for (const name of names) {
+    const entry = await readRecord(dir, name)
+    if (isRepo(entry?.github)) repos.set(name, entry.github)
+  }
+  return repos
+}
+
+// An answer into `repos`, and into the cache where it found a repo.
+export async function addRepos(dir, repos, answer) {
+  for (const [name, github] of answer) {
+    repos.set(name, github)
+    if (github) await writeRecord(dir, name, { github })
+  }
+}

package/src/cargo/crate.js

@@ -0,0 +1,53 @@
+import assert from 'node:assert/strict'
+import { readdir } from 'node:fs/promises'
+import { homedir } from 'node:os'
+import { join, resolve } from 'node:path'
+
+import { assertCrate, assertSha256, printable, show } from '../args.js'
+import { verifiedDownload } from '../download.js'
+import { CRATES_INDEX, CRATES_STATIC, buildUrl, encodeSegment, request } from '../http.js'
+
+const DIR = 'cargo/crates' // No expiry: crates.io never takes a version twice.
+
+// The sparse index, what cargo reads: a file per crate, by its lowercased
+// name under 1/, 2/, 3/<first letter>/ or <1st-2nd>/<3rd-4th>/, with a
+// JSON line per version, `cksum` the .crate's sha256.
+async function getChecksum(method, name, version) {
+  const lower = name.toLowerCase()
+  const prefix = lower.length < 3 ? [String(lower.length)] : lower.length === 3 ? ['3', lower[0]] : [lower.slice(0, 2), lower.slice(2, 4)]
+  const text = await request(buildUrl(CRATES_INDEX, [...prefix, lower]), { as: 'text' })
+  const parse = (line) => {
+    try {
+      return JSON.parse(line)
+    } catch {
+      assert.fail(`${method}: the index for ${name} has a line that is not JSON: ${printable(line.slice(0, 200))}`)
+    }
+  }
+  const entry = text.split('\n').filter(Boolean).map(parse).find((candidate) => candidate?.vers === version)
+  assert.ok(entry, `${method}: the index has no ${name}@${version}`)
+  assert.ok(entry.name === name, `${method}: the index answered for ${show(entry.name)}, not ${name}`)
+  assertSha256(method, 'the index cksum', entry.cksum)
+  return entry.cksum
+}
+
+// Cargo's own downloads, a directory per registry under
+// $CARGO_HOME/registry/cache, each holding <name>-<version>.crate.
+async function localPaths(name, version) {
+  const root = join(resolve(process.env.CARGO_HOME || join(homedir(), '.cargo')), 'registry', 'cache')
+  return (await readdir(root).catch(() => [])).map((registry) => join(root, registry, `${name}-${version}.crate`))
+}
+
+export async function verifyChecksum(name, version, checksum) {
+  assertCrate('verifyChecksum', name, version)
+  assertSha256('verifyChecksum', 'checksum', checksum)
+  const cksum = await getChecksum('verifyChecksum', name, version)
+  assert.ok(cksum === checksum, `verifyChecksum: ${name}@${version} is ${cksum} on crates.io, not ${checksum}`)
+}
+
+export async function getCrate(name, version, checksum) {
+  assertCrate('getCrate', name, version)
+  if (checksum !== undefined) assertSha256('getCrate', 'checksum', checksum)
+  const expected = checksum ?? await getChecksum('getCrate', name, version)
+  const locate = () => buildUrl(CRATES_STATIC, ['crates', name, `${name}-${encodeSegment(version)}.crate`])
+  return await verifiedDownload({ method: 'getCrate', dir: DIR, what: `${name}@${version}`, ext: 'crate', algorithm: 'sha256', expected, local: await localPaths(name, version), locate })
+}

package/src/cargo/repos.js

@@ -0,0 +1,52 @@
+import assert from 'node:assert/strict'
+import { setTimeout as sleep } from 'node:timers/promises'
+
+import { assertCrateName, assertNames, show } from '../args.js'
+import { addRepos, readRepos } from '../cache.js'
+import { CRATES_API, buildUrl, request } from '../http.js'
+import { chunks } from '../pool.js'
+import { githubRepoOfUrl } from '../remote.js'
+
+const DIR = 'cargo/repos'
+const PER_REQUEST = 100
+const PACE_MS = 1000 // crates.io asks for a request a second at most, and a user agent.
+const USER_AGENT = '@preventive/upstream (https://github.com/PreventiveMeasures/libraries)'
+
+// One request for up to a hundred crates: each name → the GitHub repo its
+// `repository` names, or null where it names none. A crate crates.io
+// leaves out of its answer is one it does not have, null too.
+async function askCrates(method, names) {
+  const url = buildUrl(CRATES_API, ['api', 'v1', 'crates'], { 'ids[]': names, per_page: PER_REQUEST })
+  const answer = await request(url, { as: 'json', headers: { 'User-Agent': USER_AGENT } })
+  assert.ok(Array.isArray(answer?.crates), `${method}: expected a list of crates from crates.io`)
+  const found = new Map(names.map((name) => [name, null]))
+  for (const crate of answer.crates) {
+    assert.ok(names.includes(crate?.id), `${method}: crates.io answered for ${show(crate?.id)}, which was not asked`)
+    found.set(crate.id, githubRepoOfUrl(crate.repository) ?? null)
+  }
+  return found
+}
+
+// Name → its GitHub repo, or null where crates.io has no such crate or it
+// names no GitHub repo, through the cache: only a repo found is cached,
+// for a month. A failed request throws, or with `soft` leaves its names
+// out; `cachedOnly` asks nothing.
+export async function lookUpCrateRepos(method, names, { soft = false, cachedOnly = false } = {}) {
+  const repos = await readRepos(DIR, names)
+  if (cachedOnly) return repos
+  for (const [i, chunk] of chunks(names.filter((name) => !repos.has(name)), PER_REQUEST).entries()) {
+    if (i > 0) await sleep(PACE_MS)
+    const answer = await askCrates(method, chunk).catch((error) => {
+      if (soft) return new Map()
+      throw error
+    })
+    await addRepos(DIR, repos, answer)
+  }
+  return repos
+}
+
+export async function resolveCrateRepos(crateNames, options = {}) {
+  const names = assertNames('resolveCrateRepos', 'crateNames', crateNames, options, assertCrateName)
+  const repos = await lookUpCrateRepos('resolveCrateRepos', names, { soft: true, cachedOnly: options.cachedOnly })
+  return new Map(names.flatMap((name) => (repos.get(name) ? [[name, { github: repos.get(name) }]] : [])))
+}

package/src/download.js

@@ -0,0 +1,42 @@
+import assert from 'node:assert/strict'
+import { createHash } from 'node:crypto'
+
+import { ourCachePaths, readCache, readRegularFile, writeCache } from './cache.js'
+import { request } from './http.js'
+import { gitTreeOfTarball } from './tree.js'
+
+// A hash as each registry writes it: npm's integrity, one sha512 in base64
+// after `sha512-`; crates.io's and Soldeer's checksum, a sha256 in hex; a
+// git tree id for a tarball of that tree.
+const DIGESTS = {
+  sha512: (bytes) => `sha512-${createHash('sha512').update(bytes).digest('base64')}`,
+  sha256: (bytes) => createHash('sha256').update(bytes).digest('hex'),
+  tree: gitTreeOfTarball,
+}
+
+// Bytes whose `algorithm` hash is `expected`, which the registry or the
+// caller gives, never a disk; without it nothing is read. From `local`,
+// other tools' caches, and with `ours` our default cache and stasis's,
+// whether set or not (read, never written; a mismatch is passed over),
+// else the cache set (a mismatch throws), else what `locate` answers,
+// checked and then cached.
+export async function verifiedDownload({ method, dir, what, ext, algorithm, expected, local = [], ours = false, locate, options = {}, list }) {
+  assert.ok(Object.hasOwn(DIGESTS, algorithm) && typeof expected === 'string' && expected !== '', `${method}: nothing to check ${what} against`)
+  const digest = (bytes) => DIGESTS[algorithm](bytes, { expected, list })
+  const key = `${what}.${ext}`
+  for (const path of ours ? [...local, ...ourCachePaths(dir, key)] : local) {
+    const bytes = await readRegularFile(path)
+    if (bytes && await digest(bytes) === expected) return bytes
+  }
+  const check = async (bytes, from) => {
+    const actual = await digest(bytes)
+    assert.ok(actual === expected, `${method}: integrity mismatch for ${what} from ${from}: expected ${expected}, got ${actual}`)
+    return bytes
+  }
+  const cached = await readCache(dir, key)
+  if (cached) return await check(cached, 'the cache')
+  const url = await locate()
+  const bytes = await check(await request(url, { ...options, as: 'bytes' }), url)
+  await writeCache(dir, key, bytes)
+  return bytes
+}