@preventive/upstream 1.0.0-alpha.0 → 1.0.0-alpha.1

package/cargo.d.ts

@@ -0,0 +1,19 @@
+// Hand-written against cargo.js; a change to either belongs with the other.
+//
+// The name is asserted to be a crate name, the version an exact semver
+// version, and a checksum 64 lowercase hex digits, before any request or
+// cache path. The cache is setCacheDir's (npm.js).
+
+export { HttpError } from './npm.js'
+
+// A published version's .crate, whole, in memory. `checksum` is its
+// sha256 as Cargo.lock records it; without one, crates.io's index is read
+// for it on every call, cached crate or not. Either way the bytes are
+// checked against it, whether they were downloaded from static.crates.io
+// (before they are cached) or read from a cache.
+//
+// Cargo's own cache is read first, and never written: each registry's
+// `<name>-<version>.crate` under $CARGO_HOME/registry/cache, else under
+// `~/.cargo`. A file there that does not match is passed over. One in
+// setCacheDir's cache that does not match throws.
+export function getCrate(name: string, version: string, checksum?: string): Promise<Uint8Array>

package/cargo.js

@@ -0,0 +1,2 @@
+export { getCrate } from './src/cargo.js'
+export { HttpError } from './src/http.js'

package/npm.d.ts

@@ -13,8 +13,9 @@
 // The directory is trusted: a package's repo is answered from it as it
 // was written, with no request. Point it only at storage that nothing
 // less trusted than the caller can write, never a cache shared with or
-// restored from lower-trust jobs. Tarballs don't rely on this, since they
-// are checked against the registry's integrity on every call.
+// restored from lower-trust jobs. Tarballs, crates (cargo.js) and zips
+// (soldeer.js) don't rely on this, since they are checked against their
+// integrity on every call.
 export function setCacheDir(dir: string): void
 
 // A failed request: `status` is the HTTP status the registry answered

package/package.json

@@ -1,13 +1,17 @@
 {
   "name": "@preventive/upstream",
-  "version": "1.0.0-alpha.0",
-  "description": "Minimal npm registry and GitHub clients, and a git checkout reader: the calls we use, nothing more",
+  "version": "1.0.0-alpha.1",
+  "description": "Minimal npm, crates.io and Soldeer registry clients, a GitHub client, and a git checkout reader: the calls we use, nothing more",
   "type": "module",
   "exports": {
     "./advisories.js": {
       "types": "./advisories.d.ts",
       "default": "./advisories.js"
     },
+    "./cargo.js": {
+      "types": "./cargo.d.ts",
+      "default": "./cargo.js"
+    },
     "./github.js": {
       "types": "./github.d.ts",
       "default": "./github.js"
@@ -31,11 +35,17 @@
     "./semver.js": {
       "types": "./semver.d.ts",
       "default": "./semver.js"
+    },
+    "./soldeer.js": {
+      "types": "./soldeer.d.ts",
+      "default": "./soldeer.js"
     }
   },
   "files": [
     "advisories.d.ts",
     "advisories.js",
+    "cargo.d.ts",
+    "cargo.js",
     "github.d.ts",
     "github.js",
     "git.d.ts",
@@ -48,6 +58,8 @@
     "package.js",
     "semver.d.ts",
     "semver.js",
+    "soldeer.d.ts",
+    "soldeer.js",
     "src/advisories/common.js",
     "src/advisories/github.js",
     "src/advisories/index.js",
@@ -56,6 +68,8 @@
     "src/advisories/repos.js",
     "src/args.js",
     "src/cache.js",
+    "src/cargo.js",
+    "src/download.js",
     "src/git.js",
     "src/github/client.js",
     "src/github/read.js",
@@ -66,7 +80,8 @@
     "src/package.js",
     "src/pool.js",
     "src/remote.js",
-    "src/semver.js"
+    "src/semver.js",
+    "src/soldeer.js"
   ],
   "engines": {
     "node": ">=24.14.0"

package/semver.d.ts

@@ -9,3 +9,7 @@ export function compareVersions(a: string, b: string): -1 | 0 | 1
 export function valid(version: unknown, options?: { loose?: boolean }): string | null
 // A string that valid() answers unchanged.
 export function isExactVersion(version: unknown): version is string
+// The range as semver normalizes it, or null where it is not one.
+export function validRange(range: unknown, options?: { loose?: boolean; includePrerelease?: boolean }): string | null
+// Whether some version is in both ranges; throws for one that is not a range.
+export function intersects(a: string, b: string, options?: { loose?: boolean; includePrerelease?: boolean }): boolean

package/semver.js

@@ -1 +1 @@
-export { compareVersions, isExactVersion, satisfies, valid } from './src/semver.js'
+export { compareVersions, intersects, isExactVersion, satisfies, valid, validRange } from './src/semver.js'

package/soldeer.d.ts

@@ -0,0 +1,17 @@
+// Hand-written against soldeer.js; a change to either belongs with the
+// other.
+//
+// The name is asserted to be one Soldeer takes, the version a string of
+// letters, digits, `.`, `_`, `+` and `-`, and the checksum 64 lowercase hex
+// digits, before any request or cache path. The cache is setCacheDir's
+// (npm.js).
+
+export { HttpError } from './npm.js'
+
+// A version's zip from Soldeer's registry, whole, in memory. The registry
+// says where the zip is but not what it hashes to, so `checksum` is
+// required: the zip's sha256, as soldeer.lock records it. The bytes are
+// checked against it, whether they were downloaded (before they are
+// cached) or read from setCacheDir's cache, where one that does not match
+// throws. A cached zip that matches is answered with no request at all.
+export function getZip(name: string, version: string, checksum: string): Promise<Uint8Array>

package/soldeer.js

@@ -0,0 +1,2 @@
+export { HttpError } from './src/http.js'
+export { getZip } from './src/soldeer.js'

package/src/advisories/github.js

@@ -2,6 +2,7 @@ import assert from 'node:assert/strict'
 
 import { assertRepo, assertion, isGhsa, isRefName, isStrings } from '../args.js'
 import { isGone } from '../github/client.js'
+import { recover } from '../http.js'
 import { pool } from '../pool.js'
 import { valid } from '../semver.js'
 import { covered, inRange, isText, metrics } from './common.js'
@@ -14,10 +15,7 @@ const isRepoAdvisory = (advisory) => advisory && typeof advisory === 'object' &&
 
 export const assertClient = assertion('a GitHub client from createClient', (value) => typeof value?.listRepoAdvisories === 'function')
 
-const listAdvisories = (github, repo) => github.listRepoAdvisories({ repo }).catch((err) => {
-  if (isGone(err)) return []
-  throw err
-})
+const listAdvisories = (github, repo) => github.listRepoAdvisories({ repo }).catch(recover(isGone, []))
 
 // GitHub's `>= 1.0.0, < 1.2.6` is npm's with the commas dropped.
 // Maintainers write these unreviewed: one without a range, or with one

package/src/advisories/npm.js

@@ -2,6 +2,7 @@ import assert from 'node:assert/strict'
 
 import { assertPackageName, assertPackageVersion, isGhsa, isStrings, show } from '../args.js'
 import { NPM_REGISTRY, buildUrl, request } from '../http.js'
+import { chunks } from '../pool.js'
 import { compareVersions } from '../semver.js'
 import { covered, isText, metrics } from './common.js'
 import { withRepositories } from './github.js'
@@ -34,8 +35,7 @@ function fromRegistry(name, row, asked) {
 async function registryAdvisories(asked) {
   const names = [...asked.keys()]
   const advisories = []
-  for (let i = 0; i < names.length; i += NAMES_PER_REQUEST) {
-    const chunk = names.slice(i, i + NAMES_PER_REQUEST)
+  for (const chunk of chunks(names, NAMES_PER_REQUEST)) {
     const body = Object.fromEntries(chunk.map((name) => [name, asked.get(name)]))
     const answer = await request(BULK_URL, { method: 'POST', body, as: 'json' })
     assert.ok(answer && typeof answer === 'object' && !Array.isArray(answer), 'advisories: expected an object from the registry')

package/src/advisories/osv.js

@@ -1,8 +1,8 @@
 import assert from 'node:assert/strict'
 
-import { assertion, isGhsa, isStrings, matches, show } from '../args.js'
+import { assertCrateName, assertCrateVersion, assertion, isGhsa, isStrings, matches, show } from '../args.js'
 import { OSV_API, buildUrl, request } from '../http.js'
-import { pool } from '../pool.js'
+import { chunks, pool } from '../pool.js'
 import { isExactVersion } from '../semver.js'
 import { inRange, isText, metrics, order } from './common.js'
 import { withRepositories } from './github.js'
@@ -18,8 +18,8 @@ export const CARGO = {
   osv: 'crates.io',
   github: 'rust',
   lookUp: crateRepos,
-  assertName: assertion('a crate name', matches(/^[A-Za-z][\w-]{0,63}$/u)),
-  assertVersion: assertion('a semver version', matches(/^(?=.{5,256}$)(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:0|[1-9]\d*|\d*[A-Za-z-][\dA-Za-z-]*)(?:\.(?:0|[1-9]\d*|\d*[A-Za-z-][\dA-Za-z-]*))*)?(?:\+[\dA-Za-z-]+(?:\.[\dA-Za-z-]+)*)?$/u)),
+  assertName: assertCrateName,
+  assertVersion: assertCrateVersion,
   keep: (id) => id.startsWith('RUSTSEC-'), // What `cargo audit` reads; the GHSA records mirror it.
   advisories: (asked, options) => osvAdvisories(CARGO, asked, options),
 }
@@ -77,8 +77,7 @@ async function osvAdvisories(ecosystem, asked, options) {
   const { osv, github, lookUp, covers, keep = () => true } = ecosystem
   const list = [...asked].flatMap(([name, versions]) => versions.map((version) => ({ name, version })))
   const hits = new Map() // id → name → versions, in `list` order
-  for (let i = 0; i < list.length; i += QUERIES_PER_REQUEST) {
-    const chunk = list.slice(i, i + QUERIES_PER_REQUEST)
+  for (const chunk of chunks(list, QUERIES_PER_REQUEST)) {
     const body = { queries: chunk.map(({ name, version }) => ({ package: { name, ecosystem: osv }, version })) }
     const answer = await request(QUERY_URL, { method: 'POST', body, as: 'json' })
     assert.ok(Array.isArray(answer?.results) && answer.results.length === chunk.length, 'advisories: expected one OSV result per query')

package/src/advisories/repos.js

@@ -2,13 +2,12 @@ import assert from 'node:assert/strict'
 import { setTimeout as sleep } from 'node:timers/promises'
 
 import { isRepo, show } from '../args.js'
-import { readCacheJSON, writeCacheJSON } from '../cache.js'
-import { CRATES_API, HttpError, PACKAGIST_REPO, buildUrl, request } from '../http.js'
+import { readRecord, writeRecord } from '../cache.js'
+import { CRATES_API, PACKAGIST_REPO, buildUrl, isNotFound, recover, request } from '../http.js'
 import { lookUpPackageRepo } from '../npm/repos.js'
-import { pool } from '../pool.js'
+import { chunks, pool } from '../pool.js'
 import { githubRepoOfUrl } from '../remote.js'
 
-const TTL_MS = 30 * 24 * 60 * 60 * 1000
 const CRATES_PER_REQUEST = 100
 const CRATES_PACE_MS = 1000 // crates.io asks for a request a second at most, and a user agent.
 const USER_AGENT = '@preventive/upstream (https://github.com/PreventiveMeasures/libraries)'
@@ -20,14 +19,13 @@ const PACKAGES_AT_ONCE = 8
 async function cachedRepos(dir, names, fetchMissing) {
   const repos = new Map()
   for (const name of names) {
-    const entry = await readCacheJSON(dir, `${name}.json`)
-    const age = typeof entry?.at === 'number' ? Date.now() - entry.at : Number.NaN
-    if (age >= 0 && age <= TTL_MS && entry.name === name && isRepo(entry.github)) repos.set(name, entry.github)
+    const entry = await readRecord(dir, name)
+    if (isRepo(entry?.github)) repos.set(name, entry.github)
   }
   const missing = names.filter((name) => !repos.has(name))
   for (const [name, github] of await fetchMissing(missing)) {
     repos.set(name, github)
-    if (github) await writeCacheJSON(dir, `${name}.json`, { at: Date.now(), name, github })
+    if (github) await writeRecord(dir, name, { github })
   }
   return repos
 }
@@ -35,9 +33,8 @@ async function cachedRepos(dir, names, fetchMissing) {
 // A crate crates.io leaves out of its answer is one it does not have.
 async function fetchCrates(names) {
   const found = new Map(names.map((name) => [name, null]))
-  for (let i = 0; i < names.length; i += CRATES_PER_REQUEST) {
+  for (const [i, chunk] of chunks(names, CRATES_PER_REQUEST).entries()) {
     if (i > 0) await sleep(CRATES_PACE_MS)
-    const chunk = names.slice(i, i + CRATES_PER_REQUEST)
     const url = buildUrl(CRATES_API, ['api', 'v1', 'crates'], { 'ids[]': chunk, per_page: CRATES_PER_REQUEST })
     const answer = await request(url, { as: 'json', headers: { 'User-Agent': USER_AGENT } })
     assert.ok(Array.isArray(answer?.crates), 'advisories: expected a list of crates from crates.io')
@@ -57,12 +54,9 @@ async function fetchEach(names, fetch) {
 // The first version in Packagist's file is its latest, in full.
 async function fetchComposerRepo(name) {
   const [vendor, pkg] = name.split('/')
-  const answer = await request(buildUrl(PACKAGIST_REPO, ['p2', vendor, `${pkg}.json`]), { as: 'json' }).catch((err) => {
-    if (err instanceof HttpError && err.status === 404) return null
-    throw err
-  })
+  const answer = await request(buildUrl(PACKAGIST_REPO, ['p2', vendor, `${pkg}.json`]), { as: 'json' }).catch(recover(isNotFound, null))
   if (answer === null) return null
-  const versions = Object.hasOwn(answer?.packages ?? {}, name) ? answer.packages[name] : undefined
+  const versions = Object.hasOwn(answer.packages ?? {}, name) ? answer.packages[name] : undefined
   assert.ok(Array.isArray(versions), `advisories: Packagist answered without ${name}`)
   return githubRepoOfUrl(versions[0]?.source?.url) ?? null
 }

package/src/args.js

@@ -70,3 +70,7 @@ export const assertUserAgent = assertion('a printable user agent', matches(/^[ -
 export const assertPackageName = assertion('an npm package name', isPackageName)
 export const assertPackageVersion = assertion('an exact semver version', isExactVersion)
 export const assertGhsa = assertion('a GHSA id', isGhsa)
+export const assertCrateName = assertion('a crate name', matches(/^[A-Za-z][\w-]{0,63}$/u))
+export const assertCrateVersion = assertion('a semver version', matches(/^(?=.{5,256}$)(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:0|[1-9]\d*|\d*[A-Za-z-][\dA-Za-z-]*)(?:\.(?:0|[1-9]\d*|\d*[A-Za-z-][\dA-Za-z-]*))*)?(?:\+[\dA-Za-z-]+(?:\.[\dA-Za-z-]+)*)?$/u))
+// As Cargo.lock and soldeer.lock write a checksum.
+export const assertSha256 = assertion('a sha256 in lowercase hex', matches(/^[\da-f]{64}$/u))

package/src/cache.js

@@ -1,11 +1,13 @@
 import assert from 'node:assert/strict'
-import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'
+import { constants } from 'node:fs'
+import { mkdir, open, rename, rm, writeFile } from 'node:fs/promises'
 import { dirname, join, resolve } from 'node:path'
 
 import { assertDirectoryPath } from './args.js'
-import { decode } from './http.js'
+import { MAX_BYTES, decode } from './http.js'
 
-const DIRS = new Set(['npm/repos', 'npm/tarballs', 'cargo/repos', 'composer/repos'])
+const DIRS = new Set(['npm/repos', 'npm/tarballs', 'cargo/repos', 'cargo/crates', 'composer/repos', 'soldeer/zips'])
+const RECORD_TTL_MS = 30 * 24 * 60 * 60 * 1000 // A link only moves on a transfer or rename, and GitHub redirects those.
 
 let root
 let tmpSeq = 0
@@ -18,14 +20,41 @@ export function setCacheDir(dir) {
 // URI-encoded, so a `/` or `..` in a key stays inside one file name, then
 // `@` kept and `/` written `+`, as pnpm names its store: `@babel+core@7.29.7`.
 // encodeURIComponent writes neither `@` nor `+` bare, so no two keys meet.
+// Where case is not told apart (macOS, Windows), JSONStream and jsonstream
+// would still meet, so a capital is written `!` and the letter, and a `!`
+// as `!!`. A Windows device name (`con.json` is the console) has its first
+// letter escaped, which encodeURIComponent never does.
 function cachePath(dir, key) {
   assert.ok(DIRS.has(dir) && key && typeof key === 'string', `Unexpected cache entry: ${dir}`)
-  return root === undefined ? null : join(root, dir, encodeURIComponent(key).replaceAll('%40', '@').replaceAll('%2F', '+'))
+  const name = encodeURIComponent(key.replace(/[!A-Z]/gu, (char) => `!${char.toLowerCase()}`)).replaceAll('%40', '@').replaceAll('%2F', '+')
+  return root === undefined ? null : join(root, dir, name.replace(/^(?=(?:con|prn|aux|nul|com\d|lpt\d)(?:\.|$))./u, (char) => `%${char.codePointAt(0).toString(16).toUpperCase()}`))
+}
+
+// Only a regular file, checked on the file as opened, so one swapped in
+// after a check can't pass: a FIFO or a device would block or never end.
+// Read to the size it had when opened, and no further.
+export async function readRegularFile(path) {
+  const handle = await open(path, constants.O_RDONLY | (constants.O_NONBLOCK ?? 0)).catch(() => null)
+  try {
+    const stats = await handle?.stat()
+    if (!stats?.isFile() || stats.size > MAX_BYTES) return null
+    const bytes = new Uint8Array(stats.size)
+    for (let at = 0; at < bytes.length;) {
+      const { bytesRead } = await handle.read(bytes, at, bytes.length - at, at)
+      if (bytesRead === 0) return null
+      at += bytesRead
+    }
+    return bytes
+  } catch {
+    return null
+  } finally {
+    await handle?.close()
+  }
 }
 
 export async function readCache(dir, key) {
   const path = cachePath(dir, key)
-  return path && await readFile(path).then((bytes) => new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength), () => null)
+  return path && await readRegularFile(path)
 }
 
 export async function readCacheJSON(dir, key) {
@@ -55,3 +84,12 @@ export async function writeCache(dir, key, data) {
 export async function writeCacheJSON(dir, key, value) {
   return await writeCache(dir, key, JSON.stringify(value))
 }
+
+export async function readRecord(dir, name) {
+  const record = await readCacheJSON(dir, `${name}.json`)
+  const age = typeof record?.at === 'number' ? Date.now() - record.at : Number.NaN
+  const fresh = age >= 0 && age <= RECORD_TTL_MS // An entry from the future is not fresh forever.
+  return fresh && record.name === name ? record : null
+}
+
+export const writeRecord = (dir, name, value) => writeCacheJSON(dir, `${name}.json`, { at: Date.now(), name, ...value })

package/src/cargo.js

@@ -0,0 +1,47 @@
+import assert from 'node:assert/strict'
+import { readdir } from 'node:fs/promises'
+import { homedir } from 'node:os'
+import { join, resolve } from 'node:path'
+
+import { assertCrateName, assertCrateVersion, assertSha256, printable, show } from './args.js'
+import { verifiedDownload } from './download.js'
+import { CRATES_INDEX, CRATES_STATIC, buildUrl, encodeSegment, request } from './http.js'
+
+const DIR = 'cargo/crates' // No expiry: crates.io never takes a version twice.
+
+// The sparse index, what cargo reads: a file per crate, by its lowercased
+// name under 1/, 2/, 3/<first letter>/ or <1st-2nd>/<3rd-4th>/, with a
+// JSON line per version, `cksum` the .crate's sha256.
+async function getChecksum(name, version) {
+  const lower = name.toLowerCase()
+  const prefix = lower.length < 3 ? [String(lower.length)] : lower.length === 3 ? ['3', lower[0]] : [lower.slice(0, 2), lower.slice(2, 4)]
+  const text = await request(buildUrl(CRATES_INDEX, [...prefix, lower]), { as: 'text' })
+  const parse = (line) => {
+    try {
+      return JSON.parse(line)
+    } catch {
+      assert.fail(`getCrate: the index for ${name} has a line that is not JSON: ${printable(line.slice(0, 200))}`)
+    }
+  }
+  const entry = text.split('\n').filter(Boolean).map(parse).find((candidate) => candidate?.vers === version)
+  assert.ok(entry, `getCrate: the index has no ${name}@${version}`)
+  assert.ok(entry.name === name, `getCrate: the index answered for ${show(entry.name)}, not ${name}`)
+  assertSha256('getCrate', 'the index cksum', entry.cksum)
+  return entry.cksum
+}
+
+// Cargo's own downloads, a directory per registry under
+// $CARGO_HOME/registry/cache, each holding <name>-<version>.crate.
+async function localPaths(name, version) {
+  const root = join(resolve(process.env.CARGO_HOME || join(homedir(), '.cargo')), 'registry', 'cache')
+  return (await readdir(root).catch(() => [])).map((registry) => join(root, registry, `${name}-${version}.crate`))
+}
+
+export async function getCrate(name, version, checksum) {
+  assertCrateName('getCrate', 'name', name)
+  assertCrateVersion('getCrate', 'version', version)
+  if (checksum !== undefined) assertSha256('getCrate', 'checksum', checksum)
+  const expected = checksum ?? await getChecksum(name, version)
+  const locate = () => buildUrl(CRATES_STATIC, ['crates', name, `${name}-${encodeSegment(version)}.crate`])
+  return await verifiedDownload({ method: 'getCrate', dir: DIR, what: `${name}@${version}`, ext: 'crate', algorithm: 'sha256', expected, local: await localPaths(name, version), locate })
+}

package/src/download.js

@@ -0,0 +1,38 @@
+import assert from 'node:assert/strict'
+import { createHash } from 'node:crypto'
+
+import { readCache, readRegularFile, writeCache } from './cache.js'
+import { request } from './http.js'
+
+// A hash as each registry writes it: npm's integrity, one sha512 in base64
+// after `sha512-`; crates.io's and Soldeer's checksum, a sha256 in hex.
+const DIGESTS = {
+  sha512: (bytes) => `sha512-${createHash('sha512').update(bytes).digest('base64')}`,
+  sha256: (bytes) => createHash('sha256').update(bytes).digest('hex'),
+}
+
+// Bytes whose `algorithm` hash is `expected`, which the registry or the
+// caller gives, never a disk; without it nothing is read. From `local`,
+// other tools' caches (read, never written; a mismatch is passed over),
+// else ours (a mismatch throws), else what `locate` answers, checked and
+// then cached.
+export async function verifiedDownload({ method, dir, what, ext, algorithm, expected, local = [], locate }) {
+  assert.ok(Object.hasOwn(DIGESTS, algorithm) && typeof expected === 'string' && expected !== '', `${method}: nothing to check ${what} against`)
+  const digest = DIGESTS[algorithm]
+  for (const path of local) {
+    const bytes = await readRegularFile(path)
+    if (bytes && digest(bytes) === expected) return bytes
+  }
+  const check = (bytes, from) => {
+    const actual = digest(bytes)
+    assert.ok(actual === expected, `${method}: integrity mismatch for ${what} from ${from}: expected ${expected}, got ${actual}`)
+    return bytes
+  }
+  const key = `${what}.${ext}`
+  const cached = await readCache(dir, key)
+  if (cached) return check(cached, 'the cache')
+  const url = await locate()
+  const bytes = check(await request(url, { as: 'bytes' }), url)
+  await writeCache(dir, key, bytes)
+  return bytes
+}

package/src/github/write.js

@@ -66,19 +66,13 @@ export function parseGraphQLResponse(status, text) {
     throw new HttpError(status, `GitHub GraphQL ${status}: ${printable(text.slice(0, 4096)) || '(empty body)'}`)
   }
   let json
-  try { json = JSON.parse(text) } catch (err) {
-    throw new Error(`GitHub GraphQL ${status}: malformed JSON response (${err.message}): ${printable(text.slice(0, 200))}`, { cause: err })
+  try { json = JSON.parse(text) } catch (cause) {
+    throw new Error(`GitHub GraphQL ${status}: malformed JSON response (${cause.message}): ${printable(text.slice(0, 200))}`, { cause })
   }
   if (json.errors) throw new Error(`GitHub GraphQL: ${printable(JSON.stringify(json.errors))}`)
   return json.data
 }
 
-// A signed commit; the branch must already exist.
-async function createCommitOnBranch(headers, input) {
-  const res = await send(api(['graphql']), { method: 'POST', headers, body: { query: CREATE_COMMIT_MUTATION, variables: { input } }, as: 'json' })
-  return parseGraphQLResponse(res.status, decode(await readBody(res, 1024 * 1024), 'GitHub GraphQL')).createCommitOnBranch.commit
-}
-
 async function forkRepo(headers, options) {
   assertArgs('forkRepo', options, { repo: assertRepo, name: optional(assertRepoName), organization: optional(assertLogin), defaultBranchOnly: optional(assertBoolean) })
   const { repo, name, organization, defaultBranchOnly } = options
@@ -93,13 +87,16 @@ async function createBranch(headers, options) {
   return await call(headers, repoApi(repo, ['git', 'refs']), { method: 'POST', body: { ref: `refs/heads/${branch}`, sha } })
 }
 
+// A signed commit; the branch must already exist.
 async function createCommit(headers, options) {
   assertArgs('createCommit', options, { repo: assertRepo, branch: assertRef, message: null, additions: null, deletions: null, expectedHeadOid: optional(assertSha) })
   const { repo, branch } = options
   const changes = fileChanges(options.additions, options.deletions)
   const message = commitMessage(options.message)
   const expectedHeadOid = options.expectedHeadOid ?? (await getRepoHead(headers, { repo, branch })).oid
-  return await createCommitOnBranch(headers, { branch: { repositoryNameWithOwner: repo, branchName: branch }, message, fileChanges: changes, expectedHeadOid })
+  const input = { branch: { repositoryNameWithOwner: repo, branchName: branch }, message, fileChanges: changes, expectedHeadOid }
+  const res = await send(api(['graphql']), { method: 'POST', headers, body: { query: CREATE_COMMIT_MUTATION, variables: { input } }, as: 'json' })
+  return parseGraphQLResponse(res.status, decode(await readBody(res, 1024 * 1024), 'GitHub GraphQL')).createCommitOnBranch.commit
 }
 
 async function createPR(headers, options) {

package/src/http.js

@@ -11,9 +11,13 @@ export const GITHUB_API = 'https://api.github.com'
 export const OSV_API = 'https://api.osv.dev'
 export const CRATES_API = 'https://crates.io'
 export const PACKAGIST_REPO = 'https://repo.packagist.org'
+export const CRATES_INDEX = 'https://index.crates.io'
+export const CRATES_STATIC = 'https://static.crates.io'
+export const SOLDEER_API = 'https://api.soldeer.xyz'
+export const SOLDEER_REVISIONS = 'https://soldeer-revisions.s3.amazonaws.com'
 export const MAX_BYTES = 512 * 1024 * 1024
 
-const ORIGINS = new Set([NPM_REGISTRY, GITHUB_API, OSV_API, CRATES_API, PACKAGIST_REPO])
+const ORIGINS = new Set([NPM_REGISTRY, GITHUB_API, OSV_API, CRATES_API, PACKAGIST_REPO, CRATES_INDEX, CRATES_STATIC, SOLDEER_API, SOLDEER_REVISIONS])
 const isSegment = matches(/^(?!\.\.?$)(?:[\w.~@-]|%[\dA-F]{2})+$/u)
 const isQueryKey = matches(/^[a-z_]+(?:\[\])?$/u)
 const isQueryValue = (value) => (typeof value === 'string' && value !== '') || (Number.isSafeInteger(value) && value >= 0)
@@ -36,11 +40,17 @@ export class HttpError extends Error {
   }
 }
 
+export const isNotFound = (err) => err instanceof HttpError && err.status === 404
+export const recover = (expected, value) => (err) => {
+  if (!expected(err)) throw err
+  return value
+}
+
 export function decode(bytes, from) {
   try {
     return decoder.decode(bytes)
-  } catch (err) {
-    throw new Error(`Malformed UTF-8 from ${from}`, { cause: err })
+  } catch (cause) {
+    throw new Error(`Malformed UTF-8 from ${from}`, { cause })
   }
 }
 
@@ -106,7 +116,7 @@ export async function request(url, options) {
   if (options.as === 'text') return text
   try {
     return JSON.parse(text)
-  } catch (err) {
-    throw new Error(`Malformed JSON from ${url}: ${printable(text.slice(0, 200))}`, { cause: err })
+  } catch (cause) {
+    throw new Error(`Malformed JSON from ${url}: ${printable(text.slice(0, 200))}`, { cause })
   }
 }

package/src/npm/repos.js

@@ -1,13 +1,12 @@
 import assert from 'node:assert/strict'
 
 import { assertArgs, assertBoolean, assertPackageName, assertRepo, isRepo, optional, show } from '../args.js'
-import { readCacheJSON, writeCacheJSON } from '../cache.js'
-import { HttpError, NPM_REGISTRY, buildUrl, request } from '../http.js'
+import { readRecord, writeRecord } from '../cache.js'
+import { NPM_REGISTRY, buildUrl, isNotFound, recover, request } from '../http.js'
 import { assertRepoDirectory, getRepo, isRepoDirectory } from '../package.js'
 import { pool } from '../pool.js'
 
 const DIR = 'npm/repos'
-const TTL_MS = 30 * 24 * 60 * 60 * 1000 // A link only moves on a transfer or rename, and GitHub redirects those.
 const CONCURRENCY = 8
 
 // `latest`, not the full packument, which is megabytes of version history.
@@ -28,10 +27,8 @@ export async function getGitHub(name) {
 // at the repo root.
 export async function readPackageRepoCache(name) {
   assertPackageName('readPackageRepoCache', 'name', name)
-  const entry = await readCacheJSON(DIR, `${name}.json`)
-  const age = typeof entry?.at === 'number' ? Date.now() - entry.at : Number.NaN
-  const fresh = age >= 0 && age <= TTL_MS // An entry from the future is not fresh forever.
-  if (!fresh || entry.name !== name || !isRepo(entry.github) || !isRepoDirectory(entry.directory)) return null
+  const entry = await readRecord(DIR, name)
+  if (!isRepo(entry?.github) || !isRepoDirectory(entry.directory)) return null
   return { github: entry.github, ...(entry.directory && { directory: entry.directory }) }
 }
 
@@ -41,7 +38,7 @@ export async function writePackageRepoCache(name, github, directory = '') {
   assertPackageName('writePackageRepoCache', 'name', name)
   assertRepo('writePackageRepoCache', 'github', github)
   assertRepoDirectory('writePackageRepoCache', 'directory', directory)
-  return await writeCacheJSON(DIR, `${name}.json`, { at: Date.now(), name, github, directory })
+  return await writeRecord(DIR, name, { github, directory })
 }
 
 export async function resolvePackageRepos(packageNames, options = {}) {
@@ -58,10 +55,7 @@ export async function resolvePackageRepos(packageNames, options = {}) {
 export async function lookUpPackageRepo(name) {
   const stored = await readPackageRepoCache(name)
   if (stored) return stored
-  const { github, directory } = await fetchRepo('lookUpPackageRepo', name).catch((err) => {
-    if (err instanceof HttpError && err.status === 404) return {}
-    throw err
-  })
+  const { github, directory } = await fetchRepo('lookUpPackageRepo', name).catch(recover(isNotFound, {}))
   if (!github) return null
   await writePackageRepoCache(name, github, directory)
   return { github, ...(directory && { directory }) }

package/src/npm/tarball.js

@@ -1,31 +1,23 @@
 import assert from 'node:assert/strict'
 import { Buffer } from 'node:buffer'
-import { createHash } from 'node:crypto'
-import { readFile, stat } from 'node:fs/promises'
 import { homedir } from 'node:os'
 import { join, resolve } from 'node:path'
 
 import { assertArgs, assertPackageName, assertPackageVersion, assertion, matches, show } from '../args.js'
-import { readCache, writeCache } from '../cache.js'
-import { MAX_BYTES, NPM_REGISTRY, buildUrl, request } from '../http.js'
+import { verifiedDownload } from '../download.js'
+import { NPM_REGISTRY, buildUrl, request } from '../http.js'
 
 const DIR = 'npm/tarballs' // No expiry: the registry never takes a version twice.
 // One sha512 and nothing else, as the registry writes it: a sha1, a second
 // hash or an option is refused rather than trusted. 64 bytes leave the
 // last character before `==` two bits, so only A, Q, g or w is canonical.
 const assertIntegrity = assertion('"sha512-" and a base64 sha512', matches(/^sha512-[\dA-Za-z+/]{85}[AQgw]==$/u))
-const sha512 = (bytes) => `sha512-${createHash('sha512').update(bytes).digest('base64')}`
 
 function assertPackage(method, name, version) {
   assertPackageName(method, 'name', name)
   assertPackageVersion(method, 'version', version)
 }
 
-function tarballUrl(name, version) {
-  const segments = name.split('/')
-  return buildUrl(NPM_REGISTRY, [...segments, '-', `${segments.at(-1)}-${version}.tgz`])
-}
-
 // `{ tarball, integrity }` and nothing else, the tarball exactly the
 // registry's own URL for that version. Each field is read once, and only
 // as the object's own, into a copy that is checked and used from then on:
@@ -35,7 +27,8 @@ function checkedDist(method, name, version, dist) {
   assertArgs(method, dist, { tarball: null, integrity: null }, 'dist')
   const [tarball, integrity] = ['tarball', 'integrity'].map((key) => (Object.hasOwn(dist, key) ? dist[key] : undefined))
   assertIntegrity(method, 'dist.integrity', integrity)
-  const expected = tarballUrl(name, version)
+  const segments = name.split('/')
+  const expected = buildUrl(NPM_REGISTRY, [...segments, '-', `${segments.at(-1)}-${version}.tgz`])
   assert.ok(tarball === expected, `${method}: dist.tarball must be ${expected}, got ${show(tarball)}`)
   return { tarball, integrity }
 }
@@ -46,11 +39,6 @@ async function getDist(method, name, version) {
   return checkedDist(method, name, version, { tarball: json.dist?.tarball, integrity: json.dist?.integrity })
 }
 
-function assertBytes(bytes, integrity, what) {
-  const actual = sha512(bytes)
-  assert.ok(actual === integrity, `getTarball: integrity mismatch for ${what}: expected ${integrity}, got ${actual}`)
-}
-
 // Where npm keeps its cache, short of an .npmrc moving it: npm_config_cache,
 // as npm sets it for what it runs, else npm's default. On Windows, npm 4
 // and before defaulted to %APPDATA%, later ones to %LOCALAPPDATA%.
@@ -61,24 +49,12 @@ function npmCacheDirs() {
   return [process.env.LOCALAPPDATA, process.env.APPDATA].map((dir) => join(dir || homedir(), 'npm-cache'))
 }
 
-// Only a regular file: a FIFO or a device would block or never end.
-async function readRegularFile(path) {
-  const stats = await stat(path).catch(() => null)
-  if (!stats?.isFile() || stats.size > MAX_BYTES) return null
-  return await readFile(path).then((bytes) => new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength), () => null)
-}
-
 // npm 5+ (cacache) files a tarball by its sha512, npm 4 and before as
 // <name>/<version>/package.tgz; ~/.audit as <org>:<name>-<version>.tgz.
-async function readLocalCaches(name, version, integrity) {
+function localPaths(name, version, integrity) {
   const hex = Buffer.from(integrity.slice('sha512-'.length), 'base64').toString('hex')
   const npm = npmCacheDirs().flatMap((root) => [join(root, '_cacache/content-v2/sha512', hex.slice(0, 2), hex.slice(2, 4), hex.slice(4)), join(root, name, version, 'package.tgz')])
-  const audit = join(homedir(), '.audit/cache/tgz', `${name.replace(/^@/u, '').replace('/', ':')}-${version}.tgz`)
-  for (const path of [...npm, audit]) {
-    const bytes = await readRegularFile(path)
-    if (bytes && sha512(bytes) === integrity) return bytes
-  }
-  return null
+  return [...npm, join(homedir(), '.audit/cache/tgz', `${name.replace(/^@/u, '').replace('/', ':')}-${version}.tgz`)]
 }
 
 export async function getMeta(name, version) {
@@ -95,21 +71,9 @@ export async function verifyDist(name, version, dist) {
 
 // Without `dist`, the version document is read every time, cache or not:
 // bytes from a cache are checked against the registry's integrity, never
-// against anything a cache itself holds. A mismatch in ours throws rather
-// than fetching over it.
+// against anything a cache itself holds.
 export async function getTarball(name, version, dist) {
   assertPackage('getTarball', name, version)
-  dist = dist === undefined ? await getDist('getTarball', name, version) : checkedDist('getTarball', name, version, dist)
-  const local = await readLocalCaches(name, version, dist.integrity)
-  if (local) return local
-  const key = `${name}@${version}.tgz`
-  const cached = await readCache(DIR, key)
-  if (cached) {
-    assertBytes(cached, dist.integrity, `${name}@${version} from the cache`)
-    return cached
-  }
-  const bytes = await request(dist.tarball, { as: 'bytes' })
-  assertBytes(bytes, dist.integrity, `${name}@${version} from ${dist.tarball}`)
-  await writeCache(DIR, key, bytes)
-  return bytes
+  const { tarball, integrity } = dist === undefined ? await getDist('getTarball', name, version) : checkedDist('getTarball', name, version, dist)
+  return await verifiedDownload({ method: 'getTarball', dir: DIR, what: `${name}@${version}`, ext: 'tgz', algorithm: 'sha512', expected: integrity, local: localPaths(name, version, integrity), locate: () => tarball })
 }

package/src/pool.js

@@ -10,3 +10,5 @@ export async function pool(items, limit, fn) {
   }))
   return results
 }
+
+export const chunks = (items, size) => Array.from({ length: Math.ceil(items.length / size) }, (_, i) => items.slice(i * size, (i + 1) * size))

package/src/semver.js

@@ -22,7 +22,7 @@ function find() {
   for (const prefix of ['../lib', '.']) {
     try {
       const lib = require(resolve(dirname(argv0), prefix, 'node_modules/npm/node_modules/semver'))
-      if (['compare', 'satisfies', 'valid', 'validRange'].every((name) => typeof lib?.[name] === 'function')) return lib
+      if (['compare', 'intersects', 'satisfies', 'valid', 'validRange'].every((name) => typeof lib?.[name] === 'function')) return lib
     } catch {}
   }
   return null
@@ -36,6 +36,7 @@ function semver() {
 
 export const satisfies = (...args) => semver().satisfies(...args)
 export const validRange = (...args) => semver().validRange(...args)
+export const intersects = (...args) => semver().intersects(...args)
 export const compareVersions = (...args) => semver().compare(...args)
 export const valid = (version, ...rest) => (rest.length === 0 && typeof version === 'string' && PLAIN_RELEASE.test(version) ? version : semver().valid(version, ...rest))
 export const isExactVersion = (version) => typeof version === 'string' && valid(version) === version

package/src/soldeer.js

@@ -0,0 +1,33 @@
+import assert from 'node:assert/strict'
+
+import { assertSha256, assertion, matches, show } from './args.js'
+import { verifiedDownload } from './download.js'
+import { SOLDEER_API, SOLDEER_REVISIONS, buildUrl, request } from './http.js'
+
+const DIR = 'soldeer/zips' // No expiry: the registry refuses a version it already has.
+// Soldeer's own rule for a name: 3 to 100 of lowercase letters, digits and
+// `-`, starting with one of them or `@`, ending with a letter or digit.
+const assertName = assertion('a Soldeer package name', matches(/^(?=.{3,100}$)[@\da-z][\da-z-]*[\da-z]$/u))
+// Soldeer takes any version but an empty one. Its registry has semver
+// (`5.7.0-rc.0`, `1.0.2-solc-0.8-simulate`), bare numbers and commit hashes.
+const assertVersion = assertion('letters, digits, `.`, `_`, `+` and `-`', matches(/^(?=.{1,128}$)[\dA-Za-z][\w.+-]*$/u))
+
+// Where the registry keeps the version's zip; it says nothing of what the
+// zip hashes to.
+async function zipUrl(name, version) {
+  const json = await request(buildUrl(SOLDEER_API, ['api', 'v1', 'revision-cli'], { project_name: name, revision: version }), { as: 'json' })
+  const revision = json?.data?.[0]
+  assert.ok(revision?.version === version, `getZip: the registry answered for ${show(revision?.version)}, not ${name}@${version}`)
+  const prefix = `${SOLDEER_REVISIONS}/${name}/`
+  assert.ok(typeof revision.url === 'string' && revision.url.startsWith(prefix), `getZip: the registry keeps ${name}@${version} outside ${prefix}`)
+  return revision.url
+}
+
+// The registry has no hash to check a zip against, so the caller brings
+// one, as soldeer.lock records it.
+export async function getZip(name, version, checksum) {
+  assertName('getZip', 'name', name)
+  assertVersion('getZip', 'version', version)
+  assertSha256('getZip', 'checksum', checksum)
+  return await verifiedDownload({ method: 'getZip', dir: DIR, what: `${name}@${version}`, ext: 'zip', algorithm: 'sha256', expected: checksum, locate: () => zipUrl(name, version) })
+}