@preventive/upstream 1.0.0-alpha.0 → 1.0.0-alpha.2

package/cargo.d.ts

@@ -0,0 +1,19 @@
+// Hand-written against cargo.js; a change to either belongs with the other.
+//
+// The name is asserted to be a crate name, the version an exact semver
+// version, and a checksum 64 lowercase hex digits, before any request or
+// cache path. The cache is setCacheDir's (npm.js).
+
+export { HttpError } from './npm.js'
+
+// A published version's .crate, whole, in memory. `checksum` is its
+// sha256 as Cargo.lock records it; without one, crates.io's index is read
+// for it on every call, cached crate or not. Either way the bytes are
+// checked against it, whether they were downloaded from static.crates.io
+// (before they are cached) or read from a cache.
+//
+// Cargo's own cache is read first, and never written: each registry's
+// `<name>-<version>.crate` under $CARGO_HOME/registry/cache, else under
+// `~/.cargo`. A file there that does not match is passed over. One in
+// setCacheDir's cache that does not match throws.
+export function getCrate(name: string, version: string, checksum?: string): Promise<Uint8Array>

package/cargo.js

@@ -0,0 +1,2 @@
+export { getCrate } from './src/cargo.js'
+export { HttpError } from './src/http.js'

package/github.d.ts

@@ -1,16 +1,17 @@
 // Hand-written against github.js; a change to either belongs with the other.
 //
-// Every method checks its arguments hard before any request is built:
-// an options object holding only the keys listed here, a repo that is
+// Every method checks its arguments hard before any request is built: an
+// options object holding only the keys listed here, a repo that is
 // `owner/name` by GitHub's rules, branches and tags that git would take,
-// full commit shas, paths with no empty, `.` or `..` component, titles
-// and headlines on one line. A bad one is a rejection naming the method;
-// nothing is sent. Every URL is built from checked, encoded segments and
-// has to come back out of URL parsing unchanged. Redirects are refused,
-// except the tarball's, so an answer is about the repo asked for. A
-// response is read up to a size limit (64 MiB of JSON, 128 MiB of a file,
-// 512 MiB of a tarball) and within a timeout (30 seconds, 5 minutes for a
-// tarball), and what it says goes into an error message escaped.
+// full commit shas and tree ids, paths with no empty, `.` or `..`
+// component, titles and headlines on one line. A bad one is a rejection
+// naming the method; nothing is sent. Every URL is built from checked,
+// encoded segments and has to come back out of URL parsing unchanged.
+// Redirects are refused, except the tarballs', so an answer is about the
+// repo asked for. A response is read up to a size limit (64 MiB of JSON,
+// 128 MiB of a file, 512 MiB of a tarball) and within a timeout (30
+// seconds, 5 minutes for a tarball), and what it says goes into an error
+// message escaped.
 
 // `owner/name`.
 export type RepoName = string
@@ -54,8 +55,25 @@ export interface Client {
   getRepoFile(options: { repo: RepoName; path: string; ref?: string }): Promise<string>
   // The head of `branch`, or of the default branch without one.
   getRepoHead(options: { repo: RepoName; branch?: string }): Promise<{ branch: string; oid: string }>
-  // The repo's gzipped tarball at the full commit `sha`, whole, in memory.
+  // The repo's gzipped tarball at the full commit `sha`, whole, in memory:
+  // getRepoTreeTarball's for the tree GitHub names for that commit, asked
+  // on every call, cached or not. So its top directory is named for the
+  // tree, and files marked `export-subst` are as committed, not rewritten.
   getRepoTarball(options: { repo: RepoName; sha: string }): Promise<Uint8Array>
+  // The gzipped tarball of a tree, the repo's own or any subdirectory's, by
+  // its id, whole, in memory: GitHub's, its files under one top directory.
+  // Its files are hashed back into git's tree and the id must be `tree`,
+  // whether downloaded or read from setCacheDir's cache (npm.js), where it
+  // is kept by the id alone, for good; a cached copy that does not match
+  // throws. A tarball shows a submodule as an empty directory, and leaves
+  // out a subtree with no file in it: the submodule's commit and the
+  // subtrees come from GitHub's listings of the trees, asked only then, and
+  // the id must still come out `tree`: the bytes are the tree's own, the
+  // same for every repo. So GitHub's tarball is refused for a tree with
+  // files marked `export-ignore`, which it leaves out, and from a repo set
+  // to include Git LFS objects in archives, which replace the pointers git
+  // has; that repo still gets a tree already cached, pointers and all.
+  getRepoTreeTarball(options: { repo: RepoName; tree: string }): Promise<Uint8Array>
   // The repository's published security advisories, as GitHub's
   // repository advisory objects. One page of 100: GitHub pages this list
   // by cursor, so a repository with a full page is refused.

package/npm.d.ts

@@ -13,8 +13,9 @@
 // The directory is trusted: a package's repo is answered from it as it
 // was written, with no request. Point it only at storage that nothing
 // less trusted than the caller can write, never a cache shared with or
-// restored from lower-trust jobs. Tarballs don't rely on this, since they
-// are checked against the registry's integrity on every call.
+// restored from lower-trust jobs. Tarballs, crates (cargo.js), zips
+// (soldeer.js) and tree tarballs (github.js) don't rely on this, since
+// they are checked against their integrity on every call.
 export function setCacheDir(dir: string): void
 
 // A failed request: `status` is the HTTP status the registry answered

package/package.json

@@ -1,13 +1,17 @@
 {
   "name": "@preventive/upstream",
-  "version": "1.0.0-alpha.0",
-  "description": "Minimal npm registry and GitHub clients, and a git checkout reader: the calls we use, nothing more",
+  "version": "1.0.0-alpha.2",
+  "description": "Minimal npm, crates.io and Soldeer registry clients, a GitHub client, and a git checkout reader: the calls we use, nothing more",
   "type": "module",
   "exports": {
     "./advisories.js": {
       "types": "./advisories.d.ts",
       "default": "./advisories.js"
     },
+    "./cargo.js": {
+      "types": "./cargo.d.ts",
+      "default": "./cargo.js"
+    },
     "./github.js": {
       "types": "./github.d.ts",
       "default": "./github.js"
@@ -31,11 +35,17 @@
     "./semver.js": {
       "types": "./semver.d.ts",
       "default": "./semver.js"
+    },
+    "./soldeer.js": {
+      "types": "./soldeer.d.ts",
+      "default": "./soldeer.js"
     }
   },
   "files": [
     "advisories.d.ts",
     "advisories.js",
+    "cargo.d.ts",
+    "cargo.js",
     "github.d.ts",
     "github.js",
     "git.d.ts",
@@ -48,6 +58,8 @@
     "package.js",
     "semver.d.ts",
     "semver.js",
+    "soldeer.d.ts",
+    "soldeer.js",
     "src/advisories/common.js",
     "src/advisories/github.js",
     "src/advisories/index.js",
@@ -56,6 +68,8 @@
     "src/advisories/repos.js",
     "src/args.js",
     "src/cache.js",
+    "src/cargo.js",
+    "src/download.js",
     "src/git.js",
     "src/github/client.js",
     "src/github/read.js",
@@ -66,7 +80,9 @@
     "src/package.js",
     "src/pool.js",
     "src/remote.js",
-    "src/semver.js"
+    "src/semver.js",
+    "src/tree.js",
+    "src/soldeer.js"
   ],
   "engines": {
     "node": ">=24.14.0"

package/semver.d.ts

@@ -4,8 +4,17 @@
 // as it is. Without it, every call throws, except valid() on a plain
 // MAJOR.MINOR.PATCH with no options.
 export function satisfies(version: string, range: string, options?: { includePrerelease?: boolean; loose?: boolean }): boolean
-export function compareVersions(a: string, b: string): -1 | 0 | 1
+export function compareVersions(a: string, b: string, options?: { loose?: boolean }): -1 | 0 | 1
 // The version as semver spells it, or null: `v1.2.3` answers `1.2.3`.
 export function valid(version: unknown, options?: { loose?: boolean }): string | null
 // A string that valid() answers unchanged.
 export function isExactVersion(version: unknown): version is string
+// The range as semver normalizes it, or null where it is not one.
+export function validRange(range: unknown, options?: { loose?: boolean; includePrerelease?: boolean }): string | null
+// Whether some version is in both ranges; throws for one that is not a range.
+export function intersects(a: string, b: string, options?: { loose?: boolean; includePrerelease?: boolean }): boolean
+// The version as semver cleans it, spaces and a leading `=` or `v` dropped,
+// or null: ` =v1.2.3 ` answers `1.2.3`.
+export function clean(version: string, options?: { loose?: boolean } | boolean): string | null
+// The version's major, as semver reads it; throws for one that is not a version.
+export function major(version: string, options?: { loose?: boolean } | boolean): number

package/semver.js

@@ -1 +1 @@
-export { compareVersions, isExactVersion, satisfies, valid } from './src/semver.js'
+export { clean, compareVersions, intersects, isExactVersion, major, satisfies, valid, validRange } from './src/semver.js'

package/soldeer.d.ts

@@ -0,0 +1,17 @@
+// Hand-written against soldeer.js; a change to either belongs with the
+// other.
+//
+// The name is asserted to be one Soldeer takes, the version a string of
+// letters, digits, `.`, `_`, `+` and `-`, and the checksum 64 lowercase hex
+// digits, before any request or cache path. The cache is setCacheDir's
+// (npm.js).
+
+export { HttpError } from './npm.js'
+
+// A version's zip from Soldeer's registry, whole, in memory. The registry
+// says where the zip is but not what it hashes to, so `checksum` is
+// required: the zip's sha256, as soldeer.lock records it. The bytes are
+// checked against it, whether they were downloaded (before they are
+// cached) or read from setCacheDir's cache, where one that does not match
+// throws. A cached zip that matches is answered with no request at all.
+export function getZip(name: string, version: string, checksum: string): Promise<Uint8Array>

package/soldeer.js

@@ -0,0 +1,2 @@
+export { HttpError } from './src/http.js'
+export { getZip } from './src/soldeer.js'

package/src/advisories/github.js

@@ -2,6 +2,7 @@ import assert from 'node:assert/strict'
 
 import { assertRepo, assertion, isGhsa, isRefName, isStrings } from '../args.js'
 import { isGone } from '../github/client.js'
+import { recover } from '../http.js'
 import { pool } from '../pool.js'
 import { valid } from '../semver.js'
 import { covered, inRange, isText, metrics } from './common.js'
@@ -14,10 +15,7 @@ const isRepoAdvisory = (advisory) => advisory && typeof advisory === 'object' &&
 
 export const assertClient = assertion('a GitHub client from createClient', (value) => typeof value?.listRepoAdvisories === 'function')
 
-const listAdvisories = (github, repo) => github.listRepoAdvisories({ repo }).catch((err) => {
-  if (isGone(err)) return []
-  throw err
-})
+const listAdvisories = (github, repo) => github.listRepoAdvisories({ repo }).catch(recover(isGone, []))
 
 // GitHub's `>= 1.0.0, < 1.2.6` is npm's with the commas dropped.
 // Maintainers write these unreviewed: one without a range, or with one

package/src/advisories/npm.js

@@ -2,6 +2,7 @@ import assert from 'node:assert/strict'
 
 import { assertPackageName, assertPackageVersion, isGhsa, isStrings, show } from '../args.js'
 import { NPM_REGISTRY, buildUrl, request } from '../http.js'
+import { chunks } from '../pool.js'
 import { compareVersions } from '../semver.js'
 import { covered, isText, metrics } from './common.js'
 import { withRepositories } from './github.js'
@@ -34,8 +35,7 @@ function fromRegistry(name, row, asked) {
 async function registryAdvisories(asked) {
   const names = [...asked.keys()]
   const advisories = []
-  for (let i = 0; i < names.length; i += NAMES_PER_REQUEST) {
-    const chunk = names.slice(i, i + NAMES_PER_REQUEST)
+  for (const chunk of chunks(names, NAMES_PER_REQUEST)) {
     const body = Object.fromEntries(chunk.map((name) => [name, asked.get(name)]))
     const answer = await request(BULK_URL, { method: 'POST', body, as: 'json' })
     assert.ok(answer && typeof answer === 'object' && !Array.isArray(answer), 'advisories: expected an object from the registry')

package/src/advisories/osv.js

@@ -1,8 +1,8 @@
 import assert from 'node:assert/strict'
 
-import { assertion, isGhsa, isStrings, matches, show } from '../args.js'
+import { assertCrateName, assertCrateVersion, assertion, isGhsa, isStrings, matches, show } from '../args.js'
 import { OSV_API, buildUrl, request } from '../http.js'
-import { pool } from '../pool.js'
+import { chunks, pool } from '../pool.js'
 import { isExactVersion } from '../semver.js'
 import { inRange, isText, metrics, order } from './common.js'
 import { withRepositories } from './github.js'
@@ -18,8 +18,8 @@ export const CARGO = {
   osv: 'crates.io',
   github: 'rust',
   lookUp: crateRepos,
-  assertName: assertion('a crate name', matches(/^[A-Za-z][\w-]{0,63}$/u)),
-  assertVersion: assertion('a semver version', matches(/^(?=.{5,256}$)(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:0|[1-9]\d*|\d*[A-Za-z-][\dA-Za-z-]*)(?:\.(?:0|[1-9]\d*|\d*[A-Za-z-][\dA-Za-z-]*))*)?(?:\+[\dA-Za-z-]+(?:\.[\dA-Za-z-]+)*)?$/u)),
+  assertName: assertCrateName,
+  assertVersion: assertCrateVersion,
   keep: (id) => id.startsWith('RUSTSEC-'), // What `cargo audit` reads; the GHSA records mirror it.
   advisories: (asked, options) => osvAdvisories(CARGO, asked, options),
 }
@@ -77,8 +77,7 @@ async function osvAdvisories(ecosystem, asked, options) {
   const { osv, github, lookUp, covers, keep = () => true } = ecosystem
   const list = [...asked].flatMap(([name, versions]) => versions.map((version) => ({ name, version })))
   const hits = new Map() // id → name → versions, in `list` order
-  for (let i = 0; i < list.length; i += QUERIES_PER_REQUEST) {
-    const chunk = list.slice(i, i + QUERIES_PER_REQUEST)
+  for (const chunk of chunks(list, QUERIES_PER_REQUEST)) {
     const body = { queries: chunk.map(({ name, version }) => ({ package: { name, ecosystem: osv }, version })) }
     const answer = await request(QUERY_URL, { method: 'POST', body, as: 'json' })
     assert.ok(Array.isArray(answer?.results) && answer.results.length === chunk.length, 'advisories: expected one OSV result per query')

package/src/advisories/repos.js

@@ -2,13 +2,12 @@ import assert from 'node:assert/strict'
 import { setTimeout as sleep } from 'node:timers/promises'
 
 import { isRepo, show } from '../args.js'
-import { readCacheJSON, writeCacheJSON } from '../cache.js'
-import { CRATES_API, HttpError, PACKAGIST_REPO, buildUrl, request } from '../http.js'
+import { readRecord, writeRecord } from '../cache.js'
+import { CRATES_API, PACKAGIST_REPO, buildUrl, isNotFound, recover, request } from '../http.js'
 import { lookUpPackageRepo } from '../npm/repos.js'
-import { pool } from '../pool.js'
+import { chunks, pool } from '../pool.js'
 import { githubRepoOfUrl } from '../remote.js'
 
-const TTL_MS = 30 * 24 * 60 * 60 * 1000
 const CRATES_PER_REQUEST = 100
 const CRATES_PACE_MS = 1000 // crates.io asks for a request a second at most, and a user agent.
 const USER_AGENT = '@preventive/upstream (https://github.com/PreventiveMeasures/libraries)'
@@ -20,14 +19,13 @@ const PACKAGES_AT_ONCE = 8
 async function cachedRepos(dir, names, fetchMissing) {
   const repos = new Map()
   for (const name of names) {
-    const entry = await readCacheJSON(dir, `${name}.json`)
-    const age = typeof entry?.at === 'number' ? Date.now() - entry.at : Number.NaN
-    if (age >= 0 && age <= TTL_MS && entry.name === name && isRepo(entry.github)) repos.set(name, entry.github)
+    const entry = await readRecord(dir, name)
+    if (isRepo(entry?.github)) repos.set(name, entry.github)
   }
   const missing = names.filter((name) => !repos.has(name))
   for (const [name, github] of await fetchMissing(missing)) {
     repos.set(name, github)
-    if (github) await writeCacheJSON(dir, `${name}.json`, { at: Date.now(), name, github })
+    if (github) await writeRecord(dir, name, { github })
   }
   return repos
 }
@@ -35,9 +33,8 @@ async function cachedRepos(dir, names, fetchMissing) {
 // A crate crates.io leaves out of its answer is one it does not have.
 async function fetchCrates(names) {
   const found = new Map(names.map((name) => [name, null]))
-  for (let i = 0; i < names.length; i += CRATES_PER_REQUEST) {
+  for (const [i, chunk] of chunks(names, CRATES_PER_REQUEST).entries()) {
     if (i > 0) await sleep(CRATES_PACE_MS)
-    const chunk = names.slice(i, i + CRATES_PER_REQUEST)
     const url = buildUrl(CRATES_API, ['api', 'v1', 'crates'], { 'ids[]': chunk, per_page: CRATES_PER_REQUEST })
     const answer = await request(url, { as: 'json', headers: { 'User-Agent': USER_AGENT } })
     assert.ok(Array.isArray(answer?.crates), 'advisories: expected a list of crates from crates.io')
@@ -57,12 +54,9 @@ async function fetchEach(names, fetch) {
 // The first version in Packagist's file is its latest, in full.
 async function fetchComposerRepo(name) {
   const [vendor, pkg] = name.split('/')
-  const answer = await request(buildUrl(PACKAGIST_REPO, ['p2', vendor, `${pkg}.json`]), { as: 'json' }).catch((err) => {
-    if (err instanceof HttpError && err.status === 404) return null
-    throw err
-  })
+  const answer = await request(buildUrl(PACKAGIST_REPO, ['p2', vendor, `${pkg}.json`]), { as: 'json' }).catch(recover(isNotFound, null))
   if (answer === null) return null
-  const versions = Object.hasOwn(answer?.packages ?? {}, name) ? answer.packages[name] : undefined
+  const versions = Object.hasOwn(answer.packages ?? {}, name) ? answer.packages[name] : undefined
   assert.ok(Array.isArray(versions), `advisories: Packagist answered without ${name}`)
   return githubRepoOfUrl(versions[0]?.source?.url) ?? null
 }

package/src/args.js

@@ -63,6 +63,8 @@ export const assertRepoName = assertion('a repository name', isRepoName)
 export const assertRepo = assertion('"owner/name"', isRepo)
 export const assertRef = assertion('a branch or tag name', isRefName)
 export const assertSha = assertion('a full commit sha', isSha)
+export const isSha1 = matches(/^[\da-f]{40}$/u)
+export const assertTreeId = assertion('a full tree id', isSha1)
 export const assertPath = assertion('a path inside a repository', isRepoPath)
 export const assertToken = assertion('a token', isToken)
 export const assertTokenOrNull = assertion('a token, or null for anonymous access', (value) => value === null || isToken(value))
@@ -70,3 +72,7 @@ export const assertUserAgent = assertion('a printable user agent', matches(/^[ -
 export const assertPackageName = assertion('an npm package name', isPackageName)
 export const assertPackageVersion = assertion('an exact semver version', isExactVersion)
 export const assertGhsa = assertion('a GHSA id', isGhsa)
+export const assertCrateName = assertion('a crate name', matches(/^[A-Za-z][\w-]{0,63}$/u))
+export const assertCrateVersion = assertion('a semver version', matches(/^(?=.{5,256}$)(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:0|[1-9]\d*|\d*[A-Za-z-][\dA-Za-z-]*)(?:\.(?:0|[1-9]\d*|\d*[A-Za-z-][\dA-Za-z-]*))*)?(?:\+[\dA-Za-z-]+(?:\.[\dA-Za-z-]+)*)?$/u))
+// As Cargo.lock and soldeer.lock write a checksum.
+export const assertSha256 = assertion('a sha256 in lowercase hex', matches(/^[\da-f]{64}$/u))

package/src/cache.js

@@ -1,11 +1,13 @@
 import assert from 'node:assert/strict'
-import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'
+import { constants } from 'node:fs'
+import { mkdir, open, rename, rm, writeFile } from 'node:fs/promises'
 import { dirname, join, resolve } from 'node:path'
 
 import { assertDirectoryPath } from './args.js'
-import { decode } from './http.js'
+import { MAX_BYTES, decode } from './http.js'
 
-const DIRS = new Set(['npm/repos', 'npm/tarballs', 'cargo/repos', 'composer/repos'])
+const DIRS = new Set(['npm/repos', 'npm/tarballs', 'cargo/repos', 'cargo/crates', 'composer/repos', 'soldeer/zips', 'github/trees'])
+const RECORD_TTL_MS = 30 * 24 * 60 * 60 * 1000 // A link only moves on a transfer or rename, and GitHub redirects those.
 
 let root
 let tmpSeq = 0
@@ -18,14 +20,41 @@ export function setCacheDir(dir) {
 // URI-encoded, so a `/` or `..` in a key stays inside one file name, then
 // `@` kept and `/` written `+`, as pnpm names its store: `@babel+core@7.29.7`.
 // encodeURIComponent writes neither `@` nor `+` bare, so no two keys meet.
+// Where case is not told apart (macOS, Windows), JSONStream and jsonstream
+// would still meet, so a capital is written `!` and the letter, and a `!`
+// as `!!`. A Windows device name (`con.json` is the console) has its first
+// letter escaped, which encodeURIComponent never does.
 function cachePath(dir, key) {
   assert.ok(DIRS.has(dir) && key && typeof key === 'string', `Unexpected cache entry: ${dir}`)
-  return root === undefined ? null : join(root, dir, encodeURIComponent(key).replaceAll('%40', '@').replaceAll('%2F', '+'))
+  const name = encodeURIComponent(key.replace(/[!A-Z]/gu, (char) => `!${char.toLowerCase()}`)).replaceAll('%40', '@').replaceAll('%2F', '+')
+  return root === undefined ? null : join(root, dir, name.replace(/^(?=(?:con|prn|aux|nul|com\d|lpt\d)(?:\.|$))./u, (char) => `%${char.codePointAt(0).toString(16).toUpperCase()}`))
+}
+
+// Only a regular file, checked on the file as opened, so one swapped in
+// after a check can't pass: a FIFO or a device would block or never end.
+// Read to the size it had when opened, and no further.
+export async function readRegularFile(path) {
+  const handle = await open(path, constants.O_RDONLY | (constants.O_NONBLOCK ?? 0)).catch(() => null)
+  try {
+    const stats = await handle?.stat()
+    if (!stats?.isFile() || stats.size > MAX_BYTES) return null
+    const bytes = new Uint8Array(stats.size)
+    for (let at = 0; at < bytes.length;) {
+      const { bytesRead } = await handle.read(bytes, at, bytes.length - at, at)
+      if (bytesRead === 0) return null
+      at += bytesRead
+    }
+    return bytes
+  } catch {
+    return null
+  } finally {
+    await handle?.close()
+  }
 }
 
 export async function readCache(dir, key) {
   const path = cachePath(dir, key)
-  return path && await readFile(path).then((bytes) => new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength), () => null)
+  return path && await readRegularFile(path)
 }
 
 export async function readCacheJSON(dir, key) {
@@ -55,3 +84,12 @@ export async function writeCache(dir, key, data) {
 export async function writeCacheJSON(dir, key, value) {
   return await writeCache(dir, key, JSON.stringify(value))
 }
+
+export async function readRecord(dir, name) {
+  const record = await readCacheJSON(dir, `${name}.json`)
+  const age = typeof record?.at === 'number' ? Date.now() - record.at : Number.NaN
+  const fresh = age >= 0 && age <= RECORD_TTL_MS // An entry from the future is not fresh forever.
+  return fresh && record.name === name ? record : null
+}
+
+export const writeRecord = (dir, name, value) => writeCacheJSON(dir, `${name}.json`, { at: Date.now(), name, ...value })

package/src/cargo.js

@@ -0,0 +1,47 @@
+import assert from 'node:assert/strict'
+import { readdir } from 'node:fs/promises'
+import { homedir } from 'node:os'
+import { join, resolve } from 'node:path'
+
+import { assertCrateName, assertCrateVersion, assertSha256, printable, show } from './args.js'
+import { verifiedDownload } from './download.js'
+import { CRATES_INDEX, CRATES_STATIC, buildUrl, encodeSegment, request } from './http.js'
+
+const DIR = 'cargo/crates' // No expiry: crates.io never takes a version twice.
+
+// The sparse index, what cargo reads: a file per crate, by its lowercased
+// name under 1/, 2/, 3/<first letter>/ or <1st-2nd>/<3rd-4th>/, with a
+// JSON line per version, `cksum` the .crate's sha256.
+async function getChecksum(name, version) {
+  const lower = name.toLowerCase()
+  const prefix = lower.length < 3 ? [String(lower.length)] : lower.length === 3 ? ['3', lower[0]] : [lower.slice(0, 2), lower.slice(2, 4)]
+  const text = await request(buildUrl(CRATES_INDEX, [...prefix, lower]), { as: 'text' })
+  const parse = (line) => {
+    try {
+      return JSON.parse(line)
+    } catch {
+      assert.fail(`getCrate: the index for ${name} has a line that is not JSON: ${printable(line.slice(0, 200))}`)
+    }
+  }
+  const entry = text.split('\n').filter(Boolean).map(parse).find((candidate) => candidate?.vers === version)
+  assert.ok(entry, `getCrate: the index has no ${name}@${version}`)
+  assert.ok(entry.name === name, `getCrate: the index answered for ${show(entry.name)}, not ${name}`)
+  assertSha256('getCrate', 'the index cksum', entry.cksum)
+  return entry.cksum
+}
+
+// Cargo's own downloads, a directory per registry under
+// $CARGO_HOME/registry/cache, each holding <name>-<version>.crate.
+async function localPaths(name, version) {
+  const root = join(resolve(process.env.CARGO_HOME || join(homedir(), '.cargo')), 'registry', 'cache')
+  return (await readdir(root).catch(() => [])).map((registry) => join(root, registry, `${name}-${version}.crate`))
+}
+
+export async function getCrate(name, version, checksum) {
+  assertCrateName('getCrate', 'name', name)
+  assertCrateVersion('getCrate', 'version', version)
+  if (checksum !== undefined) assertSha256('getCrate', 'checksum', checksum)
+  const expected = checksum ?? await getChecksum(name, version)
+  const locate = () => buildUrl(CRATES_STATIC, ['crates', name, `${name}-${encodeSegment(version)}.crate`])
+  return await verifiedDownload({ method: 'getCrate', dir: DIR, what: `${name}@${version}`, ext: 'crate', algorithm: 'sha256', expected, local: await localPaths(name, version), locate })
+}

package/src/download.js

@@ -0,0 +1,41 @@
+import assert from 'node:assert/strict'
+import { createHash } from 'node:crypto'
+
+import { readCache, readRegularFile, writeCache } from './cache.js'
+import { request } from './http.js'
+import { gitTreeOfTarball } from './tree.js'
+
+// A hash as each registry writes it: npm's integrity, one sha512 in base64
+// after `sha512-`; crates.io's and Soldeer's checksum, a sha256 in hex; a
+// git tree id for a tarball of that tree.
+const DIGESTS = {
+  sha512: (bytes) => `sha512-${createHash('sha512').update(bytes).digest('base64')}`,
+  sha256: (bytes) => createHash('sha256').update(bytes).digest('hex'),
+  tree: gitTreeOfTarball,
+}
+
+// Bytes whose `algorithm` hash is `expected`, which the registry or the
+// caller gives, never a disk; without it nothing is read. From `local`,
+// other tools' caches (read, never written; a mismatch is passed over),
+// else ours (a mismatch throws), else what `locate` answers, checked and
+// then cached.
+export async function verifiedDownload({ method, dir, what, ext, algorithm, expected, local = [], locate, options = {}, list }) {
+  assert.ok(Object.hasOwn(DIGESTS, algorithm) && typeof expected === 'string' && expected !== '', `${method}: nothing to check ${what} against`)
+  const digest = (bytes) => DIGESTS[algorithm](bytes, { expected, list })
+  for (const path of local) {
+    const bytes = await readRegularFile(path)
+    if (bytes && await digest(bytes) === expected) return bytes
+  }
+  const check = async (bytes, from) => {
+    const actual = await digest(bytes)
+    assert.ok(actual === expected, `${method}: integrity mismatch for ${what} from ${from}: expected ${expected}, got ${actual}`)
+    return bytes
+  }
+  const key = `${what}.${ext}`
+  const cached = await readCache(dir, key)
+  if (cached) return await check(cached, 'the cache')
+  const url = await locate()
+  const bytes = await check(await request(url, { ...options, as: 'bytes' }), url)
+  await writeCache(dir, key, bytes)
+  return bytes
+}

package/src/github/read.js

@@ -2,7 +2,8 @@ import assert from 'node:assert/strict'
 import { Buffer } from 'node:buffer'
 import { createHash } from 'node:crypto'
 
-import { assertArgs, assertGhsa, assertLogin, assertNumber, assertPath, assertRef, assertRepo, assertSha, isSha, optional, sameName, show } from '../args.js'
+import { assertArgs, assertGhsa, assertLogin, assertNumber, assertPath, assertRef, assertRepo, assertSha, assertTreeId, isSha, isSha1, optional, sameName, show } from '../args.js'
+import { verifiedDownload } from '../download.js'
 import { decode, encodeSegment } from '../http.js'
 import { api, bindMethods, call, clientHeaders, isGone, repoApi } from './client.js'
 
@@ -66,11 +67,36 @@ async function getRepoFile(headers, options) {
   return decode(bytes, url)
 }
 
-// The one request that follows a redirect, to codeload.github.com. A full
-// sha only, so the bytes are that commit's, not wherever a ref points now.
+// Follows the redirect to codeload.github.com. A tree id names its content,
+// so the bytes are held to it, downloaded or cached, and cached by it alone,
+// for good. What a tarball cannot show, a submodule's commit or a subtree
+// with nothing in it, comes from GitHub's listings of the trees, which the
+// id checks as well: a directory at a time, as a recursive listing of a
+// large tree is cut short.
+async function treeTarball(method, headers, repo, tree) {
+  const listings = new Map()
+  const list = (sha) => {
+    if (!listings.has(sha)) listings.set(sha, call(headers, repoApi(repo, ['git', 'trees', sha])).then((listing) => (Array.isArray(listing?.tree) ? listing.tree : [])))
+    return listings.get(sha)
+  }
+  const locate = () => repoApi(repo, ['tarball', tree])
+  return await verifiedDownload({ method, dir: 'github/trees', what: tree, ext: 'tgz', algorithm: 'tree', expected: tree, locate, options: { headers, redirect: 'follow' }, list })
+}
+
+// A full sha only, so the bytes are that commit's, not wherever a ref points
+// now. Its tree's tarball, not its own, in which `git archive` rewrites the
+// files marked `export-subst`.
 async function getRepoTarball(headers, options) {
   assertArgs('getRepoTarball', options, { repo: assertRepo, sha: assertSha })
-  return await call(headers, repoApi(options.repo, ['tarball', options.sha]), { as: 'bytes', redirect: 'follow' })
+  const { repo, sha } = options
+  const commit = await call(headers, repoApi(repo, ['git', 'commits', sha]))
+  assert.ok(commit?.sha === sha && isSha1(commit.tree?.sha), `getRepoTarball: GitHub names no tree for ${repo}@${sha}`)
+  return await treeTarball('getRepoTarball', headers, repo, commit.tree.sha)
+}
+
+async function getRepoTreeTarball(headers, options) {
+  assertArgs('getRepoTreeTarball', options, { repo: assertRepo, tree: assertTreeId })
+  return await treeTarball('getRepoTreeTarball', headers, options.repo, options.tree)
 }
 
 async function getPullRequest(headers, options) {
@@ -120,5 +146,5 @@ async function listRepoAdvisories(headers, options) {
   return list
 }
 
-export const readMethods = { getCurrentUser, listUserRepos, getRepo, getRepoHead, getRepoFile, getRepoTarball, getPullRequest, getCollaboratorPermission, getAdvisory, listRepoAdvisories }
+export const readMethods = { getCurrentUser, listUserRepos, getRepo, getRepoHead, getRepoFile, getRepoTarball, getRepoTreeTarball, getPullRequest, getCollaboratorPermission, getAdvisory, listRepoAdvisories }
 export const createClient = (options) => bindMethods(clientHeaders('createClient', options, true), readMethods)

package/src/github/write.js

@@ -66,19 +66,13 @@ export function parseGraphQLResponse(status, text) {
     throw new HttpError(status, `GitHub GraphQL ${status}: ${printable(text.slice(0, 4096)) || '(empty body)'}`)
   }
   let json
-  try { json = JSON.parse(text) } catch (err) {
-    throw new Error(`GitHub GraphQL ${status}: malformed JSON response (${err.message}): ${printable(text.slice(0, 200))}`, { cause: err })
+  try { json = JSON.parse(text) } catch (cause) {
+    throw new Error(`GitHub GraphQL ${status}: malformed JSON response (${cause.message}): ${printable(text.slice(0, 200))}`, { cause })
   }
   if (json.errors) throw new Error(`GitHub GraphQL: ${printable(JSON.stringify(json.errors))}`)
   return json.data
 }
 
-// A signed commit; the branch must already exist.
-async function createCommitOnBranch(headers, input) {
-  const res = await send(api(['graphql']), { method: 'POST', headers, body: { query: CREATE_COMMIT_MUTATION, variables: { input } }, as: 'json' })
-  return parseGraphQLResponse(res.status, decode(await readBody(res, 1024 * 1024), 'GitHub GraphQL')).createCommitOnBranch.commit
-}
-
 async function forkRepo(headers, options) {
   assertArgs('forkRepo', options, { repo: assertRepo, name: optional(assertRepoName), organization: optional(assertLogin), defaultBranchOnly: optional(assertBoolean) })
   const { repo, name, organization, defaultBranchOnly } = options
@@ -93,13 +87,16 @@ async function createBranch(headers, options) {
   return await call(headers, repoApi(repo, ['git', 'refs']), { method: 'POST', body: { ref: `refs/heads/${branch}`, sha } })
 }
 
+// A signed commit; the branch must already exist.
 async function createCommit(headers, options) {
   assertArgs('createCommit', options, { repo: assertRepo, branch: assertRef, message: null, additions: null, deletions: null, expectedHeadOid: optional(assertSha) })
   const { repo, branch } = options
   const changes = fileChanges(options.additions, options.deletions)
   const message = commitMessage(options.message)
   const expectedHeadOid = options.expectedHeadOid ?? (await getRepoHead(headers, { repo, branch })).oid
-  return await createCommitOnBranch(headers, { branch: { repositoryNameWithOwner: repo, branchName: branch }, message, fileChanges: changes, expectedHeadOid })
+  const input = { branch: { repositoryNameWithOwner: repo, branchName: branch }, message, fileChanges: changes, expectedHeadOid }
+  const res = await send(api(['graphql']), { method: 'POST', headers, body: { query: CREATE_COMMIT_MUTATION, variables: { input } }, as: 'json' })
+  return parseGraphQLResponse(res.status, decode(await readBody(res, 1024 * 1024), 'GitHub GraphQL')).createCommitOnBranch.commit
 }
 
 async function createPR(headers, options) {

package/src/http.js

@@ -11,9 +11,13 @@ export const GITHUB_API = 'https://api.github.com'
 export const OSV_API = 'https://api.osv.dev'
 export const CRATES_API = 'https://crates.io'
 export const PACKAGIST_REPO = 'https://repo.packagist.org'
+export const CRATES_INDEX = 'https://index.crates.io'
+export const CRATES_STATIC = 'https://static.crates.io'
+export const SOLDEER_API = 'https://api.soldeer.xyz'
+export const SOLDEER_REVISIONS = 'https://soldeer-revisions.s3.amazonaws.com'
 export const MAX_BYTES = 512 * 1024 * 1024
 
-const ORIGINS = new Set([NPM_REGISTRY, GITHUB_API, OSV_API, CRATES_API, PACKAGIST_REPO])
+const ORIGINS = new Set([NPM_REGISTRY, GITHUB_API, OSV_API, CRATES_API, PACKAGIST_REPO, CRATES_INDEX, CRATES_STATIC, SOLDEER_API, SOLDEER_REVISIONS])
 const isSegment = matches(/^(?!\.\.?$)(?:[\w.~@-]|%[\dA-F]{2})+$/u)
 const isQueryKey = matches(/^[a-z_]+(?:\[\])?$/u)
 const isQueryValue = (value) => (typeof value === 'string' && value !== '') || (Number.isSafeInteger(value) && value >= 0)
@@ -36,11 +40,17 @@ export class HttpError extends Error {
   }
 }
 
+export const isNotFound = (err) => err instanceof HttpError && err.status === 404
+export const recover = (expected, value) => (err) => {
+  if (!expected(err)) throw err
+  return value
+}
+
 export function decode(bytes, from) {
   try {
     return decoder.decode(bytes)
-  } catch (err) {
-    throw new Error(`Malformed UTF-8 from ${from}`, { cause: err })
+  } catch (cause) {
+    throw new Error(`Malformed UTF-8 from ${from}`, { cause })
   }
 }
 
@@ -106,7 +116,7 @@ export async function request(url, options) {
   if (options.as === 'text') return text
   try {
     return JSON.parse(text)
-  } catch (err) {
-    throw new Error(`Malformed JSON from ${url}: ${printable(text.slice(0, 200))}`, { cause: err })
+  } catch (cause) {
+    throw new Error(`Malformed JSON from ${url}: ${printable(text.slice(0, 200))}`, { cause })
   }
 }

package/src/npm/repos.js

@@ -1,13 +1,12 @@
 import assert from 'node:assert/strict'
 
 import { assertArgs, assertBoolean, assertPackageName, assertRepo, isRepo, optional, show } from '../args.js'
-import { readCacheJSON, writeCacheJSON } from '../cache.js'
-import { HttpError, NPM_REGISTRY, buildUrl, request } from '../http.js'
+import { readRecord, writeRecord } from '../cache.js'
+import { NPM_REGISTRY, buildUrl, isNotFound, recover, request } from '../http.js'
 import { assertRepoDirectory, getRepo, isRepoDirectory } from '../package.js'
 import { pool } from '../pool.js'
 
 const DIR = 'npm/repos'
-const TTL_MS = 30 * 24 * 60 * 60 * 1000 // A link only moves on a transfer or rename, and GitHub redirects those.
 const CONCURRENCY = 8
 
 // `latest`, not the full packument, which is megabytes of version history.
@@ -28,10 +27,8 @@ export async function getGitHub(name) {
 // at the repo root.
 export async function readPackageRepoCache(name) {
   assertPackageName('readPackageRepoCache', 'name', name)
-  const entry = await readCacheJSON(DIR, `${name}.json`)
-  const age = typeof entry?.at === 'number' ? Date.now() - entry.at : Number.NaN
-  const fresh = age >= 0 && age <= TTL_MS // An entry from the future is not fresh forever.
-  if (!fresh || entry.name !== name || !isRepo(entry.github) || !isRepoDirectory(entry.directory)) return null
+  const entry = await readRecord(DIR, name)
+  if (!isRepo(entry?.github) || !isRepoDirectory(entry.directory)) return null
   return { github: entry.github, ...(entry.directory && { directory: entry.directory }) }
 }
 
@@ -41,7 +38,7 @@ export async function writePackageRepoCache(name, github, directory = '') {
   assertPackageName('writePackageRepoCache', 'name', name)
   assertRepo('writePackageRepoCache', 'github', github)
   assertRepoDirectory('writePackageRepoCache', 'directory', directory)
-  return await writeCacheJSON(DIR, `${name}.json`, { at: Date.now(), name, github, directory })
+  return await writeRecord(DIR, name, { github, directory })
 }
 
 export async function resolvePackageRepos(packageNames, options = {}) {
@@ -58,10 +55,7 @@ export async function resolvePackageRepos(packageNames, options = {}) {
 export async function lookUpPackageRepo(name) {
   const stored = await readPackageRepoCache(name)
   if (stored) return stored
-  const { github, directory } = await fetchRepo('lookUpPackageRepo', name).catch((err) => {
-    if (err instanceof HttpError && err.status === 404) return {}
-    throw err
-  })
+  const { github, directory } = await fetchRepo('lookUpPackageRepo', name).catch(recover(isNotFound, {}))
   if (!github) return null
   await writePackageRepoCache(name, github, directory)
   return { github, ...(directory && { directory }) }

package/src/npm/tarball.js

@@ -1,31 +1,23 @@
 import assert from 'node:assert/strict'
 import { Buffer } from 'node:buffer'
-import { createHash } from 'node:crypto'
-import { readFile, stat } from 'node:fs/promises'
 import { homedir } from 'node:os'
 import { join, resolve } from 'node:path'
 
 import { assertArgs, assertPackageName, assertPackageVersion, assertion, matches, show } from '../args.js'
-import { readCache, writeCache } from '../cache.js'
-import { MAX_BYTES, NPM_REGISTRY, buildUrl, request } from '../http.js'
+import { verifiedDownload } from '../download.js'
+import { NPM_REGISTRY, buildUrl, request } from '../http.js'
 
 const DIR = 'npm/tarballs' // No expiry: the registry never takes a version twice.
 // One sha512 and nothing else, as the registry writes it: a sha1, a second
 // hash or an option is refused rather than trusted. 64 bytes leave the
 // last character before `==` two bits, so only A, Q, g or w is canonical.
 const assertIntegrity = assertion('"sha512-" and a base64 sha512', matches(/^sha512-[\dA-Za-z+/]{85}[AQgw]==$/u))
-const sha512 = (bytes) => `sha512-${createHash('sha512').update(bytes).digest('base64')}`
 
 function assertPackage(method, name, version) {
   assertPackageName(method, 'name', name)
   assertPackageVersion(method, 'version', version)
 }
 
-function tarballUrl(name, version) {
-  const segments = name.split('/')
-  return buildUrl(NPM_REGISTRY, [...segments, '-', `${segments.at(-1)}-${version}.tgz`])
-}
-
 // `{ tarball, integrity }` and nothing else, the tarball exactly the
 // registry's own URL for that version. Each field is read once, and only
 // as the object's own, into a copy that is checked and used from then on:
@@ -35,7 +27,8 @@ function checkedDist(method, name, version, dist) {
   assertArgs(method, dist, { tarball: null, integrity: null }, 'dist')
   const [tarball, integrity] = ['tarball', 'integrity'].map((key) => (Object.hasOwn(dist, key) ? dist[key] : undefined))
   assertIntegrity(method, 'dist.integrity', integrity)
-  const expected = tarballUrl(name, version)
+  const segments = name.split('/')
+  const expected = buildUrl(NPM_REGISTRY, [...segments, '-', `${segments.at(-1)}-${version}.tgz`])
   assert.ok(tarball === expected, `${method}: dist.tarball must be ${expected}, got ${show(tarball)}`)
   return { tarball, integrity }
 }
@@ -46,11 +39,6 @@ async function getDist(method, name, version) {
   return checkedDist(method, name, version, { tarball: json.dist?.tarball, integrity: json.dist?.integrity })
 }
 
-function assertBytes(bytes, integrity, what) {
-  const actual = sha512(bytes)
-  assert.ok(actual === integrity, `getTarball: integrity mismatch for ${what}: expected ${integrity}, got ${actual}`)
-}
-
 // Where npm keeps its cache, short of an .npmrc moving it: npm_config_cache,
 // as npm sets it for what it runs, else npm's default. On Windows, npm 4
 // and before defaulted to %APPDATA%, later ones to %LOCALAPPDATA%.
@@ -61,24 +49,12 @@ function npmCacheDirs() {
   return [process.env.LOCALAPPDATA, process.env.APPDATA].map((dir) => join(dir || homedir(), 'npm-cache'))
 }
 
-// Only a regular file: a FIFO or a device would block or never end.
-async function readRegularFile(path) {
-  const stats = await stat(path).catch(() => null)
-  if (!stats?.isFile() || stats.size > MAX_BYTES) return null
-  return await readFile(path).then((bytes) => new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength), () => null)
-}
-
 // npm 5+ (cacache) files a tarball by its sha512, npm 4 and before as
 // <name>/<version>/package.tgz; ~/.audit as <org>:<name>-<version>.tgz.
-async function readLocalCaches(name, version, integrity) {
+function localPaths(name, version, integrity) {
   const hex = Buffer.from(integrity.slice('sha512-'.length), 'base64').toString('hex')
   const npm = npmCacheDirs().flatMap((root) => [join(root, '_cacache/content-v2/sha512', hex.slice(0, 2), hex.slice(2, 4), hex.slice(4)), join(root, name, version, 'package.tgz')])
-  const audit = join(homedir(), '.audit/cache/tgz', `${name.replace(/^@/u, '').replace('/', ':')}-${version}.tgz`)
-  for (const path of [...npm, audit]) {
-    const bytes = await readRegularFile(path)
-    if (bytes && sha512(bytes) === integrity) return bytes
-  }
-  return null
+  return [...npm, join(homedir(), '.audit/cache/tgz', `${name.replace(/^@/u, '').replace('/', ':')}-${version}.tgz`)]
 }
 
 export async function getMeta(name, version) {
@@ -95,21 +71,9 @@ export async function verifyDist(name, version, dist) {
 
 // Without `dist`, the version document is read every time, cache or not:
 // bytes from a cache are checked against the registry's integrity, never
-// against anything a cache itself holds. A mismatch in ours throws rather
-// than fetching over it.
+// against anything a cache itself holds.
 export async function getTarball(name, version, dist) {
   assertPackage('getTarball', name, version)
-  dist = dist === undefined ? await getDist('getTarball', name, version) : checkedDist('getTarball', name, version, dist)
-  const local = await readLocalCaches(name, version, dist.integrity)
-  if (local) return local
-  const key = `${name}@${version}.tgz`
-  const cached = await readCache(DIR, key)
-  if (cached) {
-    assertBytes(cached, dist.integrity, `${name}@${version} from the cache`)
-    return cached
-  }
-  const bytes = await request(dist.tarball, { as: 'bytes' })
-  assertBytes(bytes, dist.integrity, `${name}@${version} from ${dist.tarball}`)
-  await writeCache(DIR, key, bytes)
-  return bytes
+  const { tarball, integrity } = dist === undefined ? await getDist('getTarball', name, version) : checkedDist('getTarball', name, version, dist)
+  return await verifiedDownload({ method: 'getTarball', dir: DIR, what: `${name}@${version}`, ext: 'tgz', algorithm: 'sha512', expected: integrity, local: localPaths(name, version, integrity), locate: () => tarball })
 }

package/src/pool.js

@@ -10,3 +10,5 @@ export async function pool(items, limit, fn) {
   }))
   return results
 }
+
+export const chunks = (items, size) => Array.from({ length: Math.ceil(items.length / size) }, (_, i) => items.slice(i * size, (i + 1) * size))

package/src/semver.js

@@ -22,7 +22,7 @@ function find() {
   for (const prefix of ['../lib', '.']) {
     try {
       const lib = require(resolve(dirname(argv0), prefix, 'node_modules/npm/node_modules/semver'))
-      if (['compare', 'satisfies', 'valid', 'validRange'].every((name) => typeof lib?.[name] === 'function')) return lib
+      if (['compare', 'intersects', 'satisfies', 'valid', 'validRange'].every((name) => typeof lib?.[name] === 'function')) return lib
     } catch {}
   }
   return null
@@ -36,6 +36,9 @@ function semver() {
 
 export const satisfies = (...args) => semver().satisfies(...args)
 export const validRange = (...args) => semver().validRange(...args)
+export const intersects = (...args) => semver().intersects(...args)
 export const compareVersions = (...args) => semver().compare(...args)
 export const valid = (version, ...rest) => (rest.length === 0 && typeof version === 'string' && PLAIN_RELEASE.test(version) ? version : semver().valid(version, ...rest))
 export const isExactVersion = (version) => typeof version === 'string' && valid(version) === version
+export const clean = (...args) => semver().clean(...args)
+export const major = (...args) => semver().major(...args)

package/src/soldeer.js

@@ -0,0 +1,33 @@
+import assert from 'node:assert/strict'
+
+import { assertSha256, assertion, matches, show } from './args.js'
+import { verifiedDownload } from './download.js'
+import { SOLDEER_API, SOLDEER_REVISIONS, buildUrl, request } from './http.js'
+
+const DIR = 'soldeer/zips' // No expiry: the registry refuses a version it already has.
+// Soldeer's own rule for a name: 3 to 100 of lowercase letters, digits and
+// `-`, starting with one of them or `@`, ending with a letter or digit.
+const assertName = assertion('a Soldeer package name', matches(/^(?=.{3,100}$)[@\da-z][\da-z-]*[\da-z]$/u))
+// Soldeer takes any version but an empty one. Its registry has semver
+// (`5.7.0-rc.0`, `1.0.2-solc-0.8-simulate`), bare numbers and commit hashes.
+const assertVersion = assertion('letters, digits, `.`, `_`, `+` and `-`', matches(/^(?=.{1,128}$)[\dA-Za-z][\w.+-]*$/u))
+
+// Where the registry keeps the version's zip; it says nothing of what the
+// zip hashes to.
+async function zipUrl(name, version) {
+  const json = await request(buildUrl(SOLDEER_API, ['api', 'v1', 'revision-cli'], { project_name: name, revision: version }), { as: 'json' })
+  const revision = json?.data?.[0]
+  assert.ok(revision?.version === version, `getZip: the registry answered for ${show(revision?.version)}, not ${name}@${version}`)
+  const prefix = `${SOLDEER_REVISIONS}/${name}/`
+  assert.ok(typeof revision.url === 'string' && revision.url.startsWith(prefix), `getZip: the registry keeps ${name}@${version} outside ${prefix}`)
+  return revision.url
+}
+
+// The registry has no hash to check a zip against, so the caller brings
+// one, as soldeer.lock records it.
+export async function getZip(name, version, checksum) {
+  assertName('getZip', 'name', name)
+  assertVersion('getZip', 'version', version)
+  assertSha256('getZip', 'checksum', checksum)
+  return await verifiedDownload({ method: 'getZip', dir: DIR, what: `${name}@${version}`, ext: 'zip', algorithm: 'sha256', expected: checksum, locate: () => zipUrl(name, version) })
+}

package/src/tree.js

@@ -0,0 +1,180 @@
+import { Buffer } from 'node:buffer'
+import { createHash } from 'node:crypto'
+import { gunzipSync } from 'node:zlib'
+
+import { isSha1, matches } from './args.js'
+
+const BLOCK = 512
+const MAX_UNPACKED_BYTES = 2 ** 30
+// Only what git writes is read: anything else, a pax `size` or a mode of
+// 0000, would be extracted as it is not hashed. Modes are as its umask,
+// 002 or 022, leaves them: a file 0664 or 0644, 0775 or 0755 executable;
+// a directory 0775 or 0755; a symlink 0777.
+const PAX_KEYS = new Set(['path', 'linkpath'])
+const MODES = { 0: [0o664, 0o644, 0o775, 0o755], 2: [0o777], 5: [0o775, 0o755] }
+// The top directory is not hashed, so it is held to GitHub's name for it,
+// owner-repo-id: another, `..\..`, `C:` or a Windows device such as
+// `con.txt`, could take an extractor elsewhere.
+const isTop = matches(/^(?!(?:con|prn|aux|nul|com\d|lpt\d)(?:\.|$))[\da-z][\w.-]*$/iu)
+
+// Names are kept as latin1 strings, a char per byte, so they sort and hash
+// as the bytes git has.
+const field = (header, start, end) => {
+  const bytes = header.subarray(start, end)
+  const nul = bytes.indexOf(0)
+  return Buffer.from(nul === -1 ? bytes : bytes.subarray(0, nul)).toString('latin1')
+}
+const octal = (header, start, end) => {
+  const text = field(header, start, end).trim()
+  return /^[0-7]{1,12}$/u.test(text) ? Number.parseInt(text, 8) : Number.NaN
+}
+const objectId = (type, content) => createHash('sha1').update(`${type} ${content.length}\0`).update(content).digest()
+const checksum = (header) => header.reduce((sum, byte, i) => sum + (i >= 148 && i < 156 ? 32 : byte), 0)
+const isZero = (bytes) => bytes.every((byte) => byte === 0)
+
+// `<length> <key>=<value>\n`, the length counting the whole record. A NUL
+// would end a path for an extractor, and a name in a tree for git.
+function paxRecords(body) {
+  const text = Buffer.from(body).toString('latin1')
+  const records = new Map()
+  for (let at = 0; at < text.length;) {
+    const space = text.indexOf(' ', at)
+    const length = text.slice(at, space)
+    const end = at + Number(length)
+    if (!/^[1-9]\d*$/u.test(length) || end > text.length || text[end - 1] !== '\n') return null
+    const record = text.slice(space + 1, end - 1)
+    const equals = record.indexOf('=')
+    if (equals < 1 || record.includes('\0')) return null
+    records.set(record.slice(0, equals), record.slice(equals + 1))
+    at = end
+  }
+  return records
+}
+
+// Git keeps no empty directory: in a tarball, one is a submodule. Each as
+// [the directory holding it, its name, its path].
+function emptyDirs(dir, prefix = '') {
+  return [...dir].flatMap(([name, entry]) => {
+    if (!(entry instanceof Map)) return []
+    const path = `${prefix}${name}`
+    return entry.size === 0 ? [[dir, name, path]] : emptyDirs(entry, `${path}/`)
+  })
+}
+
+// Git sorts a subtree as if its name ended in `/`.
+function treeId(dir) {
+  const entries = [...dir].map(([name, entry]) => (entry instanceof Map ? { mode: '40000', name, id: treeId(entry) } : { ...entry, name }))
+  const key = ({ mode, name }) => (mode === '40000' ? `${name}/` : name)
+  entries.sort((a, b) => (key(a) < key(b) ? -1 : Number(key(a) > key(b))))
+  return objectId('tree', Buffer.concat(entries.flatMap(({ mode, name, id }) => [Buffer.from(`${mode} ${name}\0`, 'latin1'), id])))
+}
+
+const EMPTY_TREE = objectId('tree', Buffer.alloc(0)).toString('hex')
+const isName = matches(/^[^\0/]+$/u)
+const subtree = (sha) => ({ mode: '40000', id: Buffer.from(sha, 'hex') })
+
+// A subtree with nothing in it but subtrees like it, down to the empty
+// tree: its listing, read as subtrees alone, hashes to its id.
+async function holdsNothing(sha, listed) {
+  if (sha === EMPTY_TREE) return true
+  const entries = await listed(sha)
+  if (treeId(new Map(entries.map((entry) => [entry.name, subtree(entry.sha)]))).toString('hex') !== sha) return false
+  for (const entry of entries) if (!await holdsNothing(entry.sha, listed)) return false
+  return true
+}
+
+// `git archive` leaves out a subtree with no file in it, which only
+// plumbing makes. Where a directory's id is not the one listed, its listing
+// names any, and one is put back only once its own listings show it holds
+// nothing.
+async function putBackEmptyTrees(dir, sha, listed) {
+  if (treeId(dir).toString('hex') === sha) return
+  for (const entry of await listed(sha)) {
+    if (entry.type !== 'tree') continue
+    const here = dir.get(entry.name)
+    if (here instanceof Map) await putBackEmptyTrees(here, entry.sha, listed)
+    else if (here === undefined && await holdsNothing(entry.sha, listed)) dir.set(entry.name, subtree(entry.sha))
+  }
+}
+
+// The id of the git tree a gzipped tarball holds, as `git archive` writes
+// one, under a single top directory: a file's mode is its exec bit, a
+// symlink's blob its target. What a tarball cannot show comes from `list`,
+// which answers a listing of a tree by its id, as GitHub's trees API does
+// ({ type, path, sha } entries), walked from `expected`: a submodule's
+// commit for an empty directory, and, where the id comes out otherwise, the
+// subtrees with nothing in them it leaves out. The id is `expected` only if
+// those are right. Where there is no such tree, a reason, never an id.
+export async function gitTreeOfTarball(gzipped, { expected, list } = {}) {
+  let bytes
+  try {
+    bytes = gunzipSync(gzipped, { maxOutputLength: MAX_UNPACKED_BYTES })
+  } catch {
+    return 'no tree: not gzip, or larger than 1 GiB unpacked'
+  }
+  const root = new Map()
+  let top
+  let pax = null
+  for (let at = 0; ;) {
+    const header = bytes.subarray(at, at + BLOCK)
+    if (header.length < BLOCK) return 'no tree: the tarball is cut short'
+    if (isZero(header)) {
+      if (!isZero(bytes.subarray(at))) return 'no tree: data after the end of the tarball'
+      break
+    }
+    if (Buffer.from(header.subarray(257, 265)).toString('latin1') !== 'ustar\u000000') return 'no tree: a header that is not POSIX ustar'
+    if (octal(header, 148, 156) !== checksum(header)) return 'no tree: a header that fails its checksum'
+    const size = octal(header, 124, 136)
+    const body = bytes.subarray(at + BLOCK, at + BLOCK + size)
+    if (body.length !== size) return 'no tree: the tarball is cut short'
+    at += BLOCK + Math.ceil(size / BLOCK) * BLOCK
+    const type = String.fromCodePoint(header[156])
+    if (type === 'x') {
+      const records = paxRecords(body)
+      if (records === null) return 'no tree: a malformed pax header'
+      const other = [...records.keys()].find((key) => !PAX_KEYS.has(key))
+      if (other !== undefined) return `no tree: a pax record git does not write, ${JSON.stringify(other)}`
+      if (pax !== null) return 'no tree: two pax headers for one entry'
+      pax = records
+      continue
+    }
+    const prefix = field(header, 345, 500)
+    const path = pax?.get('path') ?? (prefix ? `${prefix}/${field(header, 0, 100)}` : field(header, 0, 100))
+    const target = pax?.get('linkpath') ?? field(header, 157, 257)
+    pax = null
+    if (!Object.hasOwn(MODES, type)) return `no tree: an entry of type ${JSON.stringify(type)}`
+    const mode = octal(header, 100, 108)
+    const asGitWrites = MODES[type].includes(mode) && (type === '0' || size === 0) && path.endsWith('/') === (type === '5')
+      && octal(header, 108, 116) === 0 && octal(header, 116, 124) === 0 && field(header, 265, 297) === 'root' && field(header, 297, 329) === 'root'
+    if (!asGitWrites) return `no tree: a header git does not write, ${JSON.stringify(path)}`
+    const [first, ...parts] = path.replace(/\/$/u, '').split('/')
+    top ??= first
+    const name = type === '5' ? null : parts.pop()
+    if (!isTop(first) || first !== top || name === undefined || [...parts, name].some((part) => ['', '.', '..'].includes(part))) return `no tree: an entry outside one top directory, ${JSON.stringify(path)}`
+    let dir = root
+    for (const part of parts) {
+      if (!dir.has(part)) dir.set(part, new Map())
+      dir = dir.get(part)
+      if (!(dir instanceof Map)) return `no tree: a file where a directory is, ${JSON.stringify(path)}`
+    }
+    if (name === null) continue
+    if (dir.has(name)) return `no tree: ${JSON.stringify(path)} twice`
+    dir.set(name, type === '0' ? { mode: mode & 0o100 ? '100755' : '100644', id: objectId('blob', body) } : { mode: '120000', id: objectId('blob', Buffer.from(target, 'latin1')) })
+  }
+  if (top === undefined) return 'no tree: an empty tarball'
+  const listed = async (sha) => (list && isSha1(sha) ? await list(sha) : [])
+    .filter((entry) => isName(entry?.path) && isSha1(entry.sha))
+    .map((entry) => ({ ...entry, name: Buffer.from(entry.path).toString('latin1') }))
+  for (const [dir, name, path] of emptyDirs(root)) {
+    let sha = expected
+    const parts = path.split('/')
+    for (const [i, part] of parts.entries()) {
+      const type = i === parts.length - 1 ? 'commit' : 'tree'
+      sha = (await listed(sha)).find((entry) => entry.type === type && entry.name === part)?.sha
+    }
+    if (!isSha1(sha)) return `no tree: an empty directory, ${JSON.stringify(path)}, and no submodule there`
+    dir.set(name, { mode: '160000', id: Buffer.from(sha, 'hex') })
+  }
+  await putBackEmptyTrees(root, expected, listed)
+  return treeId(root).toString('hex')
+}