@exodus/stasis 1.0.0-beta.3 → 1.0.0-beta.6

package/src/audit-corrections.js

@@ -0,0 +1,72 @@
+import { satisfies, valid } from '@preventive/upstream/semver.js'
+
+// Manual corrections to audit findings: files that must NOT count as evidence
+// that a (potentially vulnerable) package's code is present. An import edge whose
+// target is one of these files is not evidence the package is really used, so a
+// package pulled in only through corrected files is skipped by the audit (see
+// collectPackagesFromFile) -- none of its real code ships, so no advisory applies.
+//
+// Each entry names a package, the audit-irrelevant files (relative to the module
+// dir), and `range` -- the semver range VERIFIED to match the rationale. The
+// correction never applies outside it: a release could change the file, so the
+// range is widened only after re-checking the file in the new versions.
+const CORRECTIONS = [
+  {
+    // ws's browser build is a noop stub -- `module.exports = function () { throw
+    // new Error('ws does not work in the browser...') }` -- with none of the
+    // WebSocket implementation in it. Verified against the ws 8.21.1 tarball
+    // (latest at the time of writing); re-check browser.js before widening.
+    name: 'ws',
+    files: new Set(['browser.js']),
+    range: '<=8.21.1',
+  },
+  {
+    // node-fetch's browser build re-exports the environment's native fetch
+    // (`module.exports = globalObject.fetch`, plus Headers/Request/Response) --
+    // none of the node-fetch implementation, where its advisories live, is in it.
+    // browser.js ships only through 2.7.0 (3.x dropped it); verified 2.6.13/2.7.0.
+    name: 'node-fetch',
+    files: new Set(['browser.js']),
+    range: '<=2.7.0',
+  },
+]
+
+// Is `rel` (a file path relative to the module dir) audit-irrelevant for
+// `name@version`? Unknown or unparsable versions are never corrected (fail
+// closed: the package stays audited).
+function isCorrectedFile(name, version, rel) {
+  for (const { name: pkg, files, range } of CORRECTIONS) {
+    if (pkg !== name || !files.has(rel)) continue
+    if (valid(version) && satisfies(version, range)) return true
+  }
+  return false
+}
+
+// A manifest (top-level or nested) is never code: the resolution graph records
+// resolver/metadata reads of `package.json` as edges (react-native scans sibling
+// packages' manifests for Haste/asset resolution) and consumers record them
+// alongside bundled files, but no package code ships through one. So too for the
+// manifests the other ecosystems' bundles carry beside a dependency's code
+// (--cargo-manifests, --manifests): its Cargo.toml and the checksums cargo vendored
+// it with (a registry's copy, a git checkout's, or one whose source isn't known),
+// its composer.json, a Solidity dependency's foundry.toml and remappings.
+const CARGO_MANIFESTS = new Set(['Cargo.toml', 'Cargo.lock', '.cargo-checksum.json'])
+const MANIFESTS = {
+  npm: new Set(['package.json']),
+  cargo: CARGO_MANIFESTS,
+  'cargo-git': CARGO_MANIFESTS,
+  'cargo-unknown': CARGO_MANIFESTS,
+  composer: new Set(['composer.json', 'composer.lock']),
+  soldeer: new Set(['package.json', 'foundry.toml', 'remappings.txt', 'soldeer.toml']),
+  github: new Set(['package.json', 'foundry.toml', 'remappings.txt', 'soldeer.toml']),
+}
+const isManifest = (rel, ecosystem) => MANIFESTS[ecosystem]?.has(rel.slice(rel.lastIndexOf('/') + 1)) ?? false
+
+// Is `rel` evidence that `name@version`'s REAL code is present, of a dependency of
+// `ecosystem` (npm by default)? This is the one rule every audit surface shares --
+// package presence, the reason column, and the --why chain graph all count a file
+// (or an edge targeting it) only when it passes. Manifests never do; corrected
+// files (npm's) don't within their verified range.
+export function isEvidenceFile(name, version, rel, ecosystem = 'npm') {
+  return !isManifest(rel, ecosystem) && (ecosystem !== 'npm' || !isCorrectedFile(name, version, rel))
+}

package/src/audit.js

@@ -1,77 +1,135 @@
-import { moduleFileKey } from '@exodus/stasis-core/util'
-import { advisories } from './apis/npm/index.js'
+import { dependencyEcosystem, moduleFileKey, sameGithub } from '@exodus/stasis-core/util'
+import { advisories } from '@preventive/upstream/advisories.js'
+import { compareVersions, valid } from '@preventive/upstream/semver.js'
 import { isEvidenceFile } from './audit-corrections.js'
-import semver from './apis/npm/semver.cjs'
 import { parseFile } from './parse.js'
-import { collectWhy } from './why.js'
+import { collectWhy, invertReason } from './why.js'
 
-// Only audit installed dependencies; first-party packages live under non-`node_modules` keys and
-// must not be sent to the public registry (leaks names, adds noise).
+// Where advisories() asks for each ecosystem's advisories: npm's registry; OSV for crates (those
+// vendored from a registry, which OSV takes for crates.io's) and Composer packages; for Soldeer
+// packages and GitHub repos (Foundry's lib/ submodules), the advisories their GitHub repository
+// publishes, a Soldeer package's as Soldeer names it.
+const SOURCES = { npm: ['npm'], cargo: ['OSV'], composer: ['OSV'], soldeer: ['Soldeer', 'GitHub'], github: ['GitHub'] }
+
+// Why a vendored crate that may not be crates.io's is asked nowhere: by its name, crates.io's
+// advisories would be another crate's, and a private git dependency's name must not reach a public
+// database at all.
+const UNREGISTERED_CRATES = {
+  'cargo-git': 'a crate vendored from a git repository, not crates.io',
+  'cargo-unknown': 'a vendored crate with no .cargo-checksum.json, which may be a git checkout, not crates.io\'s',
+}
+
+// A dependency's version as it is audited: a GitHub repo's `.gitmodules` branch `.`, git's for the
+// superproject's own branch, which the bundle does not know and advisories() takes for no branch, is
+// the 0.0.0 stasis versions a repo with no version of its own by. Every advisory range covers both,
+// as it does a branch name.
+const versionOf = (ecosystem, version) => (ecosystem === 'github' && version === '.' ? '0.0.0' : version)
+
+// A package's key across the audit: `name@version` for npm, as collectWhy keys its chains, and
+// prefixed by the ecosystem for the others, whose names may be npm's too.
+const keyOf = (ecosystem, name, version) => `${ecosystem === 'npm' ? '' : `${ecosystem}:`}${name}@${version}`
+
+// Composer's dev versions (`dev-main`, `1.x-dev`), as Composer tells them, a `#ref` dropped: no
+// advisory database lists those, and advisories() takes releases alone.
+const isComposerDev = (version) => /^dev-|-dev$/iu.test(version.replace(/#.*$/su, ''))
+
+// Why `pkg` cannot be audited, or undefined where it can.
+function unaudited({ ecosystem, version }) {
+  if (Object.hasOwn(UNREGISTERED_CRATES, ecosystem)) return UNREGISTERED_CRATES[ecosystem]
+  if (!Object.hasOwn(SOURCES, ecosystem)) return `no advisories are looked up for ${ecosystem}`
+  if (ecosystem === 'composer' && isComposerDev(version)) return 'a Composer dev version, which no advisory database lists'
+  return undefined
+}
+
+// Only audit installed dependencies, `{ ecosystem, name, version }`: an npm package, a vendored
+// crate, a Composer package, a Soldeer package or a GitHub repo, as the bundle tags it (a crate
+// vendored from git, or from where no `.cargo-checksum.json` tells, is listed but never asked about).
 //
 // A package counts as present only when its REAL code is, and recorded = present:
 // an artifact records exactly the files it ships or attested -- imported, entry,
 // or `add`-ed (the last two have no in-edge in the resolution graph, so presence
 // must NOT be derived from edges). The package is audited only if some recorded
 // file is code evidence (see audit-corrections.js) -- not a corrected file (ws's
-// noop browser.js stub), not a package.json manifest (recorded for resolver /
-// metadata reads). So a ws shipped only as browser.js (+ manifest) is skipped,
-// while one whose real code was bundled stays. Which consumers and import EDGES
-// reach that code is the reason column's concern -- collectReasons and why.js
-// apply the same evidence rule per file/edge there.
+// noop browser.js stub), not a manifest (package.json, recorded for resolver /
+// metadata reads; Cargo.toml). So a ws shipped only as browser.js (+ manifest) is
+// skipped, while one whose real code was bundled stays. Which consumers and import
+// EDGES reach that code is the reason column's concern -- collectReasons and
+// why.js apply the same evidence rule per file/edge there.
+//
+// A package's `github` is the repository its artifact records for it (its package.json's), where
+// one does; a GitHub repo is its own, and has none.
 export function collectPackagesFromFile(file) {
   const out = []
-  for (const [dir, { name, version, files }] of parseFile(file).modules) {
-    if (!dir.includes('node_modules')) continue
+  for (const [dir, { name, version, ecosystem: tagged, repo, files }] of parseFile(file).modules) {
+    const ecosystem = dependencyEcosystem(dir, tagged)
+    // First-party code is never sent to a public registry (leaks names, adds noise).
+    if (ecosystem === undefined) continue
     if (!name || !version) continue
-    if (!Object.keys(files).some((rel) => isEvidenceFile(name, version, rel))) continue
-    out.push({ name, version })
+    if (!Object.keys(files).some((rel) => isEvidenceFile(name, version, rel, ecosystem))) continue
+    const github = ecosystem === 'github' ? undefined : repo?.github
+    out.push({ ecosystem, name, version: versionOf(ecosystem, version), ...(github === undefined ? {} : { github }) })
   }
   return out
 }
 
+// Versions in semver's order where both are semver's, else as numbers in text compare.
+const byNumbers = new Intl.Collator('en', { numeric: true }).compare
+const byVersion = (a, b) => (valid(a) && valid(b) ? compareVersions(a, b) : byNumbers(a, b))
+
+// Each package once, with the GitHub repository its artifacts record for its name, which
+// advisories() asks instead of looking one up. It takes one repository a name, so a name recorded
+// with two, by two of its versions (a package that moved) or by two artifacts, has none, and is
+// looked up as before.
 export function collectPackages(files) {
-  const seen = new Set()
-  const out = []
+  const byKey = new Map()
+  const repos = new Map() // `ecosystem:name` -> the one repository recorded for it, or null
   for (const file of files) {
-    for (const { name, version } of collectPackagesFromFile(file)) {
-      const key = `${name}@${version}`
-      if (seen.has(key)) continue
-      seen.add(key)
-      out.push({ name, version })
+    for (const { github, ...pkg } of collectPackagesFromFile(file)) {
+      const key = keyOf(pkg.ecosystem, pkg.name, pkg.version)
+      if (!byKey.has(key)) byKey.set(key, pkg)
+      if (github === undefined) continue
+      const name = `${pkg.ecosystem}:${pkg.name}`
+      const known = repos.get(name)
+      // The first spelling stands.
+      if (known === undefined) repos.set(name, github)
+      else if (known !== null && !sameGithub(known, github)) repos.set(name, null)
     }
   }
-  return out.toSorted((a, b) => a.name.localeCompare(b.name) || semver.compare(a.version, b.version))
+  return [...byKey.values()]
+    .map((pkg) => {
+      const github = repos.get(`${pkg.ecosystem}:${pkg.name}`)
+      return github ? { ...pkg, github } : pkg
+    })
+    .toSorted((a, b) => a.ecosystem.localeCompare(b.ecosystem) || a.name.localeCompare(b.name) || byVersion(a.version, b.version))
 }
 
-// Map each audited node_modules package (`name@version`) to the bundle consumers ("reasons") that
-// recorded its files. Only bundles carry a reason map ({ consumer: [file, ...] }); lockfiles omit
-// it. Each file is resolved back to its owning package via the module file listing. Only evidence
-// files attribute (see audit-corrections.js): a consumer that recorded nothing of a package but a
-// corrected file or its package.json manifest carries none of its real code and is NOT a reason --
-// e.g. webpack shipping only ws's noop browser.js stub, while `run` bundles the real ws.
+// Map each audited package (keyOf it) to the bundle consumers ("reasons") that recorded its files.
+// Only bundles carry a reason map ({ consumer: [file, ...] }); lockfiles omit it. Each file is
+// resolved back to its owning package via the module file listing. Only evidence files attribute
+// (see audit-corrections.js): a consumer that recorded nothing of a package but a corrected file or
+// its manifest carries none of its real code and is NOT a reason -- e.g. webpack shipping only ws's
+// noop browser.js stub, while `run` bundles the real ws.
 export function collectReasons(files) {
   const byPkg = new Map()
   for (const file of files) {
     const artifact = parseFile(file)
-    const reason = artifact.reason
-    if (!reason) continue
+    const fileReasons = invertReason(artifact.reason)
+    if (!fileReasons) continue
     const fileToPkg = new Map()
-    for (const [dir, { name, version, files: modFiles }] of artifact.modules) {
-      if (!dir.includes('node_modules') || !name || !version) continue
+    for (const [dir, { name, version, ecosystem: tagged, files: modFiles }] of artifact.modules) {
+      const ecosystem = dependencyEcosystem(dir, tagged)
+      if (ecosystem === undefined || !name || !version) continue
       for (const rel of Object.keys(modFiles)) {
-        if (!isEvidenceFile(name, version, rel)) continue
-        fileToPkg.set(moduleFileKey(dir, rel), `${name}@${version}`)
+        if (!isEvidenceFile(name, version, rel, ecosystem)) continue
+        fileToPkg.set(moduleFileKey(dir, rel), keyOf(ecosystem, name, versionOf(ecosystem, version)))
       }
     }
-    for (const [consumer, list] of Object.entries(reason)) {
-      if (!Array.isArray(list)) continue
-      for (const f of list) {
-        const key = fileToPkg.get(f)
-        if (key === undefined) continue
-        let set = byPkg.get(key)
-        if (set === undefined) byPkg.set(key, (set = new Set()))
-        set.add(consumer)
-      }
+    for (const [f, consumers] of fileReasons) {
+      const key = fileToPkg.get(f)
+      if (key === undefined) continue
+      let set = byPkg.get(key)
+      if (set === undefined) byPkg.set(key, (set = new Set()))
+      for (const consumer of consumers) set.add(consumer)
     }
   }
   return byPkg
@@ -91,69 +149,57 @@ const byConsumerOrder = (a, b) => consumerRank(a) - consumerRank(b) || a.localeC
 // (which REPLACE the consumer list). Consumers are ordered plugins -> run -> add.
 // `reasonFilter`, when set, narrows the cell to a single consumer: the `--why`
 // chains are already filtered upstream (see collectWhy), so only the consumer
-// list needs pruning here.
-function reasonCell(pkg, affected, reasonsByPkg, whyByPkg, reasonFilter) {
+// list needs pruning here. collectWhy follows npm's import graph alone, so an
+// advisory of another ecosystem keeps its consumer list under `--why` too.
+function reasonCell({ ecosystem, name, versions: affected }, reasonsByPkg, whyByPkg, reasonFilter) {
+  const chains = whyByPkg !== null && ecosystem === 'npm'
   const parts = new Set()
-  const source = whyByPkg ?? reasonsByPkg
+  const source = chains ? whyByPkg : reasonsByPkg
   for (const v of affected) {
-    for (const p of source.get(`${pkg}@${v}`) ?? []) parts.add(p)
+    for (const p of source.get(keyOf(ecosystem, name, v)) ?? []) parts.add(p)
   }
   // --why: group `consumer: path` lines by consumer and order the groups
   // plugins -> run -> add (this also re-unites a consumer's lines when they were
   // split across affected versions); within a group collectWhy's compressed order
   // is preserved. Otherwise: the `, `-joined consumer set in the same order.
-  if (whyByPkg) {
-    const byConsumer = new Map()
-    for (const line of parts) {
+  if (chains) {
+    const byConsumer = Map.groupBy(parts, (line) => {
       const i = line.indexOf(': ')
-      const consumer = i === -1 ? '' : line.slice(0, i)
-      if (!byConsumer.has(consumer)) byConsumer.set(consumer, [])
-      byConsumer.get(consumer).push(line)
-    }
+      return i === -1 ? '' : line.slice(0, i)
+    })
     return [...byConsumer.keys()].toSorted(byConsumerOrder).flatMap((c) => byConsumer.get(c)).join('\n')
   }
   const consumers = reasonFilter ? [...parts].filter((c) => c === reasonFilter) : [...parts]
   return consumers.toSorted(byConsumerOrder).join(', ')
 }
 
-export function flattenAdvisories(result, packages = [], reasonsByPkg = new Map(), whyByPkg = null, reasonFilter = null) {
-  const installed = new Map()
-  for (const { name, version } of packages) {
-    if (!installed.has(name)) installed.set(name, [])
-    installed.get(name).push(version)
-  }
+// `found` is what @preventive/upstream's advisories() answers: one row per advisory range, whose
+// `versions` are the audited versions it covers (never empty -- a range covering none is dropped
+// there, so the table and exit code reflect only real hits).
+export function flattenAdvisories(found, reasonsByPkg = new Map(), whyByPkg = null, reasonFilter = null) {
   const rows = []
-  for (const [pkg, list] of Object.entries(result)) {
-    if (!Array.isArray(list)) continue
-    const installedVersions = installed.get(pkg) ?? []
-    for (const adv of list) {
-      const range = adv.vulnerable_versions ?? ''
-      const affected = range
-        ? installedVersions.filter((v) => semver.satisfies(v, range))
-        : installedVersions
-      // npm sometimes returns advisories whose range matches none of the versions we submitted; drop
-      // them so the table and exit code reflect only real hits.
-      if (installedVersions.length > 0 && affected.length === 0) continue
-      const reason = reasonCell(pkg, affected, reasonsByPkg, whyByPkg, reasonFilter)
-      // --reason keeps only advisories tied to that consumer: once the cell is
-      // narrowed to it, an empty cell means this package isn't related to it.
-      if (reasonFilter && reason === '') continue
-      rows.push({
-        package: pkg,
-        installed: affected.join(', '),
-        vulnerable: range,
-        severity: adv.severity ?? '',
-        title: adv.title ?? '',
-        url: adv.url ?? '',
-        reason,
-      })
-    }
+  for (const adv of found) {
+    const reason = reasonCell(adv, reasonsByPkg, whyByPkg, reasonFilter)
+    // --reason keeps only advisories tied to that consumer: once the cell is
+    // narrowed to it, an empty cell means this package isn't related to it.
+    if (reasonFilter && reason === '') continue
+    rows.push({
+      ecosystem: adv.ecosystem,
+      package: adv.name,
+      installed: adv.versions.join(', '),
+      vulnerable: adv.range ?? '',
+      severity: adv.severity ?? '',
+      title: adv.title ?? '',
+      id: adv.id,
+      reason,
+    })
   }
   rows.sort((a, b) => {
     const sa = SEVERITY_ORDER[a.severity] ?? 99
     const sb = SEVERITY_ORDER[b.severity] ?? 99
     if (sa !== sb) return sa - sb
     if (a.package !== b.package) return a.package < b.package ? -1 : 1
+    if (a.ecosystem !== b.ecosystem) return a.ecosystem < b.ecosystem ? -1 : 1
     return a.title < b.title ? -1 : 1
   })
   return rows
@@ -199,52 +245,75 @@ export function formatTable(rows, columns, { multiline = [] } = {}) {
   ].join('\n')
 }
 
-export async function audit(files, { why = false, whyDeep = false, whyFull = false, reason = null } = {}) {
+// `repoAdvisories` and `github` (a @preventive/upstream/github.js client) go to advisories() as they
+// are; a Soldeer package or a GitHub repo needs the client, their repository being their only source.
+// A package that cannot be audited is `skipped`, with why (`because`), and asked for nothing.
+export async function audit(files, { why = false, whyDeep = false, whyFull = false, reason = null, repoAdvisories = false, github } = {}) {
   // --why-deep and --why-full imply --why. Deep keeps every chain instead of the
   // default pruning of chains whose full suffix is already a chain (see
   // collectWhy/dropSuffixed); full spells chains out with no `...` collapse.
   why = why || whyDeep || whyFull
   const packages = collectPackages(files)
-  if (packages.length === 0) {
-    return { packages, advisories: {}, rows: [], why }
+  const skipped = packages.flatMap((pkg) => {
+    const because = unaudited(pkg)
+    return because === undefined ? [] : [{ ...pkg, because }]
+  })
+  const asked = packages.filter((pkg) => unaudited(pkg) === undefined)
+  if (asked.length === 0) {
+    return { packages, skipped, advisories: [], rows: [], why }
+  }
+  let result
+  try {
+    result = await advisories(asked.map(({ version, ...pkg }) => ({ ...pkg, versions: [version] })), { repoAdvisories, github })
+  } catch (cause) {
+    // Refused input or a malformed answer is an assertion that says so itself; a transport or
+    // HTTP failure gets the context of which request it was.
+    if (cause?.code === 'ERR_ASSERTION') throw cause
+    const sources = new Set([...asked.flatMap((pkg) => SOURCES[pkg.ecosystem]), ...(repoAdvisories ? ['GitHub'] : [])])
+    throw new Error(`${[...sources].join('/')} advisories request failed: ${cause.message}`, { cause })
   }
-  const result = await advisories(packages)
   // `--why` REPLACES the consumer list with per-consumer import paths, so only
-  // one of the two is computed. Restrict the (potentially expensive) path search
-  // to the packages that actually carry an advisory. `reason` (--reason) narrows
-  // both the paths (in collectWhy) and the consumer list (in flattenAdvisories)
-  // to a single consumer, dropping advisories unrelated to it.
+  // one of the two is computed for npm's advisories, the only ones collectWhy has
+  // paths for: another ecosystem's keep their consumers. Restrict the (potentially
+  // expensive) path search to the packages that actually carry an advisory.
+  // `reason` (--reason) narrows both the paths (in collectWhy) and the consumer
+  // list (in flattenAdvisories) to a single consumer, dropping advisories
+  // unrelated to it.
   let rows
   if (why) {
-    const vulnerable = new Set(
-      Object.entries(result).filter(([, v]) => Array.isArray(v) && v.length > 0).map(([name]) => name)
-    )
-    const targetKeys = new Set(
-      packages.filter((p) => vulnerable.has(p.name)).map((p) => `${p.name}@${p.version}`)
-    )
-    rows = flattenAdvisories(result, packages, undefined, collectWhy(files, targetKeys, reason, { deep: whyDeep, full: whyFull }), reason)
+    const npm = result.filter((adv) => adv.ecosystem === 'npm')
+    const targetKeys = new Set(npm.flatMap((adv) => adv.versions.map((v) => keyOf(adv.ecosystem, adv.name, v))))
+    const reasons = npm.length < result.length ? collectReasons(files) : undefined
+    rows = flattenAdvisories(result, reasons, collectWhy(files, targetKeys, reason, { deep: whyDeep, full: whyFull }), reason)
   } else {
-    rows = flattenAdvisories(result, packages, collectReasons(files), null, reason)
+    rows = flattenAdvisories(result, collectReasons(files), null, reason)
   }
-  return { packages, advisories: result, rows, why, whyDeep, whyFull, reason }
+  return { packages, skipped, advisories: result, rows, why, whyDeep, whyFull, reason }
+}
+
+// `10 alerts, 1 critical, 5 high, 3 moderate, 1 low`: the rows, then each severity present, in the
+// rows' own order (most severe first); a row without one is `unrated`, so the parts add up.
+function alertStats(rows) {
+  const bySeverity = Map.groupBy(rows, (r) => r.severity || 'unrated')
+  return [`${rows.length} alert${rows.length === 1 ? '' : 's'}`, ...[...bySeverity].map(([severity, list]) => `${list.length} ${severity}`)].join(', ')
 }
 
-export function printAuditReport({ packages, rows, why = false, reason = null }, { out = process.stdout, err = process.stderr } = {}) {
-  err.write(`Scanned ${packages.length} package${packages.length === 1 ? '' : 's'}\n`)
+export function printAuditReport({ packages, skipped = [], rows, why = false, reason = null }, { out = process.stdout, err = process.stderr } = {}) {
+  const scanned = `Scanned ${packages.length} package${packages.length === 1 ? '' : 's'}`
   if (packages.length === 0) {
-    err.write('No node_modules entries found in the input files\n')
-    return
-  }
-  if (rows.length === 0) {
-    err.write('No advisories found\n')
+    err.write(`${scanned}\nNo dependencies found in the input files\n`)
     return
   }
-  const columns = ['severity', 'package', 'installed', 'vulnerable', 'title', 'url']
+  err.write(`${scanned}: ${alertStats(rows)}\n`)
+  for (const pkg of skipped) err.write(`Not audited: ${pkg.ecosystem} ${pkg.name}@${pkg.version}, ${pkg.because}\n`)
+  if (rows.length === 0) return
+  // The ecosystem column only where one is not npm's, which the others were all once.
+  const columns = ['severity', ...(rows.some((r) => r.ecosystem !== 'npm') ? ['ecosystem'] : []), 'package', 'installed', 'vulnerable', 'title', 'id']
   // Surface the reason column only when some advisory has provenance -- bundle
   // consumers, or (with --why) import paths. Under --reason WITHOUT --why every
   // cell is just the filter value repeated, so drop the column then; --why still
   // earns it (the import chains differ per row).
-  if (rows.some((r) => r.reason) && !(reason && !why)) columns.splice(3, 0, 'reason')
+  if (rows.some((r) => r.reason) && !(reason && !why)) columns.splice(columns.indexOf('installed') + 1, 0, 'reason')
   // Under --why the reason cell is a list of `consumer: path` lines; let it span
   // multiple physical rows instead of collapsing to one.
   out.write(formatTable(rows, columns, { multiline: why ? ['reason'] : [] }) + '\n')

package/src/bundle-from-lockfile.js

@@ -4,7 +4,7 @@ import { resolve } from 'node:path'
 
 import { Bundle } from '@exodus/stasis-core/bundle'
 import { sha512integrity } from '@exodus/stasis-core/state-util'
-import { moduleFileKey, narrowExecutable } from '@exodus/stasis-core/util'
+import { moduleFileKey, moduleInfo, narrowExecutable } from '@exodus/stasis-core/util'
 
 // Reconstruct an in-memory Bundle from a lockfile, reading every attested file from disk and
 // verifying each against the lockfile's SRI digest -- fail closed on mismatch, so this is as
@@ -14,7 +14,7 @@ export function bundleFromLockfile(lockfile, { root }) {
   if (!lockfile.imports) throw new Error('stasis: lockfile has no recorded import graph')
   const formats = lockfile.formats ?? new Map()
   const modules = new Map()
-  for (const [dir, { name, version, ecosystem, files }] of lockfile.modules) {
+  for (const [dir, { files, ...info }] of lockfile.modules) {
     const out = Object.create(null)
     for (const [rel, integrity] of Object.entries(files)) {
       const file = moduleFileKey(dir, rel)
@@ -42,9 +42,7 @@ export function bundleFromLockfile(lockfile, { root }) {
         out[rel] = bytes.toString('utf8')
       }
     }
-    modules.set(dir, ecosystem === undefined
-      ? { name, version, files: out }
-      : { name, version, ecosystem, files: out })
+    modules.set(dir, moduleInfo({ ...info, files: out }))
   }
   return new Bundle({
     config: lockfile.config,

package/src/bundle-reader.js

@@ -0,0 +1,77 @@
+import { createReadStream } from 'node:fs'
+import { pipeline } from 'node:stream/promises'
+import { createBrotliDecompress } from 'node:zlib'
+
+import { Bundle } from '@exodus/stasis-core/bundle'
+import { assert } from '@exodus/stasis-core/util'
+
+import { JsonStreamParser } from './json-stream.js'
+
+// Rejects as soon as `signal` aborts, without waiting for `promise` (onFile gets the signal to stop itself).
+const untilAborted = (promise, signal) => (signal === undefined ? promise : new Promise((resolve, reject) => {
+  const abort = () => reject(signal.reason)
+  if (signal.aborted) abort()
+  else signal.addEventListener('abort', abort, { once: true })
+  Promise.resolve(promise).then(resolve, reject).finally(() => signal.removeEventListener('abort', abort))
+}))
+
+// Left in place of each streamed file; fromJSON's contents-free mode requires one per file.
+const STREAMED = Symbol('streamed')
+
+// Bundle.parse(brotliDecompressSync(source)), without holding the decompressed bytes or the JSON text whole.
+// onFile gets each file in stream order, wherever the bundle puts its files, and before the bundle is
+// validated: what it receives (format included) is provisional until readBundle resolves. See doc/file-formats.md.
+export async function readBundle(source, { onFile, signal } = {}) {
+  assert(onFile === undefined || typeof onFile === 'function', 'readBundle: onFile must be a function')
+  let input = source
+  if (typeof source === 'string' || source instanceof URL) input = createReadStream(source)
+  else if (source instanceof ArrayBuffer) input = [new Uint8Array(source)]
+  else if (ArrayBuffer.isView(source)) input = [new Uint8Array(source.buffer, source.byteOffset, source.byteLength)]
+
+  const formats = new Map() // as far as they have streamed: newer bundles put them before any file
+  const delivered = new Map() // file -> the format onFile was given
+  const pending = []
+  const parser = new JsonStreamParser(onFile && {
+    onString(contents, path) {
+      if (path.length === 2 && path[0] === 'formats') formats.set(path[1] === '' ? '.' : path[1], contents)
+      const file = Bundle.fileKeyAt(path)
+      if (file === undefined) return contents
+      assert(!delivered.has(file), `bundle carries file '${file}' twice`)
+      const format = formats.get(file)
+      delivered.set(file, format)
+      pending.push([file, contents, format])
+      return STREAMED
+    },
+  })
+
+  // Big chunks let zlib's thread pool decompress ahead of the parser; zlib's 16 KiB default runs about 2x slower.
+  const decompress = createBrotliDecompress({ chunkSize: 1024 * 1024, readableHighWaterMark: 4 * 1024 * 1024 })
+  let drained = false
+  await pipeline(input, decompress, async (chunks) => {
+    for await (const chunk of chunks) {
+      parser.write(chunk)
+      for (const [file, contents, format] of pending.splice(0)) {
+        signal?.throwIfAborted() // one chunk can queue many files
+        // eslint-disable-next-line no-await-in-loop -- in stream order, with backpressure on the decompressor
+        await untilAborted(onFile(file, contents, { signal, format }), signal)
+      }
+    }
+    drained = true
+  }, { signal }).catch((error) => {
+    // Once drained, the stream is whole: pipeline reports unread bytes after it as an abort.
+    if (!drained || signal?.aborted) throw error
+  })
+
+  const bundle = Bundle.fromJSON(parser.end(), { contents: !onFile })
+  if (onFile) {
+    // fromJSON required a placeholder per file, so onFile got each one; it must have got nothing else.
+    let files = 0
+    for (const [, info] of bundle.modules) files += Object.keys(info.files).length
+    assert(delivered.size === files, 'bundle carries file contents outside its file list')
+    // A later `formats` key replaces the one onFile was told about.
+    for (const [file, format] of delivered) {
+      if (format !== undefined && bundle.formats.get(file) !== format) assert(false, `bundle file '${file}' changed format after onFile got it`)
+    }
+  }
+  return bundle
+}

package/src/cmd/build.js

@@ -1,4 +1,4 @@
-import { existsSync, mkdirSync, writeFileSync } from 'node:fs'
+import { existsSync } from 'node:fs'
 import { createRequire } from 'node:module'
 import { dirname, extname, join, relative, resolve, sep } from 'node:path'
 import { pathToFileURL } from 'node:url'
@@ -10,6 +10,7 @@ import { StasisEsbuild } from '@exodus/stasis-plugins/esbuild'
 import { HERMES_UNSUPPORTED, createBabelConfigTransform, createHermesTransform } from '../babel.js'
 import { bundleFromLockfile } from '../bundle-from-lockfile.js'
 import { parseFileWithKind } from '../parse.js'
+import { writeFile } from './output.js'
 
 export { bundleFromLockfile }
 
@@ -281,10 +282,7 @@ export async function buildCommand({
     for (const line of formatted) process.stderr.write(line)
   }
 
-  for (const file of result.outputFiles) {
-    mkdirSync(dirname(file.path), { recursive: true })
-    writeFileSync(file.path, file.contents)
-  }
+  for (const file of result.outputFiles) writeFile(file.path, file.contents)
 
   console.warn(`[stasis] Built ${selectedEntry} from ${kind} ${artifact} to ${output}`)
   return { entry: selectedEntry, dir: asFile ? undefined : outAbs, outfile: asFile ? outAbs : undefined }