@preventive/upstream 1.0.0-alpha.1 → 1.0.0-alpha.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@preventive/upstream",
-  "version": "1.0.0-alpha.1",
+  "version": "1.0.0-alpha.11",
   "description": "Minimal npm, crates.io and Soldeer registry clients, a GitHub client, and a git checkout reader: the calls we use, nothing more",
   "type": "module",
   "exports": {
@@ -39,9 +39,15 @@
     "./soldeer.js": {
       "types": "./soldeer.d.ts",
       "default": "./soldeer.js"
+    },
+    "./weak-edges.js": {
+      "types": "./weak-edges.d.ts",
+      "default": "./weak-edges.js"
     }
   },
   "files": [
+    "LICENSE-APACHE",
+    "LICENSE-MIT",
     "advisories.d.ts",
     "advisories.js",
     "cargo.d.ts",
@@ -60,6 +66,8 @@
     "semver.js",
     "soldeer.d.ts",
     "soldeer.js",
+    "weak-edges.d.ts",
+    "weak-edges.js",
     "src/advisories/common.js",
     "src/advisories/github.js",
     "src/advisories/index.js",
@@ -67,21 +75,27 @@
     "src/advisories/osv.js",
     "src/advisories/repos.js",
     "src/args.js",
+    "src/attributes.js",
     "src/cache.js",
-    "src/cargo.js",
+    "src/cargo/crate.js",
+    "src/cargo/repos.js",
     "src/download.js",
     "src/git.js",
     "src/github/client.js",
     "src/github/read.js",
     "src/github/write.js",
     "src/http.js",
+    "src/npm/registry.js",
     "src/npm/repos.js",
     "src/npm/tarball.js",
     "src/package.js",
     "src/pool.js",
     "src/remote.js",
+    "src/semver-peer.cjs",
     "src/semver.js",
-    "src/soldeer.js"
+    "src/tree.js",
+    "src/soldeer.js",
+    "src/weak-edges.js"
   ],
   "engines": {
     "node": ">=24.14.0"
@@ -89,7 +103,7 @@
   "scripts": {
     "test": "node --test ./tests/*.test.js"
   },
-  "license": "MIT",
+  "license": "MIT OR Apache-2.0",
   "author": {
     "name": "Nikita Skovoroda",
     "email": "chalkerx@gmail.com",
@@ -106,5 +120,13 @@
   "homepage": "https://github.com/PreventiveMeasures/libraries/tree/main/upstream",
   "publishConfig": {
     "access": "public"
+  },
+  "peerDependencies": {
+    "semver": "^7.5.2"
+  },
+  "peerDependenciesMeta": {
+    "semver": {
+      "optional": true
+    }
   }
 }

package/semver.d.ts

@@ -1,10 +1,11 @@
 // Hand-written against semver.js; a change to either belongs with the other.
 
 // npm's own semver, borrowed from the npm beside node and passed through
-// as it is. Without it, every call throws, except valid() on a plain
+// as it is; where there is no npm, the `semver` package, an optional peer
+// dependency. Without either, every call throws, except valid() on a plain
 // MAJOR.MINOR.PATCH with no options.
 export function satisfies(version: string, range: string, options?: { includePrerelease?: boolean; loose?: boolean }): boolean
-export function compareVersions(a: string, b: string): -1 | 0 | 1
+export function compareVersions(a: string, b: string, options?: { loose?: boolean }): -1 | 0 | 1
 // The version as semver spells it, or null: `v1.2.3` answers `1.2.3`.
 export function valid(version: unknown, options?: { loose?: boolean }): string | null
 // A string that valid() answers unchanged.
@@ -13,3 +14,8 @@ export function isExactVersion(version: unknown): version is string
 export function validRange(range: unknown, options?: { loose?: boolean; includePrerelease?: boolean }): string | null
 // Whether some version is in both ranges; throws for one that is not a range.
 export function intersects(a: string, b: string, options?: { loose?: boolean; includePrerelease?: boolean }): boolean
+// The version as semver cleans it, spaces and a leading `=` or `v` dropped,
+// or null: ` =v1.2.3 ` answers `1.2.3`.
+export function clean(version: string, options?: { loose?: boolean } | boolean): string | null
+// The version's major, as semver reads it; throws for one that is not a version.
+export function major(version: string, options?: { loose?: boolean } | boolean): number

package/semver.js

@@ -1 +1 @@
-export { compareVersions, intersects, isExactVersion, satisfies, valid, validRange } from './src/semver.js'
+export { clean, compareVersions, intersects, isExactVersion, major, satisfies, valid, validRange } from './src/semver.js'

package/src/advisories/common.js

@@ -1,21 +1,43 @@
-import { isStrings, matches } from '../args.js'
-import { satisfies, validRange } from '../semver.js'
+import assert from 'node:assert/strict'
+
+import { isGhsa, isRepo, isStrings, matches, show } from '../args.js'
+import { satisfies, valid, validRange } from '../semver.js'
 
 export const order = (a, b) => (a > b) - (a < b)
 
+// GitHub's page for a GHSA: the advisory database's, which has every one
+// the registry and OSV answer with, or the page of the `repo` that
+// published it, which has it before GitHub reviews it into the database.
+export function advisoryUrl(ghsa, repo) {
+  assert.ok(isGhsa(ghsa) && (repo === undefined || isRepo(repo)), `advisories: no GitHub advisory page for ${show(ghsa)} on ${show(repo)}`)
+  return repo === undefined ? `https://github.com/advisories/${ghsa}` : `https://github.com/${repo}/security/advisories/${ghsa}`
+}
+
 // As `npm audit` matches: a prerelease is in a range around it.
 const SEMVER = { includePrerelease: true, loose: true }
 
 export const inRange = (version, range) => satisfies(version, range, SEMVER)
 
-// The asked versions a range `covers`. One semver cannot read covers them
-// all: a missed advisory is worse than a spare one.
-export const covered = (asked, range, covers = inRange) => (validRange(range, SEMVER) === null ? [...asked] : asked.filter((version) => covers(version, range)))
+// A version semver cannot read may be in any range. Whatever it reads is
+// a version, `v1.2.3` and `1.2.3+build` included.
+export const mayBeInRange = (version, range) => valid(version) === null || inRange(version, range)
+
+// The asked versions a range `covers`. One semver cannot read, range or
+// version, covers them all: a missed advisory is worse than a spare one.
+export const covered = (asked, range, covers = mayBeInRange) => (validRange(range, SEMVER) === null ? [...asked] : asked.filter((version) => covers(version, range)))
 
 // Remote text, as long as it is well-formed; anything else is refused as
 // malformed by whoever reads it.
 export const isText = (value) => typeof value === 'string' && value.isWellFormed()
 
+// An advisory's full text, for `details`: none where its source has none,
+// and refused where it is not well-formed.
+export function detailsOf(text, id) {
+  if (text == null || text === '') return {}
+  assert.ok(isText(text), `advisories: malformed details in ${id}`)
+  return { details: text }
+}
+
 const SEVERITIES = new Set(['critical', 'high', 'moderate', 'low', 'info'])
 const isCvssVector = matches(/^CVSS:[34]\.\d(?:\/[A-Z]{1,4}:[A-Z]{1,2})+$/u)
 const isCwe = matches(/^CWE-\d{1,6}$/u)

package/src/advisories/github.js

@@ -1,35 +1,90 @@
 import assert from 'node:assert/strict'
 
-import { assertRepo, assertion, isGhsa, isRefName, isStrings } from '../args.js'
+import { assertRepo, assertSoldeerName, assertSoldeerVersion, assertion, isGhsa, isRefName, isStrings } from '../args.js'
+import { readRecord, writeRecord } from '../cache.js'
 import { isGone } from '../github/client.js'
 import { recover } from '../http.js'
 import { pool } from '../pool.js'
-import { valid } from '../semver.js'
-import { covered, inRange, isText, metrics } from './common.js'
+import { advisoryUrl, covered, detailsOf, isText, mayBeInRange, metrics } from './common.js'
+import { soldeerRepos } from './repos.js'
 
 const REPOS_AT_ONCE = 4
+const DIR = 'github/advisories'
+// A maintainer publishes one whenever they are ready, and seeing it before
+// GitHub reviews it is what a repository is asked for: 90 minutes, not a month.
+const LISTING_TTL_MS = 90 * 60 * 1000
+// Stamped on each entry, and raised when a listing is kept differently.
+const VERSION = 1
 
 const isRepoAdvisory = (advisory) => advisory && typeof advisory === 'object' && isGhsa(advisory.ghsa_id)
   && advisory.state === 'published' && isText(advisory.summary)
   && (advisory.vulnerabilities == null || Array.isArray(advisory.vulnerabilities)) && (advisory.cwe_ids == null || isStrings(advisory.cwe_ids))
 
 export const assertClient = assertion('a GitHub client from createClient', (value) => typeof value?.listRepoAdvisories === 'function')
+export const assertCacheStore = assertion('a store with read and write', (value) => typeof value?.read === 'function' && typeof value?.write === 'function')
 
-const listAdvisories = (github, repo) => github.listRepoAdvisories({ repo }).catch(recover(isGone, []))
+// What rows are made from, and all that is kept of a listing: each advisory
+// not withdrawn, with its CVSS vector GitHub prefers, and each vulnerable
+// range with the package it names. A listing that is malformed is refused
+// whole, and never kept; one whose only fault is a description (isDigest)
+// is answered, but not kept either.
+function digest(repo, list) {
+  return list.flatMap((advisory) => {
+    assert.ok(isRepoAdvisory(advisory), `advisories: malformed advisory from ${repo}`)
+    if (advisory.withdrawn_at) return []
+    const cvss = [advisory.cvss_severities?.cvss_v3, advisory.cvss_severities?.cvss_v4, advisory.cvss].find((entry) => typeof entry?.vector_string === 'string')
+    const ranges = (advisory.vulnerabilities ?? []).map((vulnerability) => {
+      const range = vulnerability?.vulnerable_version_range ?? ''
+      assert.ok(typeof range === 'string', `advisories: malformed range in ${advisory.ghsa_id}`)
+      const { ecosystem, name } = vulnerability?.package ?? {}
+      return { range, ...(typeof ecosystem === 'string' && { ecosystem }), ...(typeof name === 'string' && { name }) }
+    })
+    return [{
+      ghsa: advisory.ghsa_id,
+      title: advisory.summary,
+      ...(advisory.description != null && advisory.description !== '' && { description: advisory.description }),
+      ...metrics({ severity: advisory.severity, score: cvss?.score, vector: cvss?.vector_string, cwe: advisory.cwe_ids }),
+      ranges,
+    }]
+  })
+}
+
+// A description is checked only for `details` (detailsOf), so a malformed
+// one leaves the listing usable, but not kept.
+const isDigest = (advisory) => advisory && typeof advisory === 'object' && isGhsa(advisory.ghsa) && isText(advisory.title)
+  && (advisory.description === undefined || isText(advisory.description))
+  && Array.isArray(advisory.ranges) && advisory.ranges.every((entry) => typeof entry?.range === 'string')
+
+// A repository's listing, digested, through the cache: GitHub's names are
+// case-insensitive, so one entry answers every spelling. A repository gone,
+// renamed or blocked has none, and that is not kept.
+async function listAdvisories(github, repo, store) {
+  const name = repo.toLowerCase()
+  const entry = await readRecord(DIR, name, { ttl: LISTING_TTL_MS, store })
+  if (entry?.v === VERSION && Array.isArray(entry.advisories) && entry.advisories.every(isDigest)) return entry.advisories
+  const list = await github.listRepoAdvisories({ repo }).catch(recover(isGone, null))
+  if (list === null) return []
+  const advisories = digest(repo, list)
+  if (advisories.every(isDigest)) await writeRecord(DIR, name, { v: VERSION, advisories }, { store })
+  return advisories
+}
 
 // GitHub's `>= 1.0.0, < 1.2.6` is npm's with the commas dropped.
 // Maintainers write these unreviewed: one without a range, or with one
-// semver cannot read, covers every version.
-function fromRepository(name, advisory, range, asked, covers) {
-  const cvss = [advisory.cvss_severities?.cvss_v3, advisory.cvss_severities?.cvss_v4, advisory.cvss].find((entry) => typeof entry?.vector_string === 'string')
+// semver cannot read, covers every version. Its page is on `repo`, which
+// may publish it long before the advisory database has one, and its
+// `description` is the text `details` asks for: the listing has it all.
+function fromRepository(repo, name, advisory, range, asked, { covers, details }) {
   return {
     name,
     source: 'repository',
-    id: advisory.ghsa_id,
-    ghsa: advisory.ghsa_id,
+    id: advisory.ghsa,
+    ghsa: advisory.ghsa,
+    url: advisoryUrl(advisory.ghsa, repo),
     aliases: [],
-    title: advisory.summary,
-    ...metrics({ severity: advisory.severity, score: cvss?.score, vector: cvss?.vector_string, cwe: advisory.cwe_ids }),
+    title: advisory.title,
+    ...(details && detailsOf(advisory.description, advisory.ghsa)),
+    ...metrics({ severity: advisory.severity, score: advisory.cvss, vector: advisory.cvssVector, cwe: advisory.cwe }),
     range,
     versions: covered(asked, range.replaceAll(',', ' '), covers),
   }
@@ -37,21 +92,18 @@ function fromRepository(name, advisory, range, asked, covers) {
 
 // Each asked name's repository, `repoOf` it, is asked once, and its
 // advisories' vulnerable ranges become rows for the names that `takes` a
-// vulnerability, one per name, advisory and range, holding the asked
-// versions it `covers`.
-async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers }) {
+// range's package, one per name, advisory and range, holding the asked
+// versions it `covers`, and with `details`, its text.
+async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers, details, cache }) {
   // GitHub's names are case-insensitive: one spelling asks for all.
   const namesOf = Map.groupBy([...asked.keys()].filter(repoOf), (name) => repoOf(name).toLowerCase())
-  const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ repo: repoOf(names[0]), names, list: await listAdvisories(github, repoOf(names[0])) }))
+  const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ repo: repoOf(names[0]), names, list: await listAdvisories(github, repoOf(names[0]), cache) }))
   const rows = new Map()
   for (const { repo, names, list } of listed) {
     for (const advisory of list) {
-      assert.ok(isRepoAdvisory(advisory), `advisories: malformed advisory from ${repo}`)
-      for (const vulnerability of advisory.withdrawn_at ? [] : advisory.vulnerabilities ?? []) {
-        const range = vulnerability?.vulnerable_version_range ?? ''
-        assert.ok(typeof range === 'string', `advisories: malformed range in ${advisory.ghsa_id}`)
-        for (const name of names.filter((candidate) => takes(candidate, vulnerability?.package))) {
-          rows.set(`${name} ${advisory.ghsa_id} ${range}`, fromRepository(name, advisory, range, asked.get(name), covers))
+      for (const { range, ...pkg } of advisory.ranges) {
+        for (const name of names.filter((candidate) => takes(candidate, pkg))) {
+          rows.set(`${name} ${advisory.ghsa} ${range}`, fromRepository(repo, name, advisory, range, asked.get(name), { covers, details }))
         }
       }
     }
@@ -59,30 +111,46 @@ async function repositoryAdvisories(github, asked, { repoOf, takes = () => true,
   return [...rows.values()]
 }
 
+// Each asked name's repository: as given, `known`, or else looked up.
+async function reposOf(asked, known, lookUp) {
+  const found = await lookUp([...asked.keys()].filter((name) => !known.has(name)))
+  return (name) => known.get(name) ?? found.get(name)
+}
+
 // A maintainer's advisory is on the repository before GitHub reviews it
 // into the databases the registries answer from. With `repoAdvisories`,
 // each package's repository, `known` or else looked up, adds what it
 // publishes for GitHub's `ecosystem` entries naming the package, for the
 // versions `rows` do not already report under that GHSA.
-export async function withRepositories(rows, asked, { github, repoAdvisories, known }, { ecosystem, lookUp, covers }) {
+export async function withRepositories(rows, asked, { github, repoAdvisories, known, details, cache }, { ecosystem, lookUp, covers }) {
   if (!repoAdvisories) return rows
   const reported = new Set(rows.flatMap((row) => [row.id, row.ghsa, ...row.aliases].filter(isGhsa).flatMap((id) => row.versions.map((version) => `${row.name} ${id} ${version}`))))
-  const found = await lookUp([...asked.keys()].filter((name) => !known.has(name)))
-  const repoOf = (name) => known.get(name) ?? found.get(name)
-  const takes = (name, pkg) => pkg?.ecosystem === ecosystem && pkg.name === name
-  const added = await repositoryAdvisories(github, asked, { repoOf, takes, covers })
+  const repoOf = await reposOf(asked, known, lookUp)
+  const takes = (name, pkg) => pkg.ecosystem === ecosystem && pkg.name === name
+  const added = await repositoryAdvisories(github, asked, { repoOf, takes, covers, details, cache })
   return [...rows, ...added.map((row) => ({ ...row, versions: row.versions.filter((version) => !reported.has(`${row.name} ${row.id} ${version}`)) }))]
 }
 
 // stasis versions a repository with no version of its own by its branch,
-// or 0.0.0: every range covers those. Whatever semver reads is a version,
-// `v1.2.3` and `1.2.3+build` included.
-const coversPlaceholder = (version, range) => version === '0.0.0' || valid(version) === null || inRange(version, range)
+// or 0.0.0: every range covers those.
+const coversPlaceholder = (version, range) => version === '0.0.0' || mayBeInRange(version, range)
 
 // Dependencies that are GitHub repositories themselves: every range their
 // own published advisories list counts, whichever package it names.
 export const GITHUB = {
+  repositoryOnly: true,
   assertName: assertRepo,
   assertVersion: assertion('a version or a branch name', isRefName),
-  advisories: (asked, { github }) => repositoryAdvisories(github, asked, { repoOf: (repo) => repo, covers: coversPlaceholder }),
+  advisories: (asked, { github, details, cache }) => repositoryAdvisories(github, asked, { repoOf: (repo) => repo, covers: coversPlaceholder, details, cache }),
+}
+
+// Soldeer packages, which no advisory database has: the repository each
+// is published from, as given or else as Soldeer's project names it, is
+// their only source, and every range its advisories list counts, whichever
+// package it names (often the npm package the same release went out as).
+export const SOLDEER = {
+  repositoryOnly: true,
+  assertName: assertSoldeerName,
+  assertVersion: assertSoldeerVersion,
+  advisories: async (asked, { github, known, details, cache }) => await repositoryAdvisories(github, asked, { repoOf: await reposOf(asked, known, soldeerRepos), details, cache }),
 }

package/src/advisories/index.js

@@ -1,12 +1,12 @@
 import assert from 'node:assert/strict'
 
-import { assertArgs, assertBoolean, assertRepo, assertion, optional, sameName } from '../args.js'
+import { assertArgs, assertBoolean, assertRepo, assertion, isIterable, optional, sameName } from '../args.js'
 import { order } from './common.js'
-import { GITHUB, assertClient } from './github.js'
+import { GITHUB, SOLDEER, assertCacheStore, assertClient } from './github.js'
 import { NPM } from './npm.js'
 import { CARGO, COMPOSER } from './osv.js'
 
-const ECOSYSTEMS = { npm: NPM, cargo: CARGO, composer: COMPOSER, github: GITHUB }
+const ECOSYSTEMS = { npm: NPM, cargo: CARGO, composer: COMPOSER, soldeer: SOLDEER, github: GITHUB }
 const byNumbers = new Intl.Collator('en', { numeric: true }).compare
 const assertEcosystem = assertion(`one of ${Object.keys(ECOSYSTEMS).join(', ')}`, (value) => typeof value === 'string' && Object.hasOwn(ECOSYSTEMS, value))
 const assertVersions = assertion('a non-empty array', (value) => Array.isArray(value) && value.length > 0)
@@ -14,7 +14,7 @@ const assertVersions = assertion('a non-empty array', (value) => Array.isArray(v
 // Ecosystem → name → { versions, github? }, a name given twice merged.
 // Every package is checked before any is used.
 function collect(packages) {
-  assert.ok(typeof packages?.[Symbol.iterator] === 'function' && typeof packages !== 'string', 'advisories: packages must be an iterable of { ecosystem, name, github?, versions }')
+  assert.ok(isIterable(packages), 'advisories: packages must be an iterable of { ecosystem, name, github?, versions }')
   const byEcosystem = new Map()
   for (const pkg of packages) {
     assertArgs('advisories', pkg, { ecosystem: assertEcosystem, name: null, github: optional(assertRepo), versions: assertVersions }, 'package')
@@ -34,16 +34,16 @@ function collect(packages) {
 }
 
 export async function advisories(packages, options = {}) {
-  assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean) })
+  assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean), cache: optional(assertCacheStore) })
   assert.ok(!options.repoAdvisories || options.github, 'advisories: repoAdvisories needs a github client')
   const byEcosystem = collect(packages)
-  assert.ok(!byEcosystem.has('github') || options.github, 'advisories: github packages need a github client')
+  for (const ecosystem of byEcosystem.keys()) assert.ok(!ECOSYSTEMS[ecosystem].repositoryOnly || options.github, `advisories: ${ecosystem} packages need a github client`)
   const found = await Promise.all([...byEcosystem].map(async ([ecosystem, named]) => {
     const { compare = byNumbers, advisories: find } = ECOSYSTEMS[ecosystem]
     const names = [...named.keys()].toSorted()
     const asked = new Map(names.map((name) => [name, [...named.get(name).versions].toSorted(compare)]))
     const known = new Map(names.filter((name) => named.get(name).github).map((name) => [name, named.get(name).github]))
-    const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, known })
+    const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, known, cache: options.cache })
     return rows.map((row) => ({ ecosystem, ...row }))
   }))
   return found.flat().filter((row) => row.versions.length > 0).toSorted((a, b) => order(a.ecosystem, b.ecosystem) || order(a.name, b.name))

package/src/advisories/npm.js

@@ -1,11 +1,12 @@
 import assert from 'node:assert/strict'
 
 import { assertPackageName, assertPackageVersion, isGhsa, isStrings, show } from '../args.js'
-import { NPM_REGISTRY, buildUrl, request } from '../http.js'
-import { chunks } from '../pool.js'
+import { NPM_REGISTRY, buildUrl, isNotFound, recover, request } from '../http.js'
+import { chunks, pool } from '../pool.js'
 import { compareVersions } from '../semver.js'
-import { covered, isText, metrics } from './common.js'
+import { advisoryUrl, covered, detailsOf, isText, metrics } from './common.js'
 import { withRepositories } from './github.js'
+import { RECORDS_AT_ONCE, getVuln } from './osv.js'
 import { npmRepos } from './repos.js'
 
 const BULK_URL = buildUrl(NPM_REGISTRY, ['-', 'npm', 'v1', 'security', 'advisories', 'bulk'])
@@ -23,7 +24,7 @@ function fromRegistry(name, row, asked) {
     name,
     source: 'registry',
     id: ghsa ?? `npm:${row.id}`,
-    ...(ghsa && { ghsa }),
+    ...(ghsa && { ghsa, url: advisoryUrl(ghsa) }),
     aliases: [],
     title: row.title,
     ...metrics({ severity: row.severity, score: row.cvss?.score, vector: row.cvss?.vectorString, cwe: row.cwe }),
@@ -49,10 +50,23 @@ async function registryAdvisories(asked) {
   return advisories
 }
 
+// The registry's rows carry no text: each GHSA's is OSV's record of it,
+// GitHub's advisory database as published, asked once for the rows that
+// cover an asked version. One OSV does not have yet has none.
+async function withDetails(rows) {
+  const ghsas = [...new Set(rows.filter((row) => row.ghsa && row.versions.length > 0).map((row) => row.ghsa))]
+  const records = await pool(ghsas, RECORDS_AT_ONCE, (ghsa) => getVuln(ghsa).catch(recover(isNotFound, undefined)))
+  const text = new Map(ghsas.map((ghsa, i) => [ghsa, records[i]?.details]))
+  return rows.map((row) => ({ ...row, ...detailsOf(text.get(row.ghsa), row.ghsa) }))
+}
+
 // What `npm audit` asks the registry, one row per vulnerable range.
 export const NPM = {
   assertName: assertPackageName,
   assertVersion: assertPackageVersion,
   compare: compareVersions,
-  advisories: async (asked, options) => await withRepositories(await registryAdvisories(asked), asked, options, { ecosystem: 'npm', lookUp: npmRepos }),
+  advisories: async (asked, options) => {
+    const rows = await registryAdvisories(asked)
+    return await withRepositories(options.details ? await withDetails(rows) : rows, asked, options, { ecosystem: 'npm', lookUp: npmRepos })
+  },
 }

package/src/advisories/osv.js

@@ -4,15 +4,24 @@ import { assertCrateName, assertCrateVersion, assertion, isGhsa, isStrings, matc
 import { OSV_API, buildUrl, request } from '../http.js'
 import { chunks, pool } from '../pool.js'
 import { isExactVersion } from '../semver.js'
-import { inRange, isText, metrics, order } from './common.js'
+import { advisoryUrl, detailsOf, inRange, isText, metrics, order } from './common.js'
 import { withRepositories } from './github.js'
 import { composerRepos, crateRepos } from './repos.js'
 
 const QUERY_URL = buildUrl(OSV_API, ['v1', 'querybatch'])
 const QUERIES_PER_REQUEST = 1000
-const RECORDS_AT_ONCE = 8
+export const RECORDS_AT_ONCE = 8
 const isOsvId = matches(/^(?=.{1,128}$)[A-Z][\dA-Z]*(?:-[\dA-Za-z]+)+$/u)
+// osv.dev's page for one of its records. An id that is only another's
+// alias (PKSA-…) has none, and redirects to a search.
+const osvPage = (id) => `https://osv.dev/vulnerability/${id}`
 const INFORMATIONAL = new Set(['unmaintained', 'unsound', 'notice'])
+// Of the records that publish one advisory, the first answered for a
+// version stands for the rest on it: RustSec's, what `cargo audit` reads
+// and GitHub mirrors, then the GHSA, then any other (MAL-…), then by id.
+const rank = (id) => (id.startsWith('RUSTSEC-') ? 0 : (isGhsa(id) ? 1 : 2))
+const first = (a, b) => rank(a.id) - rank(b.id) || order(a.id, b.id)
+const named = (a, b) => a.aliases.includes(b.id) || b.aliases.includes(a.id)
 
 export const CARGO = {
   osv: 'crates.io',
@@ -20,7 +29,6 @@ export const CARGO = {
   lookUp: crateRepos,
   assertName: assertCrateName,
   assertVersion: assertCrateVersion,
-  keep: (id) => id.startsWith('RUSTSEC-'), // What `cargo audit` reads; the GHSA records mirror it.
   advisories: (asked, options) => osvAdvisories(CARGO, asked, options),
 }
 // TODO: packagist.org's API, what `composer audit` reads, also has the
@@ -41,7 +49,7 @@ export const COMPOSER = {
   advisories: (asked, options) => osvAdvisories(COMPOSER, asked, options),
 }
 
-async function getVuln(id) {
+export async function getVuln(id) {
   const record = await request(buildUrl(OSV_API, ['v1', 'vulns', id]), { as: 'json' })
   assert.ok(record?.id === id, `advisories: OSV answered for ${show(record?.id)}, not ${id}`)
   assert.ok((record.aliases === undefined || isStrings(record.aliases)) && (record.summary === undefined || isText(record.summary))
@@ -49,7 +57,7 @@ async function getVuln(id) {
   return { ...record, aliases: (record.aliases ?? []).filter(isOsvId) }
 }
 
-function toAdvisory(ecosystem, name, versions, record) {
+function toAdvisory(ecosystem, name, versions, record, details) {
   const { aliases } = record
   const ghsas = aliases.filter(isGhsa)
   const ghsa = isGhsa(record.id) ? record.id : (ghsas.length === 1 ? ghsas[0] : undefined)
@@ -61,8 +69,10 @@ function toAdvisory(ecosystem, name, versions, record) {
     source: 'osv',
     id: record.id,
     ...(ghsa && { ghsa }),
+    url: ghsa ? advisoryUrl(ghsa) : osvPage(record.id),
     aliases,
     ...(record.summary && { title: record.summary }),
+    ...(details && detailsOf(record.details, record.id)),
     ...metrics({ severity: record.database_specific?.severity, vector, cwe: record.database_specific?.cwe_ids }),
     ...(INFORMATIONAL.has(informational) && { informational }),
     versions,
@@ -70,11 +80,13 @@ function toAdvisory(ecosystem, name, versions, record) {
 }
 
 // OSV's batch query matches versions on its side, but answers ids only:
-// each record is fetched once after. A record another database also
-// publishes comes back under both ids, so one that aliases a GHSA keeps
-// only the versions that GHSA was not answered for.
+// each record is fetched once after. An advisory more than one database
+// publishes comes back under each one's id, the records naming each other
+// as aliases, either way round, directly or through others: one group, in
+// which each version is reported under the first answered for it, and
+// each record has the others' ids as aliases.
 async function osvAdvisories(ecosystem, asked, options) {
-  const { osv, github, lookUp, covers, keep = () => true } = ecosystem
+  const { osv, github, lookUp, covers } = ecosystem
   const list = [...asked].flatMap(([name, versions]) => versions.map((version) => ({ name, version })))
   const hits = new Map() // id → name → versions, in `list` order
   for (const chunk of chunks(list, QUERIES_PER_REQUEST)) {
@@ -85,21 +97,28 @@ async function osvAdvisories(ecosystem, asked, options) {
       const { name, version } = chunk[j]
       const vulns = result?.vulns ?? []
       assert.ok(result && result.next_page_token === undefined && Array.isArray(vulns) && vulns.every((vuln) => isOsvId(vuln?.id)), `advisories: malformed OSV result for ${name}@${version}`)
-      for (const { id } of vulns.filter((vuln) => keep(vuln.id))) {
+      for (const { id } of vulns) {
         const byName = hits.get(id) ?? hits.set(id, new Map()).get(id)
         byName.set(name, (byName.get(name) ?? new Set()).add(version))
       }
     }
   }
   const records = (await pool([...hits.keys()], RECORDS_AT_ONCE, getVuln)).filter((record) => !record.withdrawn)
-  const live = new Set(records.map((record) => record.id))
+  const groups = new Map()
+  for (const record of records) {
+    if (groups.has(record)) continue
+    const group = [record]
+    for (const member of group) group.push(...records.filter((other) => !group.includes(other) && named(member, other)))
+    for (const member of group) groups.set(member, group)
+  }
   const rows = []
   for (const record of records) {
+    const group = groups.get(record)
+    const above = group.filter((other) => first(other, record) < 0)
+    const aliases = [...new Set([...record.aliases, ...group.map((other) => other.id)])].filter((id) => id !== record.id)
     for (const [name, versions] of hits.get(record.id)) {
-      // Only the versions a live GHSA it aliases answered for too.
-      const shadowed = isGhsa(record.id) ? [] : record.aliases.filter((alias) => isGhsa(alias) && live.has(alias)).flatMap((alias) => [...(hits.get(alias).get(name) ?? [])])
-      const rest = [...versions].filter((version) => !shadowed.includes(version))
-      if (rest.length > 0) rows.push(toAdvisory(osv, name, rest, record))
+      const rest = [...versions].filter((version) => !above.some((other) => hits.get(other.id).get(name)?.has(version)))
+      if (rest.length > 0) rows.push(toAdvisory(osv, name, rest, { ...record, aliases }, options.details))
     }
   }
   rows.sort((a, b) => order(a.name, b.name) || order(a.id, b.id))

package/src/advisories/repos.js

@@ -1,51 +1,24 @@
 import assert from 'node:assert/strict'
-import { setTimeout as sleep } from 'node:timers/promises'
 
-import { isRepo, show } from '../args.js'
-import { readRecord, writeRecord } from '../cache.js'
-import { CRATES_API, PACKAGIST_REPO, buildUrl, isNotFound, recover, request } from '../http.js'
+import { show } from '../args.js'
+import { addRepos, readRepos } from '../cache.js'
+import { lookUpCrateRepos } from '../cargo/repos.js'
+import { PACKAGIST_REPO, SOLDEER_API, buildUrl, isNotFound, recover, request } from '../http.js'
 import { lookUpPackageRepo } from '../npm/repos.js'
-import { chunks, pool } from '../pool.js'
+import { pool } from '../pool.js'
 import { githubRepoOfUrl } from '../remote.js'
 
-const CRATES_PER_REQUEST = 100
-const CRATES_PACE_MS = 1000 // crates.io asks for a request a second at most, and a user agent.
-const USER_AGENT = '@preventive/upstream (https://github.com/PreventiveMeasures/libraries)'
 const PACKAGES_AT_ONCE = 8
 
 // Name → its GitHub repo, or null where the registry has no such package
 // or it names no GitHub repo; any other failure throws. Only a repo found
 // is cached, for a month.
 async function cachedRepos(dir, names, fetchMissing) {
-  const repos = new Map()
-  for (const name of names) {
-    const entry = await readRecord(dir, name)
-    if (isRepo(entry?.github)) repos.set(name, entry.github)
-  }
-  const missing = names.filter((name) => !repos.has(name))
-  for (const [name, github] of await fetchMissing(missing)) {
-    repos.set(name, github)
-    if (github) await writeRecord(dir, name, { github })
-  }
+  const repos = await readRepos(dir, names)
+  await addRepos(dir, repos, await fetchMissing(names.filter((name) => !repos.has(name))))
   return repos
 }
 
-// A crate crates.io leaves out of its answer is one it does not have.
-async function fetchCrates(names) {
-  const found = new Map(names.map((name) => [name, null]))
-  for (const [i, chunk] of chunks(names, CRATES_PER_REQUEST).entries()) {
-    if (i > 0) await sleep(CRATES_PACE_MS)
-    const url = buildUrl(CRATES_API, ['api', 'v1', 'crates'], { 'ids[]': chunk, per_page: CRATES_PER_REQUEST })
-    const answer = await request(url, { as: 'json', headers: { 'User-Agent': USER_AGENT } })
-    assert.ok(Array.isArray(answer?.crates), 'advisories: expected a list of crates from crates.io')
-    for (const crate of answer.crates) {
-      assert.ok(chunk.includes(crate?.id), `advisories: crates.io answered for ${show(crate?.id)}, which was not asked`)
-      found.set(crate.id, githubRepoOfUrl(crate.repository) ?? null)
-    }
-  }
-  return found
-}
-
 async function fetchEach(names, fetch) {
   const found = await pool(names, PACKAGES_AT_ONCE, fetch)
   return new Map(names.map((name, i) => [name, found[i]]))
@@ -61,6 +34,15 @@ async function fetchComposerRepo(name) {
   return githubRepoOfUrl(versions[0]?.source?.url) ?? null
 }
 
-export const crateRepos = (names) => cachedRepos('cargo/repos', names, fetchCrates)
+// Soldeer answers a project it does not have with an empty list.
+async function fetchSoldeerRepo(name) {
+  const answer = await request(buildUrl(SOLDEER_API, ['api', 'v1', 'project'], { project_name: name }), { as: 'json' })
+  assert.ok(Array.isArray(answer?.data), `advisories: expected a list of projects from Soldeer for ${name}`)
+  for (const project of answer.data) assert.ok(project?.name === name, `advisories: Soldeer answered for ${show(project?.name)}, not ${name}`)
+  return githubRepoOfUrl(answer.data[0]?.github_url) ?? null
+}
+
+export const crateRepos = (names) => lookUpCrateRepos('advisories', names)
 export const composerRepos = (names) => cachedRepos('composer/repos', names, (missing) => fetchEach(missing, fetchComposerRepo))
 export const npmRepos = (names) => fetchEach(names, async (name) => (await lookUpPackageRepo(name))?.github ?? null)
+export const soldeerRepos = (names) => cachedRepos('soldeer/repos', names, (missing) => fetchEach(missing, fetchSoldeerRepo))