@preventive/upstream 1.0.0-alpha.1 → 1.0.0-alpha.10

package/semver.d.ts

@@ -1,10 +1,11 @@
 // Hand-written against semver.js; a change to either belongs with the other.
 
 // npm's own semver, borrowed from the npm beside node and passed through
-// as it is. Without it, every call throws, except valid() on a plain
+// as it is; where there is no npm, the `semver` package, an optional peer
+// dependency. Without either, every call throws, except valid() on a plain
 // MAJOR.MINOR.PATCH with no options.
 export function satisfies(version: string, range: string, options?: { includePrerelease?: boolean; loose?: boolean }): boolean
-export function compareVersions(a: string, b: string): -1 | 0 | 1
+export function compareVersions(a: string, b: string, options?: { loose?: boolean }): -1 | 0 | 1
 // The version as semver spells it, or null: `v1.2.3` answers `1.2.3`.
 export function valid(version: unknown, options?: { loose?: boolean }): string | null
 // A string that valid() answers unchanged.
@@ -13,3 +14,8 @@ export function isExactVersion(version: unknown): version is string
 export function validRange(range: unknown, options?: { loose?: boolean; includePrerelease?: boolean }): string | null
 // Whether some version is in both ranges; throws for one that is not a range.
 export function intersects(a: string, b: string, options?: { loose?: boolean; includePrerelease?: boolean }): boolean
+// The version as semver cleans it, spaces and a leading `=` or `v` dropped,
+// or null: ` =v1.2.3 ` answers `1.2.3`.
+export function clean(version: string, options?: { loose?: boolean } | boolean): string | null
+// The version's major, as semver reads it; throws for one that is not a version.
+export function major(version: string, options?: { loose?: boolean } | boolean): number

package/semver.js

@@ -1 +1 @@
-export { compareVersions, intersects, isExactVersion, satisfies, valid, validRange } from './src/semver.js'
+export { clean, compareVersions, intersects, isExactVersion, major, satisfies, valid, validRange } from './src/semver.js'

package/src/advisories/common.js

@@ -1,21 +1,43 @@
-import { isStrings, matches } from '../args.js'
-import { satisfies, validRange } from '../semver.js'
+import assert from 'node:assert/strict'
+
+import { isGhsa, isRepo, isStrings, matches, show } from '../args.js'
+import { satisfies, valid, validRange } from '../semver.js'
 
 export const order = (a, b) => (a > b) - (a < b)
 
+// GitHub's page for a GHSA: the advisory database's, which has every one
+// the registry and OSV answer with, or the page of the `repo` that
+// published it, which has it before GitHub reviews it into the database.
+export function advisoryUrl(ghsa, repo) {
+  assert.ok(isGhsa(ghsa) && (repo === undefined || isRepo(repo)), `advisories: no GitHub advisory page for ${show(ghsa)} on ${show(repo)}`)
+  return repo === undefined ? `https://github.com/advisories/${ghsa}` : `https://github.com/${repo}/security/advisories/${ghsa}`
+}
+
 // As `npm audit` matches: a prerelease is in a range around it.
 const SEMVER = { includePrerelease: true, loose: true }
 
 export const inRange = (version, range) => satisfies(version, range, SEMVER)
 
-// The asked versions a range `covers`. One semver cannot read covers them
-// all: a missed advisory is worse than a spare one.
-export const covered = (asked, range, covers = inRange) => (validRange(range, SEMVER) === null ? [...asked] : asked.filter((version) => covers(version, range)))
+// A version semver cannot read may be in any range. Whatever it reads is
+// a version, `v1.2.3` and `1.2.3+build` included.
+export const mayBeInRange = (version, range) => valid(version) === null || inRange(version, range)
+
+// The asked versions a range `covers`. One semver cannot read, range or
+// version, covers them all: a missed advisory is worse than a spare one.
+export const covered = (asked, range, covers = mayBeInRange) => (validRange(range, SEMVER) === null ? [...asked] : asked.filter((version) => covers(version, range)))
 
 // Remote text, as long as it is well-formed; anything else is refused as
 // malformed by whoever reads it.
 export const isText = (value) => typeof value === 'string' && value.isWellFormed()
 
+// An advisory's full text, for `details`: none where its source has none,
+// and refused where it is not well-formed.
+export function detailsOf(text, id) {
+  if (text == null || text === '') return {}
+  assert.ok(isText(text), `advisories: malformed details in ${id}`)
+  return { details: text }
+}
+
 const SEVERITIES = new Set(['critical', 'high', 'moderate', 'low', 'info'])
 const isCvssVector = matches(/^CVSS:[34]\.\d(?:\/[A-Z]{1,4}:[A-Z]{1,2})+$/u)
 const isCwe = matches(/^CWE-\d{1,6}$/u)

package/src/advisories/github.js

@@ -1,11 +1,11 @@
 import assert from 'node:assert/strict'
 
-import { assertRepo, assertion, isGhsa, isRefName, isStrings } from '../args.js'
+import { assertRepo, assertSoldeerName, assertSoldeerVersion, assertion, isGhsa, isRefName, isStrings } from '../args.js'
 import { isGone } from '../github/client.js'
 import { recover } from '../http.js'
 import { pool } from '../pool.js'
-import { valid } from '../semver.js'
-import { covered, inRange, isText, metrics } from './common.js'
+import { advisoryUrl, covered, detailsOf, isText, mayBeInRange, metrics } from './common.js'
+import { soldeerRepos } from './repos.js'
 
 const REPOS_AT_ONCE = 4
 
@@ -19,16 +19,20 @@ const listAdvisories = (github, repo) => github.listRepoAdvisories({ repo }).cat
 
 // GitHub's `>= 1.0.0, < 1.2.6` is npm's with the commas dropped.
 // Maintainers write these unreviewed: one without a range, or with one
-// semver cannot read, covers every version.
-function fromRepository(name, advisory, range, asked, covers) {
+// semver cannot read, covers every version. Its page is on `repo`, which
+// may publish it long before the advisory database has one, and its
+// `description` is the text `details` asks for: the listing has it all.
+function fromRepository(repo, name, advisory, range, asked, { covers, details }) {
   const cvss = [advisory.cvss_severities?.cvss_v3, advisory.cvss_severities?.cvss_v4, advisory.cvss].find((entry) => typeof entry?.vector_string === 'string')
   return {
     name,
     source: 'repository',
     id: advisory.ghsa_id,
     ghsa: advisory.ghsa_id,
+    url: advisoryUrl(advisory.ghsa_id, repo),
     aliases: [],
     title: advisory.summary,
+    ...(details && detailsOf(advisory.description, advisory.ghsa_id)),
     ...metrics({ severity: advisory.severity, score: cvss?.score, vector: cvss?.vector_string, cwe: advisory.cwe_ids }),
     range,
     versions: covered(asked, range.replaceAll(',', ' '), covers),
@@ -38,8 +42,8 @@ function fromRepository(name, advisory, range, asked, covers) {
 // Each asked name's repository, `repoOf` it, is asked once, and its
 // advisories' vulnerable ranges become rows for the names that `takes` a
 // vulnerability, one per name, advisory and range, holding the asked
-// versions it `covers`.
-async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers }) {
+// versions it `covers`, and with `details`, its text.
+async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers, details }) {
   // GitHub's names are case-insensitive: one spelling asks for all.
   const namesOf = Map.groupBy([...asked.keys()].filter(repoOf), (name) => repoOf(name).toLowerCase())
   const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ repo: repoOf(names[0]), names, list: await listAdvisories(github, repoOf(names[0])) }))
@@ -51,7 +55,7 @@ async function repositoryAdvisories(github, asked, { repoOf, takes = () => true,
         const range = vulnerability?.vulnerable_version_range ?? ''
         assert.ok(typeof range === 'string', `advisories: malformed range in ${advisory.ghsa_id}`)
         for (const name of names.filter((candidate) => takes(candidate, vulnerability?.package))) {
-          rows.set(`${name} ${advisory.ghsa_id} ${range}`, fromRepository(name, advisory, range, asked.get(name), covers))
+          rows.set(`${name} ${advisory.ghsa_id} ${range}`, fromRepository(repo, name, advisory, range, asked.get(name), { covers, details }))
         }
       }
     }
@@ -59,30 +63,46 @@ async function repositoryAdvisories(github, asked, { repoOf, takes = () => true,
   return [...rows.values()]
 }
 
+// Each asked name's repository: as given, `known`, or else looked up.
+async function reposOf(asked, known, lookUp) {
+  const found = await lookUp([...asked.keys()].filter((name) => !known.has(name)))
+  return (name) => known.get(name) ?? found.get(name)
+}
+
 // A maintainer's advisory is on the repository before GitHub reviews it
 // into the databases the registries answer from. With `repoAdvisories`,
 // each package's repository, `known` or else looked up, adds what it
 // publishes for GitHub's `ecosystem` entries naming the package, for the
 // versions `rows` do not already report under that GHSA.
-export async function withRepositories(rows, asked, { github, repoAdvisories, known }, { ecosystem, lookUp, covers }) {
+export async function withRepositories(rows, asked, { github, repoAdvisories, known, details }, { ecosystem, lookUp, covers }) {
   if (!repoAdvisories) return rows
   const reported = new Set(rows.flatMap((row) => [row.id, row.ghsa, ...row.aliases].filter(isGhsa).flatMap((id) => row.versions.map((version) => `${row.name} ${id} ${version}`))))
-  const found = await lookUp([...asked.keys()].filter((name) => !known.has(name)))
-  const repoOf = (name) => known.get(name) ?? found.get(name)
+  const repoOf = await reposOf(asked, known, lookUp)
   const takes = (name, pkg) => pkg?.ecosystem === ecosystem && pkg.name === name
-  const added = await repositoryAdvisories(github, asked, { repoOf, takes, covers })
+  const added = await repositoryAdvisories(github, asked, { repoOf, takes, covers, details })
   return [...rows, ...added.map((row) => ({ ...row, versions: row.versions.filter((version) => !reported.has(`${row.name} ${row.id} ${version}`)) }))]
 }
 
 // stasis versions a repository with no version of its own by its branch,
-// or 0.0.0: every range covers those. Whatever semver reads is a version,
-// `v1.2.3` and `1.2.3+build` included.
-const coversPlaceholder = (version, range) => version === '0.0.0' || valid(version) === null || inRange(version, range)
+// or 0.0.0: every range covers those.
+const coversPlaceholder = (version, range) => version === '0.0.0' || mayBeInRange(version, range)
 
 // Dependencies that are GitHub repositories themselves: every range their
 // own published advisories list counts, whichever package it names.
 export const GITHUB = {
+  repositoryOnly: true,
   assertName: assertRepo,
   assertVersion: assertion('a version or a branch name', isRefName),
-  advisories: (asked, { github }) => repositoryAdvisories(github, asked, { repoOf: (repo) => repo, covers: coversPlaceholder }),
+  advisories: (asked, { github, details }) => repositoryAdvisories(github, asked, { repoOf: (repo) => repo, covers: coversPlaceholder, details }),
+}
+
+// Soldeer packages, which no advisory database has: the repository each
+// is published from, as given or else as Soldeer's project names it, is
+// their only source, and every range its advisories list counts, whichever
+// package it names (often the npm package the same release went out as).
+export const SOLDEER = {
+  repositoryOnly: true,
+  assertName: assertSoldeerName,
+  assertVersion: assertSoldeerVersion,
+  advisories: async (asked, { github, known, details }) => await repositoryAdvisories(github, asked, { repoOf: await reposOf(asked, known, soldeerRepos), details }),
 }

package/src/advisories/index.js

@@ -1,12 +1,12 @@
 import assert from 'node:assert/strict'
 
-import { assertArgs, assertBoolean, assertRepo, assertion, optional, sameName } from '../args.js'
+import { assertArgs, assertBoolean, assertRepo, assertion, isIterable, optional, sameName } from '../args.js'
 import { order } from './common.js'
-import { GITHUB, assertClient } from './github.js'
+import { GITHUB, SOLDEER, assertClient } from './github.js'
 import { NPM } from './npm.js'
 import { CARGO, COMPOSER } from './osv.js'
 
-const ECOSYSTEMS = { npm: NPM, cargo: CARGO, composer: COMPOSER, github: GITHUB }
+const ECOSYSTEMS = { npm: NPM, cargo: CARGO, composer: COMPOSER, soldeer: SOLDEER, github: GITHUB }
 const byNumbers = new Intl.Collator('en', { numeric: true }).compare
 const assertEcosystem = assertion(`one of ${Object.keys(ECOSYSTEMS).join(', ')}`, (value) => typeof value === 'string' && Object.hasOwn(ECOSYSTEMS, value))
 const assertVersions = assertion('a non-empty array', (value) => Array.isArray(value) && value.length > 0)
@@ -14,7 +14,7 @@ const assertVersions = assertion('a non-empty array', (value) => Array.isArray(v
 // Ecosystem → name → { versions, github? }, a name given twice merged.
 // Every package is checked before any is used.
 function collect(packages) {
-  assert.ok(typeof packages?.[Symbol.iterator] === 'function' && typeof packages !== 'string', 'advisories: packages must be an iterable of { ecosystem, name, github?, versions }')
+  assert.ok(isIterable(packages), 'advisories: packages must be an iterable of { ecosystem, name, github?, versions }')
   const byEcosystem = new Map()
   for (const pkg of packages) {
     assertArgs('advisories', pkg, { ecosystem: assertEcosystem, name: null, github: optional(assertRepo), versions: assertVersions }, 'package')
@@ -34,16 +34,16 @@ function collect(packages) {
 }
 
 export async function advisories(packages, options = {}) {
-  assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean) })
+  assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean) })
   assert.ok(!options.repoAdvisories || options.github, 'advisories: repoAdvisories needs a github client')
   const byEcosystem = collect(packages)
-  assert.ok(!byEcosystem.has('github') || options.github, 'advisories: github packages need a github client')
+  for (const ecosystem of byEcosystem.keys()) assert.ok(!ECOSYSTEMS[ecosystem].repositoryOnly || options.github, `advisories: ${ecosystem} packages need a github client`)
   const found = await Promise.all([...byEcosystem].map(async ([ecosystem, named]) => {
     const { compare = byNumbers, advisories: find } = ECOSYSTEMS[ecosystem]
     const names = [...named.keys()].toSorted()
     const asked = new Map(names.map((name) => [name, [...named.get(name).versions].toSorted(compare)]))
     const known = new Map(names.filter((name) => named.get(name).github).map((name) => [name, named.get(name).github]))
-    const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, known })
+    const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, known })
     return rows.map((row) => ({ ecosystem, ...row }))
   }))
   return found.flat().filter((row) => row.versions.length > 0).toSorted((a, b) => order(a.ecosystem, b.ecosystem) || order(a.name, b.name))

package/src/advisories/npm.js

@@ -1,11 +1,12 @@
 import assert from 'node:assert/strict'
 
 import { assertPackageName, assertPackageVersion, isGhsa, isStrings, show } from '../args.js'
-import { NPM_REGISTRY, buildUrl, request } from '../http.js'
-import { chunks } from '../pool.js'
+import { NPM_REGISTRY, buildUrl, isNotFound, recover, request } from '../http.js'
+import { chunks, pool } from '../pool.js'
 import { compareVersions } from '../semver.js'
-import { covered, isText, metrics } from './common.js'
+import { advisoryUrl, covered, detailsOf, isText, metrics } from './common.js'
 import { withRepositories } from './github.js'
+import { RECORDS_AT_ONCE, getVuln } from './osv.js'
 import { npmRepos } from './repos.js'
 
 const BULK_URL = buildUrl(NPM_REGISTRY, ['-', 'npm', 'v1', 'security', 'advisories', 'bulk'])
@@ -23,7 +24,7 @@ function fromRegistry(name, row, asked) {
     name,
     source: 'registry',
     id: ghsa ?? `npm:${row.id}`,
-    ...(ghsa && { ghsa }),
+    ...(ghsa && { ghsa, url: advisoryUrl(ghsa) }),
     aliases: [],
     title: row.title,
     ...metrics({ severity: row.severity, score: row.cvss?.score, vector: row.cvss?.vectorString, cwe: row.cwe }),
@@ -49,10 +50,23 @@ async function registryAdvisories(asked) {
   return advisories
 }
 
+// The registry's rows carry no text: each GHSA's is OSV's record of it,
+// GitHub's advisory database as published, asked once for the rows that
+// cover an asked version. One OSV does not have yet has none.
+async function withDetails(rows) {
+  const ghsas = [...new Set(rows.filter((row) => row.ghsa && row.versions.length > 0).map((row) => row.ghsa))]
+  const records = await pool(ghsas, RECORDS_AT_ONCE, (ghsa) => getVuln(ghsa).catch(recover(isNotFound, undefined)))
+  const text = new Map(ghsas.map((ghsa, i) => [ghsa, records[i]?.details]))
+  return rows.map((row) => ({ ...row, ...detailsOf(text.get(row.ghsa), row.ghsa) }))
+}
+
 // What `npm audit` asks the registry, one row per vulnerable range.
 export const NPM = {
   assertName: assertPackageName,
   assertVersion: assertPackageVersion,
   compare: compareVersions,
-  advisories: async (asked, options) => await withRepositories(await registryAdvisories(asked), asked, options, { ecosystem: 'npm', lookUp: npmRepos }),
+  advisories: async (asked, options) => {
+    const rows = await registryAdvisories(asked)
+    return await withRepositories(options.details ? await withDetails(rows) : rows, asked, options, { ecosystem: 'npm', lookUp: npmRepos })
+  },
 }

package/src/advisories/osv.js

@@ -4,15 +4,24 @@ import { assertCrateName, assertCrateVersion, assertion, isGhsa, isStrings, matc
 import { OSV_API, buildUrl, request } from '../http.js'
 import { chunks, pool } from '../pool.js'
 import { isExactVersion } from '../semver.js'
-import { inRange, isText, metrics, order } from './common.js'
+import { advisoryUrl, detailsOf, inRange, isText, metrics, order } from './common.js'
 import { withRepositories } from './github.js'
 import { composerRepos, crateRepos } from './repos.js'
 
 const QUERY_URL = buildUrl(OSV_API, ['v1', 'querybatch'])
 const QUERIES_PER_REQUEST = 1000
-const RECORDS_AT_ONCE = 8
+export const RECORDS_AT_ONCE = 8
 const isOsvId = matches(/^(?=.{1,128}$)[A-Z][\dA-Z]*(?:-[\dA-Za-z]+)+$/u)
+// osv.dev's page for one of its records. An id that is only another's
+// alias (PKSA-…) has none, and redirects to a search.
+const osvPage = (id) => `https://osv.dev/vulnerability/${id}`
 const INFORMATIONAL = new Set(['unmaintained', 'unsound', 'notice'])
+// Of the records that publish one advisory, the first answered for a
+// version stands for the rest on it: RustSec's, what `cargo audit` reads
+// and GitHub mirrors, then the GHSA, then any other (MAL-…), then by id.
+const rank = (id) => (id.startsWith('RUSTSEC-') ? 0 : (isGhsa(id) ? 1 : 2))
+const first = (a, b) => rank(a.id) - rank(b.id) || order(a.id, b.id)
+const named = (a, b) => a.aliases.includes(b.id) || b.aliases.includes(a.id)
 
 export const CARGO = {
   osv: 'crates.io',
@@ -20,7 +29,6 @@ export const CARGO = {
   lookUp: crateRepos,
   assertName: assertCrateName,
   assertVersion: assertCrateVersion,
-  keep: (id) => id.startsWith('RUSTSEC-'), // What `cargo audit` reads; the GHSA records mirror it.
   advisories: (asked, options) => osvAdvisories(CARGO, asked, options),
 }
 // TODO: packagist.org's API, what `composer audit` reads, also has the
@@ -41,7 +49,7 @@ export const COMPOSER = {
   advisories: (asked, options) => osvAdvisories(COMPOSER, asked, options),
 }
 
-async function getVuln(id) {
+export async function getVuln(id) {
   const record = await request(buildUrl(OSV_API, ['v1', 'vulns', id]), { as: 'json' })
   assert.ok(record?.id === id, `advisories: OSV answered for ${show(record?.id)}, not ${id}`)
   assert.ok((record.aliases === undefined || isStrings(record.aliases)) && (record.summary === undefined || isText(record.summary))
@@ -49,7 +57,7 @@ async function getVuln(id) {
   return { ...record, aliases: (record.aliases ?? []).filter(isOsvId) }
 }
 
-function toAdvisory(ecosystem, name, versions, record) {
+function toAdvisory(ecosystem, name, versions, record, details) {
   const { aliases } = record
   const ghsas = aliases.filter(isGhsa)
   const ghsa = isGhsa(record.id) ? record.id : (ghsas.length === 1 ? ghsas[0] : undefined)
@@ -61,8 +69,10 @@ function toAdvisory(ecosystem, name, versions, record) {
     source: 'osv',
     id: record.id,
     ...(ghsa && { ghsa }),
+    url: ghsa ? advisoryUrl(ghsa) : osvPage(record.id),
     aliases,
     ...(record.summary && { title: record.summary }),
+    ...(details && detailsOf(record.details, record.id)),
     ...metrics({ severity: record.database_specific?.severity, vector, cwe: record.database_specific?.cwe_ids }),
     ...(INFORMATIONAL.has(informational) && { informational }),
     versions,
@@ -70,11 +80,13 @@ function toAdvisory(ecosystem, name, versions, record) {
 }
 
 // OSV's batch query matches versions on its side, but answers ids only:
-// each record is fetched once after. A record another database also
-// publishes comes back under both ids, so one that aliases a GHSA keeps
-// only the versions that GHSA was not answered for.
+// each record is fetched once after. An advisory more than one database
+// publishes comes back under each one's id, the records naming each other
+// as aliases, either way round, directly or through others: one group, in
+// which each version is reported under the first answered for it, and
+// each record has the others' ids as aliases.
 async function osvAdvisories(ecosystem, asked, options) {
-  const { osv, github, lookUp, covers, keep = () => true } = ecosystem
+  const { osv, github, lookUp, covers } = ecosystem
   const list = [...asked].flatMap(([name, versions]) => versions.map((version) => ({ name, version })))
   const hits = new Map() // id → name → versions, in `list` order
   for (const chunk of chunks(list, QUERIES_PER_REQUEST)) {
@@ -85,21 +97,28 @@ async function osvAdvisories(ecosystem, asked, options) {
       const { name, version } = chunk[j]
       const vulns = result?.vulns ?? []
       assert.ok(result && result.next_page_token === undefined && Array.isArray(vulns) && vulns.every((vuln) => isOsvId(vuln?.id)), `advisories: malformed OSV result for ${name}@${version}`)
-      for (const { id } of vulns.filter((vuln) => keep(vuln.id))) {
+      for (const { id } of vulns) {
         const byName = hits.get(id) ?? hits.set(id, new Map()).get(id)
         byName.set(name, (byName.get(name) ?? new Set()).add(version))
       }
     }
   }
   const records = (await pool([...hits.keys()], RECORDS_AT_ONCE, getVuln)).filter((record) => !record.withdrawn)
-  const live = new Set(records.map((record) => record.id))
+  const groups = new Map()
+  for (const record of records) {
+    if (groups.has(record)) continue
+    const group = [record]
+    for (const member of group) group.push(...records.filter((other) => !group.includes(other) && named(member, other)))
+    for (const member of group) groups.set(member, group)
+  }
   const rows = []
   for (const record of records) {
+    const group = groups.get(record)
+    const above = group.filter((other) => first(other, record) < 0)
+    const aliases = [...new Set([...record.aliases, ...group.map((other) => other.id)])].filter((id) => id !== record.id)
     for (const [name, versions] of hits.get(record.id)) {
-      // Only the versions a live GHSA it aliases answered for too.
-      const shadowed = isGhsa(record.id) ? [] : record.aliases.filter((alias) => isGhsa(alias) && live.has(alias)).flatMap((alias) => [...(hits.get(alias).get(name) ?? [])])
-      const rest = [...versions].filter((version) => !shadowed.includes(version))
-      if (rest.length > 0) rows.push(toAdvisory(osv, name, rest, record))
+      const rest = [...versions].filter((version) => !above.some((other) => hits.get(other.id).get(name)?.has(version)))
+      if (rest.length > 0) rows.push(toAdvisory(osv, name, rest, { ...record, aliases }, options.details))
     }
   }
   rows.sort((a, b) => order(a.name, b.name) || order(a.id, b.id))

package/src/advisories/repos.js

@@ -1,51 +1,24 @@
 import assert from 'node:assert/strict'
-import { setTimeout as sleep } from 'node:timers/promises'
 
-import { isRepo, show } from '../args.js'
-import { readRecord, writeRecord } from '../cache.js'
-import { CRATES_API, PACKAGIST_REPO, buildUrl, isNotFound, recover, request } from '../http.js'
+import { show } from '../args.js'
+import { addRepos, readRepos } from '../cache.js'
+import { lookUpCrateRepos } from '../cargo/repos.js'
+import { PACKAGIST_REPO, SOLDEER_API, buildUrl, isNotFound, recover, request } from '../http.js'
 import { lookUpPackageRepo } from '../npm/repos.js'
-import { chunks, pool } from '../pool.js'
+import { pool } from '../pool.js'
 import { githubRepoOfUrl } from '../remote.js'
 
-const CRATES_PER_REQUEST = 100
-const CRATES_PACE_MS = 1000 // crates.io asks for a request a second at most, and a user agent.
-const USER_AGENT = '@preventive/upstream (https://github.com/PreventiveMeasures/libraries)'
 const PACKAGES_AT_ONCE = 8
 
 // Name → its GitHub repo, or null where the registry has no such package
 // or it names no GitHub repo; any other failure throws. Only a repo found
 // is cached, for a month.
 async function cachedRepos(dir, names, fetchMissing) {
-  const repos = new Map()
-  for (const name of names) {
-    const entry = await readRecord(dir, name)
-    if (isRepo(entry?.github)) repos.set(name, entry.github)
-  }
-  const missing = names.filter((name) => !repos.has(name))
-  for (const [name, github] of await fetchMissing(missing)) {
-    repos.set(name, github)
-    if (github) await writeRecord(dir, name, { github })
-  }
+  const repos = await readRepos(dir, names)
+  await addRepos(dir, repos, await fetchMissing(names.filter((name) => !repos.has(name))))
   return repos
 }
 
-// A crate crates.io leaves out of its answer is one it does not have.
-async function fetchCrates(names) {
-  const found = new Map(names.map((name) => [name, null]))
-  for (const [i, chunk] of chunks(names, CRATES_PER_REQUEST).entries()) {
-    if (i > 0) await sleep(CRATES_PACE_MS)
-    const url = buildUrl(CRATES_API, ['api', 'v1', 'crates'], { 'ids[]': chunk, per_page: CRATES_PER_REQUEST })
-    const answer = await request(url, { as: 'json', headers: { 'User-Agent': USER_AGENT } })
-    assert.ok(Array.isArray(answer?.crates), 'advisories: expected a list of crates from crates.io')
-    for (const crate of answer.crates) {
-      assert.ok(chunk.includes(crate?.id), `advisories: crates.io answered for ${show(crate?.id)}, which was not asked`)
-      found.set(crate.id, githubRepoOfUrl(crate.repository) ?? null)
-    }
-  }
-  return found
-}
-
 async function fetchEach(names, fetch) {
   const found = await pool(names, PACKAGES_AT_ONCE, fetch)
   return new Map(names.map((name, i) => [name, found[i]]))
@@ -61,6 +34,15 @@ async function fetchComposerRepo(name) {
   return githubRepoOfUrl(versions[0]?.source?.url) ?? null
 }
 
-export const crateRepos = (names) => cachedRepos('cargo/repos', names, fetchCrates)
+// Soldeer answers a project it does not have with an empty list.
+async function fetchSoldeerRepo(name) {
+  const answer = await request(buildUrl(SOLDEER_API, ['api', 'v1', 'project'], { project_name: name }), { as: 'json' })
+  assert.ok(Array.isArray(answer?.data), `advisories: expected a list of projects from Soldeer for ${name}`)
+  for (const project of answer.data) assert.ok(project?.name === name, `advisories: Soldeer answered for ${show(project?.name)}, not ${name}`)
+  return githubRepoOfUrl(answer.data[0]?.github_url) ?? null
+}
+
+export const crateRepos = (names) => lookUpCrateRepos('advisories', names)
 export const composerRepos = (names) => cachedRepos('composer/repos', names, (missing) => fetchEach(missing, fetchComposerRepo))
 export const npmRepos = (names) => fetchEach(names, async (name) => (await lookUpPackageRepo(name))?.github ?? null)
+export const soldeerRepos = (names) => cachedRepos('soldeer/repos', names, (missing) => fetchEach(missing, fetchSoldeerRepo))

package/src/args.js

@@ -15,7 +15,9 @@ const isLogin = matches(/^(?=.{1,39}$)[a-z\d](?:-?[a-z\d])*$/iu)
 const isRepoName = matches(/^(?!\.\.?$)[\w.-]{1,100}$/u)
 // `.git` is never in a repo's tree, and no commit should write into it.
 export const isRepoPath = (value) => isString(value, 4096) && value.split('/').every((part) => !['', '.', '..', '.git'].includes(part.toLowerCase()))
-const BAD_REF = /^$|^@$|^-|[ ~^:?*[\\]|\.\.|@\{|^\/|\/$|\/\/|(?:^|\/)\.|\.lock(?:\/|$)|\.$/u // git check-ref-format, a rule per alternative
+// git check-ref-format, a rule per alternative. A tag's name may be `@` or
+// start with `-`; a branch's may not.
+const BAD_REF = /^$|[ ~^:?*[\\]|\.\.|@\{|^\/|\/$|\/\/|(?:^|\/)\.|\.lock(?:\/|$)|\.$/u
 const isToken = matches(/^[!-~]+$/u)
 // npm's rules for existing names: capitals allowed (JSONStream), the
 // legacy `~'!()*` not.
@@ -37,8 +39,10 @@ export function isRepo(value) {
 }
 
 export const isStrings = (value) => Array.isArray(value) && value.every((item) => typeof item === 'string')
+export const isIterable = (value) => typeof value?.[Symbol.iterator] === 'function' && typeof value !== 'string'
 export const sameName = (a, b) => typeof a === 'string' && a.toLowerCase() === b.toLowerCase()
-export const isRefName = (value) => isString(value, 255) && !BAD_REF.test(value)
+export const isTagName = (value) => isString(value, 255) && !BAD_REF.test(value)
+export const isRefName = (value) => isTagName(value) && !/^(?:@$|-)/u.test(value)
 export const isSha = matches(/^(?:[\da-f]{40}|[\da-f]{64})$/u)
 export const isGhsa = matches(/^GHSA(?:-[\da-hj-km-np-tv-z]{4}){3}$/u)
 export const assertion = (must, predicate) => (method, what, value) => assert.ok(predicate(value), `${method}: ${what} must be ${must}, got ${show(value)}`)
@@ -52,17 +56,33 @@ export function assertArgs(method, options, spec, name) {
   for (const [key, check] of Object.entries(spec)) check?.(method, label(key), options[key])
 }
 
+// The names a resolve*Repos call is given, each checked and given once,
+// and its options.
+export function assertNames(method, what, names, options, assertName) {
+  assert.ok(isIterable(names), `${method}: ${what} must be an iterable of names`)
+  assertArgs(method, options, { cachedOnly: optional(assertBoolean) })
+  const unique = [...new Set(names)]
+  for (const name of unique) assertName(method, 'name', name)
+  return unique
+}
+
 export const assertBoolean = assertion('a boolean', (value) => typeof value === 'boolean')
 export const assertNumber = assertion('a positive integer', (value) => Number.isSafeInteger(value) && value > 0)
 export const assertLine = assertion('a non-empty single line', (value) => isString(value, 1024) && value.trim() !== '')
 // 65,536 is GitHub's own limit on a body.
 export const assertText = assertion('text with no control characters', (value) => isString(value, 65_536, '\t\n\r'))
 export const assertDirectoryPath = assertion('a directory path', (value) => isString(value, 4096) && value !== '')
+// A short ASCII name, which no filesystem refuses, reads as another or takes
+// for a device: none of those is six characters of these alone.
+export const assertCacheName = assertion('6 to 32 of a-z, A-Z, 0-9, _ and -', matches(/^[a-zA-Z0-9_-]{6,32}$/u))
 export const assertLogin = assertion('a GitHub login', isLogin)
 export const assertRepoName = assertion('a repository name', isRepoName)
 export const assertRepo = assertion('"owner/name"', isRepo)
 export const assertRef = assertion('a branch or tag name', isRefName)
+export const assertTagName = assertion('a tag name', isTagName)
 export const assertSha = assertion('a full commit sha', isSha)
+export const isSha1 = matches(/^[\da-f]{40}$/u)
+export const assertTreeId = assertion('a full tree id', isSha1)
 export const assertPath = assertion('a path inside a repository', isRepoPath)
 export const assertToken = assertion('a token', isToken)
 export const assertTokenOrNull = assertion('a token, or null for anonymous access', (value) => value === null || isToken(value))
@@ -72,5 +92,19 @@ export const assertPackageVersion = assertion('an exact semver version', isExact
 export const assertGhsa = assertion('a GHSA id', isGhsa)
 export const assertCrateName = assertion('a crate name', matches(/^[A-Za-z][\w-]{0,63}$/u))
 export const assertCrateVersion = assertion('a semver version', matches(/^(?=.{5,256}$)(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:0|[1-9]\d*|\d*[A-Za-z-][\dA-Za-z-]*)(?:\.(?:0|[1-9]\d*|\d*[A-Za-z-][\dA-Za-z-]*))*)?(?:\+[\dA-Za-z-]+(?:\.[\dA-Za-z-]+)*)?$/u))
+// Soldeer's own rule for a name: 3 to 100 of lowercase letters, digits and
+// `-`, starting with one of them or `@`, ending with a letter or digit.
+export const assertSoldeerName = assertion('a Soldeer package name', matches(/^(?=.{3,100}$)[@\da-z][\da-z-]*[\da-z]$/u))
+// Soldeer takes any version but an empty one. Its registry has semver
+// (`5.7.0-rc.0`, `1.0.2-solc-0.8-simulate`), bare numbers and commit hashes.
+export const assertSoldeerVersion = assertion('letters, digits, `.`, `_`, `+` and `-`', matches(/^(?=.{1,128}$)[\dA-Za-z][\w.+-]*$/u))
+// A package's name and version, as a registry's getters take them.
+const assertNameVersion = (assertName, assertVersion) => (method, name, version) => {
+  assertName(method, 'name', name)
+  assertVersion(method, 'version', version)
+}
+export const assertCrate = assertNameVersion(assertCrateName, assertCrateVersion)
+export const assertPackage = assertNameVersion(assertPackageName, assertPackageVersion)
+export const assertSoldeerPackage = assertNameVersion(assertSoldeerName, assertSoldeerVersion)
 // As Cargo.lock and soldeer.lock write a checksum.
 export const assertSha256 = assertion('a sha256 in lowercase hex', matches(/^[\da-f]{64}$/u))