@preventive/upstream 1.0.1 → 1.1.0

package/advisories.d.ts

@@ -50,6 +50,15 @@ export interface AdvisoryOptions {
   // and its rows take OSV's record of their GHSA, one more request each
   // (no GitHub client needed).
   details?: boolean
+  // Also asks OSV of each npm package's versions, as cargo and composer
+  // packages always are, for what the registry does not have: above all
+  // the malicious packages' records (MAL-...), which it has none of, as
+  // those of npm releases taken down that GitHub's database has no
+  // advisory of. A version a registry row reports under the same advisory,
+  // its id among an OSV record's id and aliases, is left to that row,
+  // which takes the record's ids among its aliases. Sends each npm
+  // package's name and versions to OSV, so not for a private package.
+  osv?: boolean
 }
 
 // One advisory on one package, with the versions asked about that it
@@ -58,11 +67,13 @@ export interface Advisory {
   ecosystem: Ecosystem
   name: string
   // `registry`: npm's, as `npm audit` has it, one row per vulnerable
-  // range. `osv`: OSV's records for cargo and composer, RustSec's, GitHub's
-  // and malicious packages' (MAL-...) among them; of the records naming each
-  // other as aliases, directly or through others, a version is reported
-  // under one only: RustSec's, else the GHSA, else the first by id. Each
-  // has the others' ids among its aliases.
+  // range. `osv`: OSV's records for cargo and composer, and with `osv` for
+  // npm, the versions they cover that no registry row reports under the
+  // same advisory; RustSec's, GitHub's and malicious packages' (MAL-...)
+  // among them; of the records naming each other as aliases, directly or
+  // through others, a version is reported under one only: RustSec's, else
+  // the GHSA, else the first by id. Each has the others' ids among its
+  // aliases.
   // `repository`: published on the package's repository, one row per
   // range, holding only the versions the others' answer does not report
   // under that GHSA. The only source for `soldeer` and `github`, where
@@ -81,6 +92,8 @@ export interface Advisory {
   // (https://osv.dev/vulnerability/RUSTSEC-...). Built, never fetched: only
   // a registry row without a GHSA (npm:<id>) has none.
   url?: string
+  // The advisory's other ids: an `osv` row's record's own and its group's,
+  // and with `osv`, a registry row's OSV records' of it.
   aliases: string[]
   title?: string
   // With `details`, the advisory's full text, Markdown as published: an
@@ -110,14 +123,18 @@ export interface Advisory {
 // merged, and must not name two repositories. npm is asked 250 names a
 // request, OSV 1000 versions, Soldeer one project, and with `details`, OSV
 // one GHSA among npm's rows; a project Soldeer does not have or that names
-// no GitHub repository, and a repository gone, renamed or blocked, add
-// nothing, and a GHSA OSV does not have adds no `details`. Any other
-// failure, a malformed answer, or one about something not asked, throws:
-// nothing is left out quietly.
+// no GitHub repository, a repository gone or blocked, and one GitHub
+// redirects anywhere but to a repository by its id, add nothing, and a
+// GHSA OSV does not have adds no `details`. A repository renamed or
+// transferred, which GitHub redirects to it by its id, is listed by the
+// name it has now, as getRepo's `followRedirect` (github.js) finds it, its
+// rows' `url` on it. Any other failure, a malformed answer, or one about
+// something not asked, throws: nothing is left out quietly.
 // A repository's listing is kept for 90 minutes, as one entry for every
-// spelling of its name, holding only what rows are made from, `details`
-// included: in setCacheDir's cache gzipped, in a `cache` store as it is,
-// or with `cache` false, nowhere. 90 minutes is how late an advisory its
+// spelling of the name it is asked by, one renamed or transferred too,
+// holding only what rows are made from, `details` included: in
+// setCacheDir's cache gzipped, in a `cache` store as it is, or with
+// `cache` false, nowhere. 90 minutes is how late an advisory its
 // maintainer publishes can be seen. One gone or malformed is not kept.
 // Sorted by ecosystem and name. Versions are matched by npm's semver, from
 // the npm beside node, or by the semver peer where there is no npm.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@preventive/upstream",
-  "version": "1.0.1",
-  "description": "Minimal npm, crates.io and Soldeer registry clients, a GitHub client, and a git checkout reader: the calls we use, nothing more",
+  "version": "1.1.0",
+  "description": "Minimal npm, crates.io, Soldeer, GitHub and git checkout clients",
   "type": "module",
   "exports": {
     "./advisories.js": {

package/src/advisories/github.js

@@ -1,9 +1,9 @@
 import assert from 'node:assert/strict'
 
-import { assertRepo, assertSoldeerName, assertSoldeerVersion, assertion, isGhsa, isRefName, isStrings } from '../args.js'
+import { assertRepo, assertSoldeerName, assertSoldeerVersion, assertion, isGhsa, isRefName, isRepo, isStrings } from '../args.js'
 import { readRecord, writeRecord } from '../cache.js'
 import { isGone } from '../github/client.js'
-import { recover } from '../http.js'
+import { HttpError, recover } from '../http.js'
 import { pool } from '../pool.js'
 import { isExactVersion } from '../semver.js'
 import { advisoryUrl, covered, detailsOf, isText, mayBeInRange, metrics } from './common.js'
@@ -21,7 +21,7 @@ const isRepoAdvisory = (advisory) => advisory && typeof advisory === 'object' &&
   && advisory.state === 'published' && isText(advisory.summary)
   && (advisory.vulnerabilities == null || Array.isArray(advisory.vulnerabilities)) && (advisory.cwe_ids == null || isStrings(advisory.cwe_ids))
 
-export const assertClient = assertion('a GitHub client from createClient', (value) => typeof value?.listRepoAdvisories === 'function')
+export const assertClient = assertion('a GitHub client from createClient', (value) => typeof value?.listRepoAdvisories === 'function' && typeof value.getRepo === 'function')
 
 // A maintainer may give a vulnerable range only its lower bound, and the
 // version that fixed it apart (`>= 5.0.0-beta.1`, patched in `5.0.0-rc.2`),
@@ -67,18 +67,37 @@ const isDigest = (advisory) => advisory && typeof advisory === 'object' && isGhs
   && (advisory.description === undefined || isText(advisory.description))
   && Array.isArray(advisory.ranges) && advisory.ranges.every((entry) => typeof entry?.range === 'string')
 
-// A repository's listing, digested, through the cache: GitHub's names are
-// case-insensitive, so one entry answers every spelling. A repository gone,
-// renamed or blocked has none, and that is not kept.
+// GitHub answers for a repository renamed or transferred with a redirect,
+// which getRepo follows to the repository it is now, listed by that name.
+const isRedirect = (err) => err instanceof HttpError && err.status >= 300 && err.status < 400
+
+async function listingOf(github, repo) {
+  try {
+    return { repo, list: await github.listRepoAdvisories({ repo }) }
+  } catch (err) {
+    if (!isRedirect(err)) throw err
+  }
+  const moved = (await github.getRepo({ repo, followRedirect: true })).full_name
+  return { repo: moved, list: await github.listRepoAdvisories({ repo: moved }) }
+}
+
+// A repository's listing, digested, through the cache, and the repository
+// it is of: GitHub's names are case-insensitive, so one entry answers every
+// spelling, and one renamed or transferred is kept under the name asked,
+// with `repo` the one it has now. A repository gone or blocked, or
+// redirected anywhere getRepo does not follow, has none, and that is not
+// kept.
 async function listAdvisories(github, repo, cache) {
   const name = repo.toLowerCase()
   const entry = await readRecord(DIR, name, cache, LISTING_TTL_MS)
-  if (entry?.v === VERSION && Array.isArray(entry.advisories) && entry.advisories.every(isDigest)) return entry.advisories
-  const list = await github.listRepoAdvisories({ repo }).catch(recover(isGone, null))
-  if (list === null) return []
-  const advisories = digest(repo, list)
-  if (advisories.every(isDigest)) await writeRecord(DIR, name, { v: VERSION, advisories }, cache)
-  return advisories
+  if (entry?.v === VERSION && (entry.repo === undefined || isRepo(entry.repo)) && Array.isArray(entry.advisories) && entry.advisories.every(isDigest)) {
+    return { repo: entry.repo ?? repo, advisories: entry.advisories }
+  }
+  const found = await listingOf(github, repo).catch(recover((err) => isGone(err) || isRedirect(err), null))
+  if (found === null) return { repo, advisories: [] }
+  const advisories = digest(found.repo, found.list)
+  if (advisories.every(isDigest)) await writeRecord(DIR, name, { v: VERSION, ...(found.repo !== repo && { repo: found.repo }), advisories }, cache)
+  return { repo: found.repo, advisories }
 }
 
 // GitHub's `>= 1.0.0, < 1.2.6` is npm's with the commas dropped.
@@ -109,10 +128,10 @@ function fromRepository(repo, name, advisory, range, asked, { covers, details })
 async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers, details, cache }) {
   // GitHub's names are case-insensitive: one spelling asks for all.
   const namesOf = Map.groupBy([...asked.keys()].filter(repoOf), (name) => repoOf(name).toLowerCase())
-  const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ repo: repoOf(names[0]), names, list: await listAdvisories(github, repoOf(names[0]), cache) }))
+  const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ names, ...(await listAdvisories(github, repoOf(names[0]), cache)) }))
   const rows = new Map()
-  for (const { repo, names, list } of listed) {
-    for (const advisory of list) {
+  for (const { repo, names, advisories } of listed) {
+    for (const advisory of advisories) {
       for (const { range, ...pkg } of advisory.ranges) {
         for (const name of names.filter((candidate) => takes(candidate, pkg))) {
           rows.set(`${name} ${advisory.ghsa} ${range}`, fromRepository(repo, name, advisory, range, asked.get(name), { covers, details }))

package/src/advisories/index.js

@@ -35,7 +35,7 @@ function collect(packages) {
 }
 
 export async function advisories(packages, options = {}) {
-  assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean), cache: optional(assertCache) })
+  assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean), osv: optional(assertBoolean), cache: optional(assertCache) })
   assert.ok(!options.repoAdvisories || options.github, 'advisories: repoAdvisories needs a github client')
   const byEcosystem = collect(packages)
   for (const ecosystem of byEcosystem.keys()) assert.ok(!ECOSYSTEMS[ecosystem].repositoryOnly || options.github, `advisories: ${ecosystem} packages need a github client`)
@@ -44,7 +44,7 @@ export async function advisories(packages, options = {}) {
     const names = [...named.keys()].toSorted()
     const asked = new Map(names.map((name) => [name, [...named.get(name).versions].toSorted(compare)]))
     const known = new Map(names.filter((name) => named.get(name).github).map((name) => [name, named.get(name).github]))
-    const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, known, cache: options.cache })
+    const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, osv: options.osv === true, known, cache: options.cache })
     return rows.map((row) => ({ ecosystem, ...row }))
   }))
   return found.flat().filter((row) => row.versions.length > 0).toSorted((a, b) => order(a.ecosystem, b.ecosystem) || order(a.name, b.name))

package/src/advisories/npm.js

@@ -6,7 +6,7 @@ import { chunks, pool } from '../pool.js'
 import { compareVersions } from '../semver.js'
 import { advisoryUrl, covered, detailsOf, isText, metrics } from './common.js'
 import { withRepositories } from './github.js'
-import { RECORDS_AT_ONCE, getVuln } from './osv.js'
+import { RECORDS_AT_ONCE, getVuln, osvRows } from './osv.js'
 import { npmRepos } from './repos.js'
 
 const BULK_URL = buildUrl(NPM_REGISTRY, ['-', 'npm', 'v1', 'security', 'advisories', 'bulk'])
@@ -52,21 +52,48 @@ async function registryAdvisories(asked) {
 
 // The registry's rows carry no text: each GHSA's is OSV's record of it,
 // GitHub's advisory database as published, asked once for the rows that
-// cover an asked version. One OSV does not have yet has none.
+// cover an asked version, but where `osv` already read it. One OSV does
+// not have yet has none.
 async function withDetails(rows) {
-  const ghsas = [...new Set(rows.filter((row) => row.ghsa && row.versions.length > 0).map((row) => row.ghsa))]
+  const asks = (row) => row.source === 'registry' && row.details === undefined
+  const ghsas = [...new Set(rows.filter((row) => asks(row) && row.ghsa && row.versions.length > 0).map((row) => row.ghsa))]
   const records = await pool(ghsas, RECORDS_AT_ONCE, (ghsa) => getVuln(ghsa).catch(recover(isNotFound, undefined)))
   const text = new Map(ghsas.map((ghsa, i) => [ghsa, records[i]?.details]))
-  return rows.map((row) => ({ ...row, ...detailsOf(text.get(row.ghsa), row.ghsa) }))
+  return rows.map((row) => (asks(row) ? { ...row, ...detailsOf(text.get(row.ghsa), row.ghsa) } : row))
 }
 
-// What `npm audit` asks the registry, one row per vulnerable range.
+// With `osv`, OSV's records of the asked versions besides the registry's
+// rows, the malicious packages' (MAL-...), which the registry has none
+// of, among them. A version a registry row reports under the same
+// advisory, the row's id one of an OSV row's ids or aliases, is left to
+// that row, which takes the OSV row's ids among its aliases, and its text
+// where it is the same record; an OSV row keeps the versions no such row
+// reports.
+function withOsv(registry, osv) {
+  const rows = registry.map((row) => ({ ...row }))
+  const added = []
+  for (const record of osv) {
+    const ids = [record.id, ...record.aliases]
+    const same = rows.filter((row) => row.name === record.name && ids.includes(row.id) && row.versions.some((version) => record.versions.includes(version)))
+    for (const row of same) {
+      row.aliases = [...new Set([...row.aliases, ...ids])].filter((id) => id !== row.id)
+      if (record.id === row.id && record.details !== undefined) row.details = record.details
+    }
+    const versions = record.versions.filter((version) => !same.some((row) => row.versions.includes(version)))
+    if (versions.length > 0) added.push({ ...record, versions })
+  }
+  return [...rows, ...added]
+}
+
+// What `npm audit` asks the registry, one row per vulnerable range, and
+// with `osv`, what OSV has besides.
 export const NPM = {
   assertName: assertPackageName,
   assertVersion: assertPackageVersion,
   compare: compareVersions,
   advisories: async (asked, options) => {
-    const rows = await registryAdvisories(asked)
+    const [registry, osv] = await Promise.all([registryAdvisories(asked), options.osv ? osvRows({ osv: 'npm' }, asked, options) : []])
+    const rows = withOsv(registry, osv)
     return await withRepositories(options.details ? await withDetails(rows) : rows, asked, options, { ecosystem: 'npm', lookUp: npmRepos })
   },
 }

package/src/advisories/osv.js

@@ -84,9 +84,9 @@ function toAdvisory(ecosystem, name, versions, record, details) {
 // publishes comes back under each one's id, the records naming each other
 // as aliases, either way round, directly or through others: one group, in
 // which each version is reported under the first answered for it, and
-// each record has the others' ids as aliases.
-async function osvAdvisories(ecosystem, asked, options) {
-  const { osv, github, lookUp, covers } = ecosystem
+// each record has the others' ids as aliases. `osv` is OSV's name for the
+// ecosystem.
+export async function osvRows({ osv }, asked, { details }) {
   const list = [...asked].flatMap(([name, versions]) => versions.map((version) => ({ name, version })))
   const hits = new Map() // id -> name -> versions, in `list` order
   for (const chunk of chunks(list, QUERIES_PER_REQUEST)) {
@@ -118,9 +118,14 @@ async function osvAdvisories(ecosystem, asked, options) {
     const aliases = [...new Set([...record.aliases, ...group.map((other) => other.id)])].filter((id) => id !== record.id)
     for (const [name, versions] of hits.get(record.id)) {
       const rest = [...versions].filter((version) => !above.some((other) => hits.get(other.id).get(name)?.has(version)))
-      if (rest.length > 0) rows.push(toAdvisory(osv, name, rest, { ...record, aliases }, options.details))
+      if (rest.length > 0) rows.push(toAdvisory(osv, name, rest, { ...record, aliases }, details))
     }
   }
   rows.sort((a, b) => order(a.name, b.name) || order(a.id, b.id))
-  return await withRepositories(rows, asked, options, { ecosystem: github, lookUp, covers })
+  return rows
+}
+
+async function osvAdvisories(ecosystem, asked, options) {
+  const { github, lookUp, covers } = ecosystem
+  return await withRepositories(await osvRows(ecosystem, asked, options), asked, options, { ecosystem: github, lookUp, covers })
 }