@preventive/upstream 1.1.0 → 1.2.0

package/advisories.d.ts

@@ -56,28 +56,37 @@ export interface AdvisoryOptions {
   // those of npm releases taken down that GitHub's database has no
   // advisory of. A version a registry row reports under the same advisory,
   // its id among an OSV record's id and aliases, is left to that row,
-  // which takes the record's ids among its aliases. Sends each npm
-  // package's name and versions to OSV, so not for a private package.
+  // which takes the record's ids among its aliases; the versions a record
+  // of the row's own id covers beyond the registry's range join it too.
+  // Sends each npm package's name and versions to OSV, so not for a
+  // private package.
   osv?: boolean
 }
 
 // One advisory on one package, with the versions asked about that it
-// covers (never empty).
+// covers (never empty): one row an id, however many of its sources answer
+// it, or however often one does.
 export interface Advisory {
   ecosystem: Ecosystem
   name: string
-  // `registry`: npm's, as `npm audit` has it, one row per vulnerable
-  // range. `osv`: OSV's records for cargo and composer, and with `osv` for
-  // npm, the versions they cover that no registry row reports under the
-  // same advisory; RustSec's, GitHub's and malicious packages' (MAL-...)
-  // among them; of the records naming each other as aliases, directly or
-  // through others, a version is reported under one only: RustSec's, else
-  // the GHSA, else the first by id. Each has the others' ids among its
-  // aliases.
-  // `repository`: published on the package's repository, one row per
-  // range, holding only the versions the others' answer does not report
-  // under that GHSA. The only source for `soldeer` and `github`, where
-  // every range counts, whichever package it names.
+  // `registry`: npm's, as `npm audit` has it. `osv`: OSV's records for
+  // cargo and composer, and with `osv` for npm, the versions they cover
+  // that no registry row reports under the same advisory; RustSec's,
+  // GitHub's and malicious packages' (MAL-...) among them; of the records
+  // naming each other as aliases, directly or through others, a version is
+  // reported under one only: RustSec's, else the GHSA, else the first by
+  // id. Each has the others' ids among its aliases, and stays a row of its
+  // own, as its source rates it.
+  // `repository`: published on the package's repository, holding only the
+  // versions the others' answer does not report under that GHSA. The only
+  // source for `soldeer` and `github`, where every range counts, whichever
+  // package it names.
+  // An id answered more than once -- by npm's registry once a range, by
+  // OSV's record of it or its repository for versions beyond the registry's
+  // range, by its repository once a range -- is one row: the registry's,
+  // else OSV's, else the repository's, with what it lacks of theirs (title,
+  // details), the worst severity of them, the highest CVSS score with its
+  // own vector, and all their ranges, CWEs and versions.
   source: 'registry' | 'osv' | 'repository'
   // A GHSA, RUSTSEC-..., MAL-..., DRUPAL-CORE-... or another OSV id, or
   // npm:<id> for a registry row without a GHSA.
@@ -109,11 +118,17 @@ export interface Advisory {
   cvssVector?: string
   cwe: string[]
   // As written: npm's syntax from the registry, GitHub's
-  // (`>= 1.0.0, < 1.2.6`) from a repository; OSV rows have none. A
+  // (`>= 1.0.0, < 1.2.6`) from a repository; OSV's records have none. A
   // repository range semver cannot read, or none, covers every version.
+  // Each range its id was answered with that covers an asked version,
+  // joined by ` || `; none ('') where one is none. A version only OSV
+  // reports may be outside it: hermes-engine's GHSA-f5x2-xv93-4p23 is
+  // `<=0.4.3` on npm, the last version known affected, and fixed in 0.5.2
+  // by OSV's record of it, which reports 0.5.0 too.
   range?: string
   // RustSec's kind for an advisory that is not a vulnerability as such:
-  // `unmaintained`, `unsound`, `notice`.
+  // `unmaintained`, `unsound`, `notice`. For an id answered more than once,
+  // only where each answer gives it.
   informational?: string
   versions: string[]
 }

package/github.d.ts

@@ -7,7 +7,7 @@
 // encoded segments and has to come back out of URL parsing unchanged.
 // Redirects are refused, so an answer is about the repo asked for, except
 // the tarballs' and, where asked, getRepo's for a repo renamed or
-// transferred. A response is read up to a size limit (64 MiB of JSON, 128
+// transferred. A response is read up to a size limit (128 MiB of JSON, 128
 // MiB of a file, 512 MiB of a tarball) and within a timeout (30 seconds, 5
 // minutes for a tarball), and what it says goes into an error message
 // escaped.

package/npm.d.ts

@@ -127,7 +127,11 @@ export interface Dist {
 // `bugs` an object, of which only the strings named below are read. Each is
 // absent where the document has none, or has another shape, or an object
 // with none of those strings. All of it is the publisher's word, held to no
-// repository.
+// repository. `unpackedSize` is the document's dist.unpackedSize: the bytes
+// of the files in the tarball, as the registry counted them when the
+// version was published, held to no tarball. It is absent where the
+// document has none, as for a version published before the registry
+// counted them, or has one that is not a whole number of bytes.
 export interface PackageMeta {
   name: string
   version: string
@@ -136,6 +140,7 @@ export interface PackageMeta {
   repository?: string | PackageRepository
   homepage?: string
   bugs?: string | PackageBugs
+  unpackedSize?: number
 }
 
 export interface PackageRepository {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@preventive/upstream",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Minimal npm, crates.io, Soldeer, GitHub and git checkout clients",
   "type": "module",
   "exports": {

package/src/advisories/index.js

@@ -34,6 +34,57 @@ function collect(packages) {
   return byEcosystem
 }
 
+// Which of an advisory's rows stands for it: npm's registry's, then OSV's,
+// then its repository's.
+const SOURCES = ['registry', 'osv', 'repository']
+const SEVERITIES = ['critical', 'high', 'moderate', 'low', 'info']
+const worse = (a, b) => (b === undefined || (a !== undefined && SEVERITIES.indexOf(a) < SEVERITIES.indexOf(b)) ? a : b)
+
+// An advisory's rows as one: the first's by source, what it lacks from the
+// first of the rest that has it, the others' aliases, and of them all the
+// worst severity, the highest CVSS score with its own vector (else the first
+// vector), the ranges (` || `), CWEs and versions (in the order `asked` has
+// them). RustSec's kind of advisory that is not a vulnerability as such only
+// where each of them has that kind: a row without one is a vulnerability.
+function joinRows(rows, asked) {
+  const [head, ...rest] = rows.toSorted((a, b) => SOURCES.indexOf(a.source) - SOURCES.indexOf(b.source))
+  const all = [head, ...rest]
+  const ranges = [...new Set(all.map((row) => row.range).filter((range) => range !== undefined))]
+  const versions = new Set(all.flatMap((row) => row.versions))
+  const joined = {
+    ...head,
+    aliases: [...new Set(all.flatMap((row) => row.aliases))].filter((id) => id !== head.id),
+    cwe: [...new Set(all.flatMap((row) => row.cwe))],
+    versions: asked.filter((version) => versions.has(version)),
+  }
+  if (rest.some((row) => row.informational !== head.informational)) delete joined.informational
+  for (const key of ['title', 'details']) {
+    const from = joined[key] === undefined && rest.find((row) => row[key] !== undefined)
+    if (from) joined[key] = from[key]
+  }
+  const severity = all.map((row) => row.severity).reduce(worse)
+  if (severity !== undefined) joined.severity = severity
+  const scored = all.filter((row) => row.cvss !== undefined).toSorted((a, b) => b.cvss - a.cvss)[0] ?? all.find((row) => row.cvssVector !== undefined)
+  delete joined.cvss
+  delete joined.cvssVector
+  if (scored?.cvss !== undefined) joined.cvss = scored.cvss
+  if (scored?.cvssVector !== undefined) joined.cvssVector = scored.cvssVector
+  // A range of none ('') covers every version, as they then do together.
+  if (ranges.length > 0) joined.range = ranges.includes('') ? '' : ranges.join(' || ')
+  return joined
+}
+
+// One row for each advisory id on a package, where its sources answer more
+// than one: npm's registry one per range, and OSV's record of it, or its
+// repository, another for the versions they cover beyond that. Records of
+// other ids that name it as an alias stay rows of their own, each version
+// reported under one, as their sources rate them. Rows keep the place of the
+// first answered.
+function oneRowPerAdvisory(rows, asked) {
+  const byId = Map.groupBy(rows, (row) => `${row.name}\0${row.id}`)
+  return [...byId.values()].map((same) => (same.length === 1 ? same[0] : joinRows(same, asked.get(same[0].name))))
+}
+
 export async function advisories(packages, options = {}) {
   assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean), osv: optional(assertBoolean), cache: optional(assertCache) })
   assert.ok(!options.repoAdvisories || options.github, 'advisories: repoAdvisories needs a github client')
@@ -45,7 +96,7 @@ export async function advisories(packages, options = {}) {
     const asked = new Map(names.map((name) => [name, [...named.get(name).versions].toSorted(compare)]))
     const known = new Map(names.filter((name) => named.get(name).github).map((name) => [name, named.get(name).github]))
     const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, osv: options.osv === true, known, cache: options.cache })
-    return rows.map((row) => ({ ecosystem, ...row }))
+    return oneRowPerAdvisory(rows.filter((row) => row.versions.length > 0), asked).map((row) => ({ ecosystem, ...row }))
   }))
-  return found.flat().filter((row) => row.versions.length > 0).toSorted((a, b) => order(a.ecosystem, b.ecosystem) || order(a.name, b.name))
+  return found.flat().toSorted((a, b) => order(a.ecosystem, b.ecosystem) || order(a.name, b.name))
 }

package/src/advisories/npm.js

@@ -68,7 +68,7 @@ async function withDetails(rows) {
 // advisory, the row's id one of an OSV row's ids or aliases, is left to
 // that row, which takes the OSV row's ids among its aliases, and its text
 // where it is the same record; an OSV row keeps the versions no such row
-// reports.
+// reports, which advisories() joins to a registry row of its own id.
 function withOsv(registry, osv) {
   const rows = registry.map((row) => ({ ...row }))
   const added = []

package/src/http.js

@@ -26,7 +26,7 @@ const isQuery = ([key, value]) => isQueryKey(key) && (key.endsWith('[]') ? Array
 // A line break in a value would start a header of its own.
 const isHeader = ([name, value]) => /^[A-Za-z][\w-]*$/u.test(name) && matches(/^[ -~]*$/u)(value)
 const LIMITS = {
-  json: { bytes: 64 * 1024 * 1024, ms: 30_000 },
+  json: { bytes: 128 * 1024 * 1024, ms: 30_000 },
   text: { bytes: 128 * 1024 * 1024, ms: 30_000 },
   bytes: { bytes: MAX_BYTES, ms: 300_000 },
 }

package/src/npm/tarball.js

@@ -73,24 +73,28 @@ async function npmCachedVersion(name, version, given) {
   return undefined
 }
 
+const countOr = (value) => (Number.isSafeInteger(value) && value >= 0 ? value : undefined)
+
 // The version's document: with the `dist` the caller has, npm's own cache's
 // where it matches (npmCachedVersion), and neither `cache` nor the cache
 // set is read or written; else as getVersion has it. Beside its dist, the
 // document's gitHead where that is a full commit id, and its repository,
-// homepage and bugs, in the shapes a package.json gives them. Each but
-// dist is left out where there is none.
+// homepage and bugs, in the shapes a package.json gives them; and its
+// dist's unpackedSize where that is a count. Each but dist is left out
+// where there is none.
 export async function getMeta(name, version, options = {}) {
   assertPackage('getMeta', name, version)
   assertArgs('getMeta', options, { dist: null, cache: optional(assertCache) })
   const given = options.dist === undefined ? undefined : checkedDist('getMeta', name, version, options.dist)
   const { dist, json } = (given && await npmCachedVersion(name, version, given)) ?? await getVersion('getMeta', name, version, given, options.cache)
-  const about = {
+  const said = {
     gitHead: isSha(own(json, 'gitHead')) ? json.gitHead : undefined,
     repository: linkOf(own(json, 'repository'), ['type', 'url', 'directory']),
     homepage: stringOr(own(json, 'homepage')),
     bugs: linkOf(own(json, 'bugs'), ['url', 'email']),
+    unpackedSize: countOr(own(json.dist, 'unpackedSize')),
   }
-  return { name, version, dist, ...Object.fromEntries(Object.entries(about).filter(([, value]) => value !== undefined)) }
+  return { name, version, dist, ...Object.fromEntries(Object.entries(said).filter(([, value]) => value !== undefined)) }
 }
 
 // Only the whole package's document has `time`: megabytes for some.