@preventive/upstream 1.0.0-alpha.0 → 1.0.0-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,13 +1,17 @@
1
1
  {
2
2
  "name": "@preventive/upstream",
3
- "version": "1.0.0-alpha.0",
4
- "description": "Minimal npm registry and GitHub clients, and a git checkout reader: the calls we use, nothing more",
3
+ "version": "1.0.0-alpha.10",
4
+ "description": "Minimal npm, crates.io and Soldeer registry clients, a GitHub client, and a git checkout reader: the calls we use, nothing more",
5
5
  "type": "module",
6
6
  "exports": {
7
7
  "./advisories.js": {
8
8
  "types": "./advisories.d.ts",
9
9
  "default": "./advisories.js"
10
10
  },
11
+ "./cargo.js": {
12
+ "types": "./cargo.d.ts",
13
+ "default": "./cargo.js"
14
+ },
11
15
  "./github.js": {
12
16
  "types": "./github.d.ts",
13
17
  "default": "./github.js"
@@ -31,11 +35,23 @@
31
35
  "./semver.js": {
32
36
  "types": "./semver.d.ts",
33
37
  "default": "./semver.js"
38
+ },
39
+ "./soldeer.js": {
40
+ "types": "./soldeer.d.ts",
41
+ "default": "./soldeer.js"
42
+ },
43
+ "./weak-edges.js": {
44
+ "types": "./weak-edges.d.ts",
45
+ "default": "./weak-edges.js"
34
46
  }
35
47
  },
36
48
  "files": [
49
+ "LICENSE-APACHE",
50
+ "LICENSE-MIT",
37
51
  "advisories.d.ts",
38
52
  "advisories.js",
53
+ "cargo.d.ts",
54
+ "cargo.js",
39
55
  "github.d.ts",
40
56
  "github.js",
41
57
  "git.d.ts",
@@ -48,6 +64,10 @@
48
64
  "package.js",
49
65
  "semver.d.ts",
50
66
  "semver.js",
67
+ "soldeer.d.ts",
68
+ "soldeer.js",
69
+ "weak-edges.d.ts",
70
+ "weak-edges.js",
51
71
  "src/advisories/common.js",
52
72
  "src/advisories/github.js",
53
73
  "src/advisories/index.js",
@@ -55,18 +75,27 @@
55
75
  "src/advisories/osv.js",
56
76
  "src/advisories/repos.js",
57
77
  "src/args.js",
78
+ "src/attributes.js",
58
79
  "src/cache.js",
80
+ "src/cargo/crate.js",
81
+ "src/cargo/repos.js",
82
+ "src/download.js",
59
83
  "src/git.js",
60
84
  "src/github/client.js",
61
85
  "src/github/read.js",
62
86
  "src/github/write.js",
63
87
  "src/http.js",
88
+ "src/npm/registry.js",
64
89
  "src/npm/repos.js",
65
90
  "src/npm/tarball.js",
66
91
  "src/package.js",
67
92
  "src/pool.js",
68
93
  "src/remote.js",
69
- "src/semver.js"
94
+ "src/semver-peer.cjs",
95
+ "src/semver.js",
96
+ "src/tree.js",
97
+ "src/soldeer.js",
98
+ "src/weak-edges.js"
70
99
  ],
71
100
  "engines": {
72
101
  "node": ">=24.14.0"
@@ -74,7 +103,7 @@
74
103
  "scripts": {
75
104
  "test": "node --test ./tests/*.test.js"
76
105
  },
77
- "license": "MIT",
106
+ "license": "MIT OR Apache-2.0",
78
107
  "author": {
79
108
  "name": "Nikita Skovoroda",
80
109
  "email": "chalkerx@gmail.com",
@@ -91,5 +120,13 @@
91
120
  "homepage": "https://github.com/PreventiveMeasures/libraries/tree/main/upstream",
92
121
  "publishConfig": {
93
122
  "access": "public"
123
+ },
124
+ "peerDependencies": {
125
+ "semver": "^7.5.2"
126
+ },
127
+ "peerDependenciesMeta": {
128
+ "semver": {
129
+ "optional": true
130
+ }
94
131
  }
95
132
  }
package/semver.d.ts CHANGED
@@ -1,11 +1,21 @@
1
1
  // Hand-written against semver.js; a change to either belongs with the other.
2
2
 
3
3
  // npm's own semver, borrowed from the npm beside node and passed through
4
- // as it is. Without it, every call throws, except valid() on a plain
4
+ // as it is; where there is no npm, the `semver` package, an optional peer
5
+ // dependency. Without either, every call throws, except valid() on a plain
5
6
  // MAJOR.MINOR.PATCH with no options.
6
7
  export function satisfies(version: string, range: string, options?: { includePrerelease?: boolean; loose?: boolean }): boolean
7
- export function compareVersions(a: string, b: string): -1 | 0 | 1
8
+ export function compareVersions(a: string, b: string, options?: { loose?: boolean }): -1 | 0 | 1
8
9
  // The version as semver spells it, or null: `v1.2.3` answers `1.2.3`.
9
10
  export function valid(version: unknown, options?: { loose?: boolean }): string | null
10
11
  // A string that valid() answers unchanged.
11
12
  export function isExactVersion(version: unknown): version is string
13
+ // The range as semver normalizes it, or null where it is not one.
14
+ export function validRange(range: unknown, options?: { loose?: boolean; includePrerelease?: boolean }): string | null
15
+ // Whether some version is in both ranges; throws for one that is not a range.
16
+ export function intersects(a: string, b: string, options?: { loose?: boolean; includePrerelease?: boolean }): boolean
17
+ // The version as semver cleans it, spaces and a leading `=` or `v` dropped,
18
+ // or null: ` =v1.2.3 ` answers `1.2.3`.
19
+ export function clean(version: string, options?: { loose?: boolean } | boolean): string | null
20
+ // The version's major, as semver reads it; throws for one that is not a version.
21
+ export function major(version: string, options?: { loose?: boolean } | boolean): number
package/semver.js CHANGED
@@ -1 +1 @@
1
- export { compareVersions, isExactVersion, satisfies, valid } from './src/semver.js'
1
+ export { clean, compareVersions, intersects, isExactVersion, major, satisfies, valid, validRange } from './src/semver.js'
package/soldeer.d.ts ADDED
@@ -0,0 +1,17 @@
1
+ // Hand-written against soldeer.js; a change to either belongs with the
2
+ // other.
3
+ //
4
+ // The name is asserted to be one Soldeer takes, the version a string of
5
+ // letters, digits, `.`, `_`, `+` and `-`, and the checksum 64 lowercase hex
6
+ // digits, before any request or cache path. The cache is setCacheDir's
7
+ // (npm.js).
8
+
9
+ export { HttpError } from './npm.js'
10
+
11
+ // A version's zip from Soldeer's registry, whole, in memory. The registry
12
+ // says where the zip is but not what it hashes to, so `checksum` is
13
+ // required: the zip's sha256, as soldeer.lock records it. The bytes are
14
+ // checked against it, whether they were downloaded (before they are
15
+ // cached) or read from setCacheDir's cache, where one that does not match
16
+ // throws. A cached zip that matches is answered with no request at all.
17
+ export function getZip(name: string, version: string, checksum: string): Promise<Uint8Array>
package/soldeer.js ADDED
@@ -0,0 +1,2 @@
1
+ export { HttpError } from './src/http.js'
2
+ export { getZip } from './src/soldeer.js'
@@ -1,21 +1,43 @@
1
- import { isStrings, matches } from '../args.js'
2
- import { satisfies, validRange } from '../semver.js'
1
+ import assert from 'node:assert/strict'
2
+
3
+ import { isGhsa, isRepo, isStrings, matches, show } from '../args.js'
4
+ import { satisfies, valid, validRange } from '../semver.js'
3
5
 
4
6
  export const order = (a, b) => (a > b) - (a < b)
5
7
 
8
+ // GitHub's page for a GHSA: the advisory database's, which has every one
9
+ // the registry and OSV answer with, or the page of the `repo` that
10
+ // published it, which has it before GitHub reviews it into the database.
11
+ export function advisoryUrl(ghsa, repo) {
12
+ assert.ok(isGhsa(ghsa) && (repo === undefined || isRepo(repo)), `advisories: no GitHub advisory page for ${show(ghsa)} on ${show(repo)}`)
13
+ return repo === undefined ? `https://github.com/advisories/${ghsa}` : `https://github.com/${repo}/security/advisories/${ghsa}`
14
+ }
15
+
6
16
  // As `npm audit` matches: a prerelease is in a range around it.
7
17
  const SEMVER = { includePrerelease: true, loose: true }
8
18
 
9
19
  export const inRange = (version, range) => satisfies(version, range, SEMVER)
10
20
 
11
- // The asked versions a range `covers`. One semver cannot read covers them
12
- // all: a missed advisory is worse than a spare one.
13
- export const covered = (asked, range, covers = inRange) => (validRange(range, SEMVER) === null ? [...asked] : asked.filter((version) => covers(version, range)))
21
+ // A version semver cannot read may be in any range. Whatever it reads is
22
+ // a version, `v1.2.3` and `1.2.3+build` included.
23
+ export const mayBeInRange = (version, range) => valid(version) === null || inRange(version, range)
24
+
25
+ // The asked versions a range `covers`. One semver cannot read, range or
26
+ // version, covers them all: a missed advisory is worse than a spare one.
27
+ export const covered = (asked, range, covers = mayBeInRange) => (validRange(range, SEMVER) === null ? [...asked] : asked.filter((version) => covers(version, range)))
14
28
 
15
29
  // Remote text, as long as it is well-formed; anything else is refused as
16
30
  // malformed by whoever reads it.
17
31
  export const isText = (value) => typeof value === 'string' && value.isWellFormed()
18
32
 
33
+ // An advisory's full text, for `details`: none where its source has none,
34
+ // and refused where it is not well-formed.
35
+ export function detailsOf(text, id) {
36
+ if (text == null || text === '') return {}
37
+ assert.ok(isText(text), `advisories: malformed details in ${id}`)
38
+ return { details: text }
39
+ }
40
+
19
41
  const SEVERITIES = new Set(['critical', 'high', 'moderate', 'low', 'info'])
20
42
  const isCvssVector = matches(/^CVSS:[34]\.\d(?:\/[A-Z]{1,4}:[A-Z]{1,2})+$/u)
21
43
  const isCwe = matches(/^CWE-\d{1,6}$/u)
@@ -1,10 +1,11 @@
1
1
  import assert from 'node:assert/strict'
2
2
 
3
- import { assertRepo, assertion, isGhsa, isRefName, isStrings } from '../args.js'
3
+ import { assertRepo, assertSoldeerName, assertSoldeerVersion, assertion, isGhsa, isRefName, isStrings } from '../args.js'
4
4
  import { isGone } from '../github/client.js'
5
+ import { recover } from '../http.js'
5
6
  import { pool } from '../pool.js'
6
- import { valid } from '../semver.js'
7
- import { covered, inRange, isText, metrics } from './common.js'
7
+ import { advisoryUrl, covered, detailsOf, isText, mayBeInRange, metrics } from './common.js'
8
+ import { soldeerRepos } from './repos.js'
8
9
 
9
10
  const REPOS_AT_ONCE = 4
10
11
 
@@ -14,23 +15,24 @@ const isRepoAdvisory = (advisory) => advisory && typeof advisory === 'object' &&
14
15
 
15
16
  export const assertClient = assertion('a GitHub client from createClient', (value) => typeof value?.listRepoAdvisories === 'function')
16
17
 
17
- const listAdvisories = (github, repo) => github.listRepoAdvisories({ repo }).catch((err) => {
18
- if (isGone(err)) return []
19
- throw err
20
- })
18
+ const listAdvisories = (github, repo) => github.listRepoAdvisories({ repo }).catch(recover(isGone, []))
21
19
 
22
20
  // GitHub's `>= 1.0.0, < 1.2.6` is npm's with the commas dropped.
23
21
  // Maintainers write these unreviewed: one without a range, or with one
24
- // semver cannot read, covers every version.
25
- function fromRepository(name, advisory, range, asked, covers) {
22
+ // semver cannot read, covers every version. Its page is on `repo`, which
23
+ // may publish it long before the advisory database has one, and its
24
+ // `description` is the text `details` asks for: the listing has it all.
25
+ function fromRepository(repo, name, advisory, range, asked, { covers, details }) {
26
26
  const cvss = [advisory.cvss_severities?.cvss_v3, advisory.cvss_severities?.cvss_v4, advisory.cvss].find((entry) => typeof entry?.vector_string === 'string')
27
27
  return {
28
28
  name,
29
29
  source: 'repository',
30
30
  id: advisory.ghsa_id,
31
31
  ghsa: advisory.ghsa_id,
32
+ url: advisoryUrl(advisory.ghsa_id, repo),
32
33
  aliases: [],
33
34
  title: advisory.summary,
35
+ ...(details && detailsOf(advisory.description, advisory.ghsa_id)),
34
36
  ...metrics({ severity: advisory.severity, score: cvss?.score, vector: cvss?.vector_string, cwe: advisory.cwe_ids }),
35
37
  range,
36
38
  versions: covered(asked, range.replaceAll(',', ' '), covers),
@@ -40,8 +42,8 @@ function fromRepository(name, advisory, range, asked, covers) {
40
42
  // Each asked name's repository, `repoOf` it, is asked once, and its
41
43
  // advisories' vulnerable ranges become rows for the names that `takes` a
42
44
  // vulnerability, one per name, advisory and range, holding the asked
43
- // versions it `covers`.
44
- async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers }) {
45
+ // versions it `covers`, and with `details`, its text.
46
+ async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers, details }) {
45
47
  // GitHub's names are case-insensitive: one spelling asks for all.
46
48
  const namesOf = Map.groupBy([...asked.keys()].filter(repoOf), (name) => repoOf(name).toLowerCase())
47
49
  const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ repo: repoOf(names[0]), names, list: await listAdvisories(github, repoOf(names[0])) }))
@@ -53,7 +55,7 @@ async function repositoryAdvisories(github, asked, { repoOf, takes = () => true,
53
55
  const range = vulnerability?.vulnerable_version_range ?? ''
54
56
  assert.ok(typeof range === 'string', `advisories: malformed range in ${advisory.ghsa_id}`)
55
57
  for (const name of names.filter((candidate) => takes(candidate, vulnerability?.package))) {
56
- rows.set(`${name} ${advisory.ghsa_id} ${range}`, fromRepository(name, advisory, range, asked.get(name), covers))
58
+ rows.set(`${name} ${advisory.ghsa_id} ${range}`, fromRepository(repo, name, advisory, range, asked.get(name), { covers, details }))
57
59
  }
58
60
  }
59
61
  }
@@ -61,30 +63,46 @@ async function repositoryAdvisories(github, asked, { repoOf, takes = () => true,
61
63
  return [...rows.values()]
62
64
  }
63
65
 
66
+ // Each asked name's repository: as given, `known`, or else looked up.
67
+ async function reposOf(asked, known, lookUp) {
68
+ const found = await lookUp([...asked.keys()].filter((name) => !known.has(name)))
69
+ return (name) => known.get(name) ?? found.get(name)
70
+ }
71
+
64
72
  // A maintainer's advisory is on the repository before GitHub reviews it
65
73
  // into the databases the registries answer from. With `repoAdvisories`,
66
74
  // each package's repository, `known` or else looked up, adds what it
67
75
  // publishes for GitHub's `ecosystem` entries naming the package, for the
68
76
  // versions `rows` do not already report under that GHSA.
69
- export async function withRepositories(rows, asked, { github, repoAdvisories, known }, { ecosystem, lookUp, covers }) {
77
+ export async function withRepositories(rows, asked, { github, repoAdvisories, known, details }, { ecosystem, lookUp, covers }) {
70
78
  if (!repoAdvisories) return rows
71
79
  const reported = new Set(rows.flatMap((row) => [row.id, row.ghsa, ...row.aliases].filter(isGhsa).flatMap((id) => row.versions.map((version) => `${row.name} ${id} ${version}`))))
72
- const found = await lookUp([...asked.keys()].filter((name) => !known.has(name)))
73
- const repoOf = (name) => known.get(name) ?? found.get(name)
80
+ const repoOf = await reposOf(asked, known, lookUp)
74
81
  const takes = (name, pkg) => pkg?.ecosystem === ecosystem && pkg.name === name
75
- const added = await repositoryAdvisories(github, asked, { repoOf, takes, covers })
82
+ const added = await repositoryAdvisories(github, asked, { repoOf, takes, covers, details })
76
83
  return [...rows, ...added.map((row) => ({ ...row, versions: row.versions.filter((version) => !reported.has(`${row.name} ${row.id} ${version}`)) }))]
77
84
  }
78
85
 
79
86
  // stasis versions a repository with no version of its own by its branch,
80
- // or 0.0.0: every range covers those. Whatever semver reads is a version,
81
- // `v1.2.3` and `1.2.3+build` included.
82
- const coversPlaceholder = (version, range) => version === '0.0.0' || valid(version) === null || inRange(version, range)
87
+ // or 0.0.0: every range covers those.
88
+ const coversPlaceholder = (version, range) => version === '0.0.0' || mayBeInRange(version, range)
83
89
 
84
90
  // Dependencies that are GitHub repositories themselves: every range their
85
91
  // own published advisories list counts, whichever package it names.
86
92
  export const GITHUB = {
93
+ repositoryOnly: true,
87
94
  assertName: assertRepo,
88
95
  assertVersion: assertion('a version or a branch name', isRefName),
89
- advisories: (asked, { github }) => repositoryAdvisories(github, asked, { repoOf: (repo) => repo, covers: coversPlaceholder }),
96
+ advisories: (asked, { github, details }) => repositoryAdvisories(github, asked, { repoOf: (repo) => repo, covers: coversPlaceholder, details }),
97
+ }
98
+
99
+ // Soldeer packages, which no advisory database has: the repository each
100
+ // is published from, as given or else as Soldeer's project names it, is
101
+ // their only source, and every range its advisories list counts, whichever
102
+ // package it names (often the npm package the same release went out as).
103
+ export const SOLDEER = {
104
+ repositoryOnly: true,
105
+ assertName: assertSoldeerName,
106
+ assertVersion: assertSoldeerVersion,
107
+ advisories: async (asked, { github, known, details }) => await repositoryAdvisories(github, asked, { repoOf: await reposOf(asked, known, soldeerRepos), details }),
90
108
  }
@@ -1,12 +1,12 @@
1
1
  import assert from 'node:assert/strict'
2
2
 
3
- import { assertArgs, assertBoolean, assertRepo, assertion, optional, sameName } from '../args.js'
3
+ import { assertArgs, assertBoolean, assertRepo, assertion, isIterable, optional, sameName } from '../args.js'
4
4
  import { order } from './common.js'
5
- import { GITHUB, assertClient } from './github.js'
5
+ import { GITHUB, SOLDEER, assertClient } from './github.js'
6
6
  import { NPM } from './npm.js'
7
7
  import { CARGO, COMPOSER } from './osv.js'
8
8
 
9
- const ECOSYSTEMS = { npm: NPM, cargo: CARGO, composer: COMPOSER, github: GITHUB }
9
+ const ECOSYSTEMS = { npm: NPM, cargo: CARGO, composer: COMPOSER, soldeer: SOLDEER, github: GITHUB }
10
10
  const byNumbers = new Intl.Collator('en', { numeric: true }).compare
11
11
  const assertEcosystem = assertion(`one of ${Object.keys(ECOSYSTEMS).join(', ')}`, (value) => typeof value === 'string' && Object.hasOwn(ECOSYSTEMS, value))
12
12
  const assertVersions = assertion('a non-empty array', (value) => Array.isArray(value) && value.length > 0)
@@ -14,7 +14,7 @@ const assertVersions = assertion('a non-empty array', (value) => Array.isArray(v
14
14
  // Ecosystem → name → { versions, github? }, a name given twice merged.
15
15
  // Every package is checked before any is used.
16
16
  function collect(packages) {
17
- assert.ok(typeof packages?.[Symbol.iterator] === 'function' && typeof packages !== 'string', 'advisories: packages must be an iterable of { ecosystem, name, github?, versions }')
17
+ assert.ok(isIterable(packages), 'advisories: packages must be an iterable of { ecosystem, name, github?, versions }')
18
18
  const byEcosystem = new Map()
19
19
  for (const pkg of packages) {
20
20
  assertArgs('advisories', pkg, { ecosystem: assertEcosystem, name: null, github: optional(assertRepo), versions: assertVersions }, 'package')
@@ -34,16 +34,16 @@ function collect(packages) {
34
34
  }
35
35
 
36
36
  export async function advisories(packages, options = {}) {
37
- assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean) })
37
+ assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean) })
38
38
  assert.ok(!options.repoAdvisories || options.github, 'advisories: repoAdvisories needs a github client')
39
39
  const byEcosystem = collect(packages)
40
- assert.ok(!byEcosystem.has('github') || options.github, 'advisories: github packages need a github client')
40
+ for (const ecosystem of byEcosystem.keys()) assert.ok(!ECOSYSTEMS[ecosystem].repositoryOnly || options.github, `advisories: ${ecosystem} packages need a github client`)
41
41
  const found = await Promise.all([...byEcosystem].map(async ([ecosystem, named]) => {
42
42
  const { compare = byNumbers, advisories: find } = ECOSYSTEMS[ecosystem]
43
43
  const names = [...named.keys()].toSorted()
44
44
  const asked = new Map(names.map((name) => [name, [...named.get(name).versions].toSorted(compare)]))
45
45
  const known = new Map(names.filter((name) => named.get(name).github).map((name) => [name, named.get(name).github]))
46
- const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, known })
46
+ const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, known })
47
47
  return rows.map((row) => ({ ecosystem, ...row }))
48
48
  }))
49
49
  return found.flat().filter((row) => row.versions.length > 0).toSorted((a, b) => order(a.ecosystem, b.ecosystem) || order(a.name, b.name))
@@ -1,10 +1,12 @@
1
1
  import assert from 'node:assert/strict'
2
2
 
3
3
  import { assertPackageName, assertPackageVersion, isGhsa, isStrings, show } from '../args.js'
4
- import { NPM_REGISTRY, buildUrl, request } from '../http.js'
4
+ import { NPM_REGISTRY, buildUrl, isNotFound, recover, request } from '../http.js'
5
+ import { chunks, pool } from '../pool.js'
5
6
  import { compareVersions } from '../semver.js'
6
- import { covered, isText, metrics } from './common.js'
7
+ import { advisoryUrl, covered, detailsOf, isText, metrics } from './common.js'
7
8
  import { withRepositories } from './github.js'
9
+ import { RECORDS_AT_ONCE, getVuln } from './osv.js'
8
10
  import { npmRepos } from './repos.js'
9
11
 
10
12
  const BULK_URL = buildUrl(NPM_REGISTRY, ['-', 'npm', 'v1', 'security', 'advisories', 'bulk'])
@@ -22,7 +24,7 @@ function fromRegistry(name, row, asked) {
22
24
  name,
23
25
  source: 'registry',
24
26
  id: ghsa ?? `npm:${row.id}`,
25
- ...(ghsa && { ghsa }),
27
+ ...(ghsa && { ghsa, url: advisoryUrl(ghsa) }),
26
28
  aliases: [],
27
29
  title: row.title,
28
30
  ...metrics({ severity: row.severity, score: row.cvss?.score, vector: row.cvss?.vectorString, cwe: row.cwe }),
@@ -34,8 +36,7 @@ function fromRegistry(name, row, asked) {
34
36
  async function registryAdvisories(asked) {
35
37
  const names = [...asked.keys()]
36
38
  const advisories = []
37
- for (let i = 0; i < names.length; i += NAMES_PER_REQUEST) {
38
- const chunk = names.slice(i, i + NAMES_PER_REQUEST)
39
+ for (const chunk of chunks(names, NAMES_PER_REQUEST)) {
39
40
  const body = Object.fromEntries(chunk.map((name) => [name, asked.get(name)]))
40
41
  const answer = await request(BULK_URL, { method: 'POST', body, as: 'json' })
41
42
  assert.ok(answer && typeof answer === 'object' && !Array.isArray(answer), 'advisories: expected an object from the registry')
@@ -49,10 +50,23 @@ async function registryAdvisories(asked) {
49
50
  return advisories
50
51
  }
51
52
 
53
+ // The registry's rows carry no text: each GHSA's is OSV's record of it,
54
+ // GitHub's advisory database as published, asked once for the rows that
55
+ // cover an asked version. One OSV does not have yet has none.
56
+ async function withDetails(rows) {
57
+ const ghsas = [...new Set(rows.filter((row) => row.ghsa && row.versions.length > 0).map((row) => row.ghsa))]
58
+ const records = await pool(ghsas, RECORDS_AT_ONCE, (ghsa) => getVuln(ghsa).catch(recover(isNotFound, undefined)))
59
+ const text = new Map(ghsas.map((ghsa, i) => [ghsa, records[i]?.details]))
60
+ return rows.map((row) => ({ ...row, ...detailsOf(text.get(row.ghsa), row.ghsa) }))
61
+ }
62
+
52
63
  // What `npm audit` asks the registry, one row per vulnerable range.
53
64
  export const NPM = {
54
65
  assertName: assertPackageName,
55
66
  assertVersion: assertPackageVersion,
56
67
  compare: compareVersions,
57
- advisories: async (asked, options) => await withRepositories(await registryAdvisories(asked), asked, options, { ecosystem: 'npm', lookUp: npmRepos }),
68
+ advisories: async (asked, options) => {
69
+ const rows = await registryAdvisories(asked)
70
+ return await withRepositories(options.details ? await withDetails(rows) : rows, asked, options, { ecosystem: 'npm', lookUp: npmRepos })
71
+ },
58
72
  }
@@ -1,26 +1,34 @@
1
1
  import assert from 'node:assert/strict'
2
2
 
3
- import { assertion, isGhsa, isStrings, matches, show } from '../args.js'
3
+ import { assertCrateName, assertCrateVersion, assertion, isGhsa, isStrings, matches, show } from '../args.js'
4
4
  import { OSV_API, buildUrl, request } from '../http.js'
5
- import { pool } from '../pool.js'
5
+ import { chunks, pool } from '../pool.js'
6
6
  import { isExactVersion } from '../semver.js'
7
- import { inRange, isText, metrics, order } from './common.js'
7
+ import { advisoryUrl, detailsOf, inRange, isText, metrics, order } from './common.js'
8
8
  import { withRepositories } from './github.js'
9
9
  import { composerRepos, crateRepos } from './repos.js'
10
10
 
11
11
  const QUERY_URL = buildUrl(OSV_API, ['v1', 'querybatch'])
12
12
  const QUERIES_PER_REQUEST = 1000
13
- const RECORDS_AT_ONCE = 8
13
+ export const RECORDS_AT_ONCE = 8
14
14
  const isOsvId = matches(/^(?=.{1,128}$)[A-Z][\dA-Z]*(?:-[\dA-Za-z]+)+$/u)
15
+ // osv.dev's page for one of its records. An id that is only another's
16
+ // alias (PKSA-…) has none, and redirects to a search.
17
+ const osvPage = (id) => `https://osv.dev/vulnerability/${id}`
15
18
  const INFORMATIONAL = new Set(['unmaintained', 'unsound', 'notice'])
19
+ // Of the records that publish one advisory, the first answered for a
20
+ // version stands for the rest on it: RustSec's, what `cargo audit` reads
21
+ // and GitHub mirrors, then the GHSA, then any other (MAL-…), then by id.
22
+ const rank = (id) => (id.startsWith('RUSTSEC-') ? 0 : (isGhsa(id) ? 1 : 2))
23
+ const first = (a, b) => rank(a.id) - rank(b.id) || order(a.id, b.id)
24
+ const named = (a, b) => a.aliases.includes(b.id) || b.aliases.includes(a.id)
16
25
 
17
26
  export const CARGO = {
18
27
  osv: 'crates.io',
19
28
  github: 'rust',
20
29
  lookUp: crateRepos,
21
- assertName: assertion('a crate name', matches(/^[A-Za-z][\w-]{0,63}$/u)),
22
- assertVersion: assertion('a semver version', matches(/^(?=.{5,256}$)(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:0|[1-9]\d*|\d*[A-Za-z-][\dA-Za-z-]*)(?:\.(?:0|[1-9]\d*|\d*[A-Za-z-][\dA-Za-z-]*))*)?(?:\+[\dA-Za-z-]+(?:\.[\dA-Za-z-]+)*)?$/u)),
23
- keep: (id) => id.startsWith('RUSTSEC-'), // What `cargo audit` reads; the GHSA records mirror it.
30
+ assertName: assertCrateName,
31
+ assertVersion: assertCrateVersion,
24
32
  advisories: (asked, options) => osvAdvisories(CARGO, asked, options),
25
33
  }
26
34
  // TODO: packagist.org's API, what `composer audit` reads, also has the
@@ -41,7 +49,7 @@ export const COMPOSER = {
41
49
  advisories: (asked, options) => osvAdvisories(COMPOSER, asked, options),
42
50
  }
43
51
 
44
- async function getVuln(id) {
52
+ export async function getVuln(id) {
45
53
  const record = await request(buildUrl(OSV_API, ['v1', 'vulns', id]), { as: 'json' })
46
54
  assert.ok(record?.id === id, `advisories: OSV answered for ${show(record?.id)}, not ${id}`)
47
55
  assert.ok((record.aliases === undefined || isStrings(record.aliases)) && (record.summary === undefined || isText(record.summary))
@@ -49,7 +57,7 @@ async function getVuln(id) {
49
57
  return { ...record, aliases: (record.aliases ?? []).filter(isOsvId) }
50
58
  }
51
59
 
52
- function toAdvisory(ecosystem, name, versions, record) {
60
+ function toAdvisory(ecosystem, name, versions, record, details) {
53
61
  const { aliases } = record
54
62
  const ghsas = aliases.filter(isGhsa)
55
63
  const ghsa = isGhsa(record.id) ? record.id : (ghsas.length === 1 ? ghsas[0] : undefined)
@@ -61,8 +69,10 @@ function toAdvisory(ecosystem, name, versions, record) {
61
69
  source: 'osv',
62
70
  id: record.id,
63
71
  ...(ghsa && { ghsa }),
72
+ url: ghsa ? advisoryUrl(ghsa) : osvPage(record.id),
64
73
  aliases,
65
74
  ...(record.summary && { title: record.summary }),
75
+ ...(details && detailsOf(record.details, record.id)),
66
76
  ...metrics({ severity: record.database_specific?.severity, vector, cwe: record.database_specific?.cwe_ids }),
67
77
  ...(INFORMATIONAL.has(informational) && { informational }),
68
78
  versions,
@@ -70,15 +80,16 @@ function toAdvisory(ecosystem, name, versions, record) {
70
80
  }
71
81
 
72
82
  // OSV's batch query matches versions on its side, but answers ids only:
73
- // each record is fetched once after. A record another database also
74
- // publishes comes back under both ids, so one that aliases a GHSA keeps
75
- // only the versions that GHSA was not answered for.
83
+ // each record is fetched once after. An advisory more than one database
84
+ // publishes comes back under each one's id, the records naming each other
85
+ // as aliases, either way round, directly or through others: one group, in
86
+ // which each version is reported under the first answered for it, and
87
+ // each record has the others' ids as aliases.
76
88
  async function osvAdvisories(ecosystem, asked, options) {
77
- const { osv, github, lookUp, covers, keep = () => true } = ecosystem
89
+ const { osv, github, lookUp, covers } = ecosystem
78
90
  const list = [...asked].flatMap(([name, versions]) => versions.map((version) => ({ name, version })))
79
91
  const hits = new Map() // id → name → versions, in `list` order
80
- for (let i = 0; i < list.length; i += QUERIES_PER_REQUEST) {
81
- const chunk = list.slice(i, i + QUERIES_PER_REQUEST)
92
+ for (const chunk of chunks(list, QUERIES_PER_REQUEST)) {
82
93
  const body = { queries: chunk.map(({ name, version }) => ({ package: { name, ecosystem: osv }, version })) }
83
94
  const answer = await request(QUERY_URL, { method: 'POST', body, as: 'json' })
84
95
  assert.ok(Array.isArray(answer?.results) && answer.results.length === chunk.length, 'advisories: expected one OSV result per query')
@@ -86,21 +97,28 @@ async function osvAdvisories(ecosystem, asked, options) {
86
97
  const { name, version } = chunk[j]
87
98
  const vulns = result?.vulns ?? []
88
99
  assert.ok(result && result.next_page_token === undefined && Array.isArray(vulns) && vulns.every((vuln) => isOsvId(vuln?.id)), `advisories: malformed OSV result for ${name}@${version}`)
89
- for (const { id } of vulns.filter((vuln) => keep(vuln.id))) {
100
+ for (const { id } of vulns) {
90
101
  const byName = hits.get(id) ?? hits.set(id, new Map()).get(id)
91
102
  byName.set(name, (byName.get(name) ?? new Set()).add(version))
92
103
  }
93
104
  }
94
105
  }
95
106
  const records = (await pool([...hits.keys()], RECORDS_AT_ONCE, getVuln)).filter((record) => !record.withdrawn)
96
- const live = new Set(records.map((record) => record.id))
107
+ const groups = new Map()
108
+ for (const record of records) {
109
+ if (groups.has(record)) continue
110
+ const group = [record]
111
+ for (const member of group) group.push(...records.filter((other) => !group.includes(other) && named(member, other)))
112
+ for (const member of group) groups.set(member, group)
113
+ }
97
114
  const rows = []
98
115
  for (const record of records) {
116
+ const group = groups.get(record)
117
+ const above = group.filter((other) => first(other, record) < 0)
118
+ const aliases = [...new Set([...record.aliases, ...group.map((other) => other.id)])].filter((id) => id !== record.id)
99
119
  for (const [name, versions] of hits.get(record.id)) {
100
- // Only the versions a live GHSA it aliases answered for too.
101
- const shadowed = isGhsa(record.id) ? [] : record.aliases.filter((alias) => isGhsa(alias) && live.has(alias)).flatMap((alias) => [...(hits.get(alias).get(name) ?? [])])
102
- const rest = [...versions].filter((version) => !shadowed.includes(version))
103
- if (rest.length > 0) rows.push(toAdvisory(osv, name, rest, record))
120
+ const rest = [...versions].filter((version) => !above.some((other) => hits.get(other.id).get(name)?.has(version)))
121
+ if (rest.length > 0) rows.push(toAdvisory(osv, name, rest, { ...record, aliases }, options.details))
104
122
  }
105
123
  }
106
124
  rows.sort((a, b) => order(a.name, b.name) || order(a.id, b.id))