@preventive/upstream 1.0.0-alpha.0 → 1.0.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-APACHE +202 -0
- package/advisories.d.ts +47 -16
- package/cargo.d.ts +36 -0
- package/cargo.js +3 -0
- package/github.d.ts +67 -13
- package/npm.d.ts +40 -13
- package/npm.js +1 -1
- package/package.d.ts +3 -3
- package/package.json +41 -4
- package/semver.d.ts +12 -2
- package/semver.js +1 -1
- package/soldeer.d.ts +17 -0
- package/soldeer.js +2 -0
- package/src/advisories/common.js +27 -5
- package/src/advisories/github.js +38 -20
- package/src/advisories/index.js +7 -7
- package/src/advisories/npm.js +20 -6
- package/src/advisories/osv.js +39 -21
- package/src/advisories/repos.js +18 -42
- package/src/args.js +40 -2
- package/src/attributes.js +81 -0
- package/src/cache.js +103 -9
- package/src/cargo/crate.js +53 -0
- package/src/cargo/repos.js +52 -0
- package/src/download.js +42 -0
- package/src/github/read.js +113 -14
- package/src/github/write.js +6 -9
- package/src/http.js +29 -14
- package/src/npm/registry.js +19 -0
- package/src/npm/repos.js +27 -25
- package/src/npm/tarball.js +13 -53
- package/src/package.js +17 -6
- package/src/pool.js +2 -0
- package/src/remote.js +8 -1
- package/src/semver-peer.cjs +8 -0
- package/src/semver.js +29 -12
- package/src/soldeer.js +26 -0
- package/src/tree.js +237 -0
- package/src/weak-edges.js +87 -0
- package/weak-edges.d.ts +36 -0
- package/weak-edges.js +1 -0
- /package/{LICENSE → LICENSE-MIT} +0 -0
package/package.json
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@preventive/upstream",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
4
|
-
"description": "Minimal npm
|
|
3
|
+
"version": "1.0.0-alpha.10",
|
|
4
|
+
"description": "Minimal npm, crates.io and Soldeer registry clients, a GitHub client, and a git checkout reader: the calls we use, nothing more",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
7
7
|
"./advisories.js": {
|
|
8
8
|
"types": "./advisories.d.ts",
|
|
9
9
|
"default": "./advisories.js"
|
|
10
10
|
},
|
|
11
|
+
"./cargo.js": {
|
|
12
|
+
"types": "./cargo.d.ts",
|
|
13
|
+
"default": "./cargo.js"
|
|
14
|
+
},
|
|
11
15
|
"./github.js": {
|
|
12
16
|
"types": "./github.d.ts",
|
|
13
17
|
"default": "./github.js"
|
|
@@ -31,11 +35,23 @@
|
|
|
31
35
|
"./semver.js": {
|
|
32
36
|
"types": "./semver.d.ts",
|
|
33
37
|
"default": "./semver.js"
|
|
38
|
+
},
|
|
39
|
+
"./soldeer.js": {
|
|
40
|
+
"types": "./soldeer.d.ts",
|
|
41
|
+
"default": "./soldeer.js"
|
|
42
|
+
},
|
|
43
|
+
"./weak-edges.js": {
|
|
44
|
+
"types": "./weak-edges.d.ts",
|
|
45
|
+
"default": "./weak-edges.js"
|
|
34
46
|
}
|
|
35
47
|
},
|
|
36
48
|
"files": [
|
|
49
|
+
"LICENSE-APACHE",
|
|
50
|
+
"LICENSE-MIT",
|
|
37
51
|
"advisories.d.ts",
|
|
38
52
|
"advisories.js",
|
|
53
|
+
"cargo.d.ts",
|
|
54
|
+
"cargo.js",
|
|
39
55
|
"github.d.ts",
|
|
40
56
|
"github.js",
|
|
41
57
|
"git.d.ts",
|
|
@@ -48,6 +64,10 @@
|
|
|
48
64
|
"package.js",
|
|
49
65
|
"semver.d.ts",
|
|
50
66
|
"semver.js",
|
|
67
|
+
"soldeer.d.ts",
|
|
68
|
+
"soldeer.js",
|
|
69
|
+
"weak-edges.d.ts",
|
|
70
|
+
"weak-edges.js",
|
|
51
71
|
"src/advisories/common.js",
|
|
52
72
|
"src/advisories/github.js",
|
|
53
73
|
"src/advisories/index.js",
|
|
@@ -55,18 +75,27 @@
|
|
|
55
75
|
"src/advisories/osv.js",
|
|
56
76
|
"src/advisories/repos.js",
|
|
57
77
|
"src/args.js",
|
|
78
|
+
"src/attributes.js",
|
|
58
79
|
"src/cache.js",
|
|
80
|
+
"src/cargo/crate.js",
|
|
81
|
+
"src/cargo/repos.js",
|
|
82
|
+
"src/download.js",
|
|
59
83
|
"src/git.js",
|
|
60
84
|
"src/github/client.js",
|
|
61
85
|
"src/github/read.js",
|
|
62
86
|
"src/github/write.js",
|
|
63
87
|
"src/http.js",
|
|
88
|
+
"src/npm/registry.js",
|
|
64
89
|
"src/npm/repos.js",
|
|
65
90
|
"src/npm/tarball.js",
|
|
66
91
|
"src/package.js",
|
|
67
92
|
"src/pool.js",
|
|
68
93
|
"src/remote.js",
|
|
69
|
-
"src/semver.
|
|
94
|
+
"src/semver-peer.cjs",
|
|
95
|
+
"src/semver.js",
|
|
96
|
+
"src/tree.js",
|
|
97
|
+
"src/soldeer.js",
|
|
98
|
+
"src/weak-edges.js"
|
|
70
99
|
],
|
|
71
100
|
"engines": {
|
|
72
101
|
"node": ">=24.14.0"
|
|
@@ -74,7 +103,7 @@
|
|
|
74
103
|
"scripts": {
|
|
75
104
|
"test": "node --test ./tests/*.test.js"
|
|
76
105
|
},
|
|
77
|
-
"license": "MIT",
|
|
106
|
+
"license": "MIT OR Apache-2.0",
|
|
78
107
|
"author": {
|
|
79
108
|
"name": "Nikita Skovoroda",
|
|
80
109
|
"email": "chalkerx@gmail.com",
|
|
@@ -91,5 +120,13 @@
|
|
|
91
120
|
"homepage": "https://github.com/PreventiveMeasures/libraries/tree/main/upstream",
|
|
92
121
|
"publishConfig": {
|
|
93
122
|
"access": "public"
|
|
123
|
+
},
|
|
124
|
+
"peerDependencies": {
|
|
125
|
+
"semver": "^7.5.2"
|
|
126
|
+
},
|
|
127
|
+
"peerDependenciesMeta": {
|
|
128
|
+
"semver": {
|
|
129
|
+
"optional": true
|
|
130
|
+
}
|
|
94
131
|
}
|
|
95
132
|
}
|
package/semver.d.ts
CHANGED
|
@@ -1,11 +1,21 @@
|
|
|
1
1
|
// Hand-written against semver.js; a change to either belongs with the other.
|
|
2
2
|
|
|
3
3
|
// npm's own semver, borrowed from the npm beside node and passed through
|
|
4
|
-
// as it is
|
|
4
|
+
// as it is; where there is no npm, the `semver` package, an optional peer
|
|
5
|
+
// dependency. Without either, every call throws, except valid() on a plain
|
|
5
6
|
// MAJOR.MINOR.PATCH with no options.
|
|
6
7
|
export function satisfies(version: string, range: string, options?: { includePrerelease?: boolean; loose?: boolean }): boolean
|
|
7
|
-
export function compareVersions(a: string, b: string): -1 | 0 | 1
|
|
8
|
+
export function compareVersions(a: string, b: string, options?: { loose?: boolean }): -1 | 0 | 1
|
|
8
9
|
// The version as semver spells it, or null: `v1.2.3` answers `1.2.3`.
|
|
9
10
|
export function valid(version: unknown, options?: { loose?: boolean }): string | null
|
|
10
11
|
// A string that valid() answers unchanged.
|
|
11
12
|
export function isExactVersion(version: unknown): version is string
|
|
13
|
+
// The range as semver normalizes it, or null where it is not one.
|
|
14
|
+
export function validRange(range: unknown, options?: { loose?: boolean; includePrerelease?: boolean }): string | null
|
|
15
|
+
// Whether some version is in both ranges; throws for one that is not a range.
|
|
16
|
+
export function intersects(a: string, b: string, options?: { loose?: boolean; includePrerelease?: boolean }): boolean
|
|
17
|
+
// The version as semver cleans it, spaces and a leading `=` or `v` dropped,
|
|
18
|
+
// or null: ` =v1.2.3 ` answers `1.2.3`.
|
|
19
|
+
export function clean(version: string, options?: { loose?: boolean } | boolean): string | null
|
|
20
|
+
// The version's major, as semver reads it; throws for one that is not a version.
|
|
21
|
+
export function major(version: string, options?: { loose?: boolean } | boolean): number
|
package/semver.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export { compareVersions, isExactVersion, satisfies, valid } from './src/semver.js'
|
|
1
|
+
export { clean, compareVersions, intersects, isExactVersion, major, satisfies, valid, validRange } from './src/semver.js'
|
package/soldeer.d.ts
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// Hand-written against soldeer.js; a change to either belongs with the
|
|
2
|
+
// other.
|
|
3
|
+
//
|
|
4
|
+
// The name is asserted to be one Soldeer takes, the version a string of
|
|
5
|
+
// letters, digits, `.`, `_`, `+` and `-`, and the checksum 64 lowercase hex
|
|
6
|
+
// digits, before any request or cache path. The cache is setCacheDir's
|
|
7
|
+
// (npm.js).
|
|
8
|
+
|
|
9
|
+
export { HttpError } from './npm.js'
|
|
10
|
+
|
|
11
|
+
// A version's zip from Soldeer's registry, whole, in memory. The registry
|
|
12
|
+
// says where the zip is but not what it hashes to, so `checksum` is
|
|
13
|
+
// required: the zip's sha256, as soldeer.lock records it. The bytes are
|
|
14
|
+
// checked against it, whether they were downloaded (before they are
|
|
15
|
+
// cached) or read from setCacheDir's cache, where one that does not match
|
|
16
|
+
// throws. A cached zip that matches is answered with no request at all.
|
|
17
|
+
export function getZip(name: string, version: string, checksum: string): Promise<Uint8Array>
|
package/soldeer.js
ADDED
package/src/advisories/common.js
CHANGED
|
@@ -1,21 +1,43 @@
|
|
|
1
|
-
import
|
|
2
|
-
|
|
1
|
+
import assert from 'node:assert/strict'
|
|
2
|
+
|
|
3
|
+
import { isGhsa, isRepo, isStrings, matches, show } from '../args.js'
|
|
4
|
+
import { satisfies, valid, validRange } from '../semver.js'
|
|
3
5
|
|
|
4
6
|
export const order = (a, b) => (a > b) - (a < b)
|
|
5
7
|
|
|
8
|
+
// GitHub's page for a GHSA: the advisory database's, which has every one
|
|
9
|
+
// the registry and OSV answer with, or the page of the `repo` that
|
|
10
|
+
// published it, which has it before GitHub reviews it into the database.
|
|
11
|
+
export function advisoryUrl(ghsa, repo) {
|
|
12
|
+
assert.ok(isGhsa(ghsa) && (repo === undefined || isRepo(repo)), `advisories: no GitHub advisory page for ${show(ghsa)} on ${show(repo)}`)
|
|
13
|
+
return repo === undefined ? `https://github.com/advisories/${ghsa}` : `https://github.com/${repo}/security/advisories/${ghsa}`
|
|
14
|
+
}
|
|
15
|
+
|
|
6
16
|
// As `npm audit` matches: a prerelease is in a range around it.
|
|
7
17
|
const SEMVER = { includePrerelease: true, loose: true }
|
|
8
18
|
|
|
9
19
|
export const inRange = (version, range) => satisfies(version, range, SEMVER)
|
|
10
20
|
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
export const
|
|
21
|
+
// A version semver cannot read may be in any range. Whatever it reads is
|
|
22
|
+
// a version, `v1.2.3` and `1.2.3+build` included.
|
|
23
|
+
export const mayBeInRange = (version, range) => valid(version) === null || inRange(version, range)
|
|
24
|
+
|
|
25
|
+
// The asked versions a range `covers`. One semver cannot read, range or
|
|
26
|
+
// version, covers them all: a missed advisory is worse than a spare one.
|
|
27
|
+
export const covered = (asked, range, covers = mayBeInRange) => (validRange(range, SEMVER) === null ? [...asked] : asked.filter((version) => covers(version, range)))
|
|
14
28
|
|
|
15
29
|
// Remote text, as long as it is well-formed; anything else is refused as
|
|
16
30
|
// malformed by whoever reads it.
|
|
17
31
|
export const isText = (value) => typeof value === 'string' && value.isWellFormed()
|
|
18
32
|
|
|
33
|
+
// An advisory's full text, for `details`: none where its source has none,
|
|
34
|
+
// and refused where it is not well-formed.
|
|
35
|
+
export function detailsOf(text, id) {
|
|
36
|
+
if (text == null || text === '') return {}
|
|
37
|
+
assert.ok(isText(text), `advisories: malformed details in ${id}`)
|
|
38
|
+
return { details: text }
|
|
39
|
+
}
|
|
40
|
+
|
|
19
41
|
const SEVERITIES = new Set(['critical', 'high', 'moderate', 'low', 'info'])
|
|
20
42
|
const isCvssVector = matches(/^CVSS:[34]\.\d(?:\/[A-Z]{1,4}:[A-Z]{1,2})+$/u)
|
|
21
43
|
const isCwe = matches(/^CWE-\d{1,6}$/u)
|
package/src/advisories/github.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import assert from 'node:assert/strict'
|
|
2
2
|
|
|
3
|
-
import { assertRepo, assertion, isGhsa, isRefName, isStrings } from '../args.js'
|
|
3
|
+
import { assertRepo, assertSoldeerName, assertSoldeerVersion, assertion, isGhsa, isRefName, isStrings } from '../args.js'
|
|
4
4
|
import { isGone } from '../github/client.js'
|
|
5
|
+
import { recover } from '../http.js'
|
|
5
6
|
import { pool } from '../pool.js'
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
7
|
+
import { advisoryUrl, covered, detailsOf, isText, mayBeInRange, metrics } from './common.js'
|
|
8
|
+
import { soldeerRepos } from './repos.js'
|
|
8
9
|
|
|
9
10
|
const REPOS_AT_ONCE = 4
|
|
10
11
|
|
|
@@ -14,23 +15,24 @@ const isRepoAdvisory = (advisory) => advisory && typeof advisory === 'object' &&
|
|
|
14
15
|
|
|
15
16
|
export const assertClient = assertion('a GitHub client from createClient', (value) => typeof value?.listRepoAdvisories === 'function')
|
|
16
17
|
|
|
17
|
-
const listAdvisories = (github, repo) => github.listRepoAdvisories({ repo }).catch((
|
|
18
|
-
if (isGone(err)) return []
|
|
19
|
-
throw err
|
|
20
|
-
})
|
|
18
|
+
const listAdvisories = (github, repo) => github.listRepoAdvisories({ repo }).catch(recover(isGone, []))
|
|
21
19
|
|
|
22
20
|
// GitHub's `>= 1.0.0, < 1.2.6` is npm's with the commas dropped.
|
|
23
21
|
// Maintainers write these unreviewed: one without a range, or with one
|
|
24
|
-
// semver cannot read, covers every version.
|
|
25
|
-
|
|
22
|
+
// semver cannot read, covers every version. Its page is on `repo`, which
|
|
23
|
+
// may publish it long before the advisory database has one, and its
|
|
24
|
+
// `description` is the text `details` asks for: the listing has it all.
|
|
25
|
+
function fromRepository(repo, name, advisory, range, asked, { covers, details }) {
|
|
26
26
|
const cvss = [advisory.cvss_severities?.cvss_v3, advisory.cvss_severities?.cvss_v4, advisory.cvss].find((entry) => typeof entry?.vector_string === 'string')
|
|
27
27
|
return {
|
|
28
28
|
name,
|
|
29
29
|
source: 'repository',
|
|
30
30
|
id: advisory.ghsa_id,
|
|
31
31
|
ghsa: advisory.ghsa_id,
|
|
32
|
+
url: advisoryUrl(advisory.ghsa_id, repo),
|
|
32
33
|
aliases: [],
|
|
33
34
|
title: advisory.summary,
|
|
35
|
+
...(details && detailsOf(advisory.description, advisory.ghsa_id)),
|
|
34
36
|
...metrics({ severity: advisory.severity, score: cvss?.score, vector: cvss?.vector_string, cwe: advisory.cwe_ids }),
|
|
35
37
|
range,
|
|
36
38
|
versions: covered(asked, range.replaceAll(',', ' '), covers),
|
|
@@ -40,8 +42,8 @@ function fromRepository(name, advisory, range, asked, covers) {
|
|
|
40
42
|
// Each asked name's repository, `repoOf` it, is asked once, and its
|
|
41
43
|
// advisories' vulnerable ranges become rows for the names that `takes` a
|
|
42
44
|
// vulnerability, one per name, advisory and range, holding the asked
|
|
43
|
-
// versions it `covers
|
|
44
|
-
async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers }) {
|
|
45
|
+
// versions it `covers`, and with `details`, its text.
|
|
46
|
+
async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers, details }) {
|
|
45
47
|
// GitHub's names are case-insensitive: one spelling asks for all.
|
|
46
48
|
const namesOf = Map.groupBy([...asked.keys()].filter(repoOf), (name) => repoOf(name).toLowerCase())
|
|
47
49
|
const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ repo: repoOf(names[0]), names, list: await listAdvisories(github, repoOf(names[0])) }))
|
|
@@ -53,7 +55,7 @@ async function repositoryAdvisories(github, asked, { repoOf, takes = () => true,
|
|
|
53
55
|
const range = vulnerability?.vulnerable_version_range ?? ''
|
|
54
56
|
assert.ok(typeof range === 'string', `advisories: malformed range in ${advisory.ghsa_id}`)
|
|
55
57
|
for (const name of names.filter((candidate) => takes(candidate, vulnerability?.package))) {
|
|
56
|
-
rows.set(`${name} ${advisory.ghsa_id} ${range}`, fromRepository(name, advisory, range, asked.get(name), covers))
|
|
58
|
+
rows.set(`${name} ${advisory.ghsa_id} ${range}`, fromRepository(repo, name, advisory, range, asked.get(name), { covers, details }))
|
|
57
59
|
}
|
|
58
60
|
}
|
|
59
61
|
}
|
|
@@ -61,30 +63,46 @@ async function repositoryAdvisories(github, asked, { repoOf, takes = () => true,
|
|
|
61
63
|
return [...rows.values()]
|
|
62
64
|
}
|
|
63
65
|
|
|
66
|
+
// Each asked name's repository: as given, `known`, or else looked up.
|
|
67
|
+
async function reposOf(asked, known, lookUp) {
|
|
68
|
+
const found = await lookUp([...asked.keys()].filter((name) => !known.has(name)))
|
|
69
|
+
return (name) => known.get(name) ?? found.get(name)
|
|
70
|
+
}
|
|
71
|
+
|
|
64
72
|
// A maintainer's advisory is on the repository before GitHub reviews it
|
|
65
73
|
// into the databases the registries answer from. With `repoAdvisories`,
|
|
66
74
|
// each package's repository, `known` or else looked up, adds what it
|
|
67
75
|
// publishes for GitHub's `ecosystem` entries naming the package, for the
|
|
68
76
|
// versions `rows` do not already report under that GHSA.
|
|
69
|
-
export async function withRepositories(rows, asked, { github, repoAdvisories, known }, { ecosystem, lookUp, covers }) {
|
|
77
|
+
export async function withRepositories(rows, asked, { github, repoAdvisories, known, details }, { ecosystem, lookUp, covers }) {
|
|
70
78
|
if (!repoAdvisories) return rows
|
|
71
79
|
const reported = new Set(rows.flatMap((row) => [row.id, row.ghsa, ...row.aliases].filter(isGhsa).flatMap((id) => row.versions.map((version) => `${row.name} ${id} ${version}`))))
|
|
72
|
-
const
|
|
73
|
-
const repoOf = (name) => known.get(name) ?? found.get(name)
|
|
80
|
+
const repoOf = await reposOf(asked, known, lookUp)
|
|
74
81
|
const takes = (name, pkg) => pkg?.ecosystem === ecosystem && pkg.name === name
|
|
75
|
-
const added = await repositoryAdvisories(github, asked, { repoOf, takes, covers })
|
|
82
|
+
const added = await repositoryAdvisories(github, asked, { repoOf, takes, covers, details })
|
|
76
83
|
return [...rows, ...added.map((row) => ({ ...row, versions: row.versions.filter((version) => !reported.has(`${row.name} ${row.id} ${version}`)) }))]
|
|
77
84
|
}
|
|
78
85
|
|
|
79
86
|
// stasis versions a repository with no version of its own by its branch,
|
|
80
|
-
// or 0.0.0: every range covers those.
|
|
81
|
-
|
|
82
|
-
const coversPlaceholder = (version, range) => version === '0.0.0' || valid(version) === null || inRange(version, range)
|
|
87
|
+
// or 0.0.0: every range covers those.
|
|
88
|
+
const coversPlaceholder = (version, range) => version === '0.0.0' || mayBeInRange(version, range)
|
|
83
89
|
|
|
84
90
|
// Dependencies that are GitHub repositories themselves: every range their
|
|
85
91
|
// own published advisories list counts, whichever package it names.
|
|
86
92
|
export const GITHUB = {
|
|
93
|
+
repositoryOnly: true,
|
|
87
94
|
assertName: assertRepo,
|
|
88
95
|
assertVersion: assertion('a version or a branch name', isRefName),
|
|
89
|
-
advisories: (asked, { github }) => repositoryAdvisories(github, asked, { repoOf: (repo) => repo, covers: coversPlaceholder }),
|
|
96
|
+
advisories: (asked, { github, details }) => repositoryAdvisories(github, asked, { repoOf: (repo) => repo, covers: coversPlaceholder, details }),
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Soldeer packages, which no advisory database has: the repository each
|
|
100
|
+
// is published from, as given or else as Soldeer's project names it, is
|
|
101
|
+
// their only source, and every range its advisories list counts, whichever
|
|
102
|
+
// package it names (often the npm package the same release went out as).
|
|
103
|
+
export const SOLDEER = {
|
|
104
|
+
repositoryOnly: true,
|
|
105
|
+
assertName: assertSoldeerName,
|
|
106
|
+
assertVersion: assertSoldeerVersion,
|
|
107
|
+
advisories: async (asked, { github, known, details }) => await repositoryAdvisories(github, asked, { repoOf: await reposOf(asked, known, soldeerRepos), details }),
|
|
90
108
|
}
|
package/src/advisories/index.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import assert from 'node:assert/strict'
|
|
2
2
|
|
|
3
|
-
import { assertArgs, assertBoolean, assertRepo, assertion, optional, sameName } from '../args.js'
|
|
3
|
+
import { assertArgs, assertBoolean, assertRepo, assertion, isIterable, optional, sameName } from '../args.js'
|
|
4
4
|
import { order } from './common.js'
|
|
5
|
-
import { GITHUB, assertClient } from './github.js'
|
|
5
|
+
import { GITHUB, SOLDEER, assertClient } from './github.js'
|
|
6
6
|
import { NPM } from './npm.js'
|
|
7
7
|
import { CARGO, COMPOSER } from './osv.js'
|
|
8
8
|
|
|
9
|
-
const ECOSYSTEMS = { npm: NPM, cargo: CARGO, composer: COMPOSER, github: GITHUB }
|
|
9
|
+
const ECOSYSTEMS = { npm: NPM, cargo: CARGO, composer: COMPOSER, soldeer: SOLDEER, github: GITHUB }
|
|
10
10
|
const byNumbers = new Intl.Collator('en', { numeric: true }).compare
|
|
11
11
|
const assertEcosystem = assertion(`one of ${Object.keys(ECOSYSTEMS).join(', ')}`, (value) => typeof value === 'string' && Object.hasOwn(ECOSYSTEMS, value))
|
|
12
12
|
const assertVersions = assertion('a non-empty array', (value) => Array.isArray(value) && value.length > 0)
|
|
@@ -14,7 +14,7 @@ const assertVersions = assertion('a non-empty array', (value) => Array.isArray(v
|
|
|
14
14
|
// Ecosystem → name → { versions, github? }, a name given twice merged.
|
|
15
15
|
// Every package is checked before any is used.
|
|
16
16
|
function collect(packages) {
|
|
17
|
-
assert.ok(
|
|
17
|
+
assert.ok(isIterable(packages), 'advisories: packages must be an iterable of { ecosystem, name, github?, versions }')
|
|
18
18
|
const byEcosystem = new Map()
|
|
19
19
|
for (const pkg of packages) {
|
|
20
20
|
assertArgs('advisories', pkg, { ecosystem: assertEcosystem, name: null, github: optional(assertRepo), versions: assertVersions }, 'package')
|
|
@@ -34,16 +34,16 @@ function collect(packages) {
|
|
|
34
34
|
}
|
|
35
35
|
|
|
36
36
|
export async function advisories(packages, options = {}) {
|
|
37
|
-
assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean) })
|
|
37
|
+
assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean) })
|
|
38
38
|
assert.ok(!options.repoAdvisories || options.github, 'advisories: repoAdvisories needs a github client')
|
|
39
39
|
const byEcosystem = collect(packages)
|
|
40
|
-
assert.ok(!
|
|
40
|
+
for (const ecosystem of byEcosystem.keys()) assert.ok(!ECOSYSTEMS[ecosystem].repositoryOnly || options.github, `advisories: ${ecosystem} packages need a github client`)
|
|
41
41
|
const found = await Promise.all([...byEcosystem].map(async ([ecosystem, named]) => {
|
|
42
42
|
const { compare = byNumbers, advisories: find } = ECOSYSTEMS[ecosystem]
|
|
43
43
|
const names = [...named.keys()].toSorted()
|
|
44
44
|
const asked = new Map(names.map((name) => [name, [...named.get(name).versions].toSorted(compare)]))
|
|
45
45
|
const known = new Map(names.filter((name) => named.get(name).github).map((name) => [name, named.get(name).github]))
|
|
46
|
-
const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, known })
|
|
46
|
+
const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, known })
|
|
47
47
|
return rows.map((row) => ({ ecosystem, ...row }))
|
|
48
48
|
}))
|
|
49
49
|
return found.flat().filter((row) => row.versions.length > 0).toSorted((a, b) => order(a.ecosystem, b.ecosystem) || order(a.name, b.name))
|
package/src/advisories/npm.js
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import assert from 'node:assert/strict'
|
|
2
2
|
|
|
3
3
|
import { assertPackageName, assertPackageVersion, isGhsa, isStrings, show } from '../args.js'
|
|
4
|
-
import { NPM_REGISTRY, buildUrl, request } from '../http.js'
|
|
4
|
+
import { NPM_REGISTRY, buildUrl, isNotFound, recover, request } from '../http.js'
|
|
5
|
+
import { chunks, pool } from '../pool.js'
|
|
5
6
|
import { compareVersions } from '../semver.js'
|
|
6
|
-
import { covered, isText, metrics } from './common.js'
|
|
7
|
+
import { advisoryUrl, covered, detailsOf, isText, metrics } from './common.js'
|
|
7
8
|
import { withRepositories } from './github.js'
|
|
9
|
+
import { RECORDS_AT_ONCE, getVuln } from './osv.js'
|
|
8
10
|
import { npmRepos } from './repos.js'
|
|
9
11
|
|
|
10
12
|
const BULK_URL = buildUrl(NPM_REGISTRY, ['-', 'npm', 'v1', 'security', 'advisories', 'bulk'])
|
|
@@ -22,7 +24,7 @@ function fromRegistry(name, row, asked) {
|
|
|
22
24
|
name,
|
|
23
25
|
source: 'registry',
|
|
24
26
|
id: ghsa ?? `npm:${row.id}`,
|
|
25
|
-
...(ghsa && { ghsa }),
|
|
27
|
+
...(ghsa && { ghsa, url: advisoryUrl(ghsa) }),
|
|
26
28
|
aliases: [],
|
|
27
29
|
title: row.title,
|
|
28
30
|
...metrics({ severity: row.severity, score: row.cvss?.score, vector: row.cvss?.vectorString, cwe: row.cwe }),
|
|
@@ -34,8 +36,7 @@ function fromRegistry(name, row, asked) {
|
|
|
34
36
|
async function registryAdvisories(asked) {
|
|
35
37
|
const names = [...asked.keys()]
|
|
36
38
|
const advisories = []
|
|
37
|
-
for (
|
|
38
|
-
const chunk = names.slice(i, i + NAMES_PER_REQUEST)
|
|
39
|
+
for (const chunk of chunks(names, NAMES_PER_REQUEST)) {
|
|
39
40
|
const body = Object.fromEntries(chunk.map((name) => [name, asked.get(name)]))
|
|
40
41
|
const answer = await request(BULK_URL, { method: 'POST', body, as: 'json' })
|
|
41
42
|
assert.ok(answer && typeof answer === 'object' && !Array.isArray(answer), 'advisories: expected an object from the registry')
|
|
@@ -49,10 +50,23 @@ async function registryAdvisories(asked) {
|
|
|
49
50
|
return advisories
|
|
50
51
|
}
|
|
51
52
|
|
|
53
|
+
// The registry's rows carry no text: each GHSA's is OSV's record of it,
|
|
54
|
+
// GitHub's advisory database as published, asked once for the rows that
|
|
55
|
+
// cover an asked version. One OSV does not have yet has none.
|
|
56
|
+
async function withDetails(rows) {
|
|
57
|
+
const ghsas = [...new Set(rows.filter((row) => row.ghsa && row.versions.length > 0).map((row) => row.ghsa))]
|
|
58
|
+
const records = await pool(ghsas, RECORDS_AT_ONCE, (ghsa) => getVuln(ghsa).catch(recover(isNotFound, undefined)))
|
|
59
|
+
const text = new Map(ghsas.map((ghsa, i) => [ghsa, records[i]?.details]))
|
|
60
|
+
return rows.map((row) => ({ ...row, ...detailsOf(text.get(row.ghsa), row.ghsa) }))
|
|
61
|
+
}
|
|
62
|
+
|
|
52
63
|
// What `npm audit` asks the registry, one row per vulnerable range.
|
|
53
64
|
export const NPM = {
|
|
54
65
|
assertName: assertPackageName,
|
|
55
66
|
assertVersion: assertPackageVersion,
|
|
56
67
|
compare: compareVersions,
|
|
57
|
-
advisories: async (asked, options) =>
|
|
68
|
+
advisories: async (asked, options) => {
|
|
69
|
+
const rows = await registryAdvisories(asked)
|
|
70
|
+
return await withRepositories(options.details ? await withDetails(rows) : rows, asked, options, { ecosystem: 'npm', lookUp: npmRepos })
|
|
71
|
+
},
|
|
58
72
|
}
|
package/src/advisories/osv.js
CHANGED
|
@@ -1,26 +1,34 @@
|
|
|
1
1
|
import assert from 'node:assert/strict'
|
|
2
2
|
|
|
3
|
-
import { assertion, isGhsa, isStrings, matches, show } from '../args.js'
|
|
3
|
+
import { assertCrateName, assertCrateVersion, assertion, isGhsa, isStrings, matches, show } from '../args.js'
|
|
4
4
|
import { OSV_API, buildUrl, request } from '../http.js'
|
|
5
|
-
import { pool } from '../pool.js'
|
|
5
|
+
import { chunks, pool } from '../pool.js'
|
|
6
6
|
import { isExactVersion } from '../semver.js'
|
|
7
|
-
import { inRange, isText, metrics, order } from './common.js'
|
|
7
|
+
import { advisoryUrl, detailsOf, inRange, isText, metrics, order } from './common.js'
|
|
8
8
|
import { withRepositories } from './github.js'
|
|
9
9
|
import { composerRepos, crateRepos } from './repos.js'
|
|
10
10
|
|
|
11
11
|
const QUERY_URL = buildUrl(OSV_API, ['v1', 'querybatch'])
|
|
12
12
|
const QUERIES_PER_REQUEST = 1000
|
|
13
|
-
const RECORDS_AT_ONCE = 8
|
|
13
|
+
export const RECORDS_AT_ONCE = 8
|
|
14
14
|
const isOsvId = matches(/^(?=.{1,128}$)[A-Z][\dA-Z]*(?:-[\dA-Za-z]+)+$/u)
|
|
15
|
+
// osv.dev's page for one of its records. An id that is only another's
|
|
16
|
+
// alias (PKSA-…) has none, and redirects to a search.
|
|
17
|
+
const osvPage = (id) => `https://osv.dev/vulnerability/${id}`
|
|
15
18
|
const INFORMATIONAL = new Set(['unmaintained', 'unsound', 'notice'])
|
|
19
|
+
// Of the records that publish one advisory, the first answered for a
|
|
20
|
+
// version stands for the rest on it: RustSec's, what `cargo audit` reads
|
|
21
|
+
// and GitHub mirrors, then the GHSA, then any other (MAL-…), then by id.
|
|
22
|
+
const rank = (id) => (id.startsWith('RUSTSEC-') ? 0 : (isGhsa(id) ? 1 : 2))
|
|
23
|
+
const first = (a, b) => rank(a.id) - rank(b.id) || order(a.id, b.id)
|
|
24
|
+
const named = (a, b) => a.aliases.includes(b.id) || b.aliases.includes(a.id)
|
|
16
25
|
|
|
17
26
|
export const CARGO = {
|
|
18
27
|
osv: 'crates.io',
|
|
19
28
|
github: 'rust',
|
|
20
29
|
lookUp: crateRepos,
|
|
21
|
-
assertName:
|
|
22
|
-
assertVersion:
|
|
23
|
-
keep: (id) => id.startsWith('RUSTSEC-'), // What `cargo audit` reads; the GHSA records mirror it.
|
|
30
|
+
assertName: assertCrateName,
|
|
31
|
+
assertVersion: assertCrateVersion,
|
|
24
32
|
advisories: (asked, options) => osvAdvisories(CARGO, asked, options),
|
|
25
33
|
}
|
|
26
34
|
// TODO: packagist.org's API, what `composer audit` reads, also has the
|
|
@@ -41,7 +49,7 @@ export const COMPOSER = {
|
|
|
41
49
|
advisories: (asked, options) => osvAdvisories(COMPOSER, asked, options),
|
|
42
50
|
}
|
|
43
51
|
|
|
44
|
-
async function getVuln(id) {
|
|
52
|
+
export async function getVuln(id) {
|
|
45
53
|
const record = await request(buildUrl(OSV_API, ['v1', 'vulns', id]), { as: 'json' })
|
|
46
54
|
assert.ok(record?.id === id, `advisories: OSV answered for ${show(record?.id)}, not ${id}`)
|
|
47
55
|
assert.ok((record.aliases === undefined || isStrings(record.aliases)) && (record.summary === undefined || isText(record.summary))
|
|
@@ -49,7 +57,7 @@ async function getVuln(id) {
|
|
|
49
57
|
return { ...record, aliases: (record.aliases ?? []).filter(isOsvId) }
|
|
50
58
|
}
|
|
51
59
|
|
|
52
|
-
function toAdvisory(ecosystem, name, versions, record) {
|
|
60
|
+
function toAdvisory(ecosystem, name, versions, record, details) {
|
|
53
61
|
const { aliases } = record
|
|
54
62
|
const ghsas = aliases.filter(isGhsa)
|
|
55
63
|
const ghsa = isGhsa(record.id) ? record.id : (ghsas.length === 1 ? ghsas[0] : undefined)
|
|
@@ -61,8 +69,10 @@ function toAdvisory(ecosystem, name, versions, record) {
|
|
|
61
69
|
source: 'osv',
|
|
62
70
|
id: record.id,
|
|
63
71
|
...(ghsa && { ghsa }),
|
|
72
|
+
url: ghsa ? advisoryUrl(ghsa) : osvPage(record.id),
|
|
64
73
|
aliases,
|
|
65
74
|
...(record.summary && { title: record.summary }),
|
|
75
|
+
...(details && detailsOf(record.details, record.id)),
|
|
66
76
|
...metrics({ severity: record.database_specific?.severity, vector, cwe: record.database_specific?.cwe_ids }),
|
|
67
77
|
...(INFORMATIONAL.has(informational) && { informational }),
|
|
68
78
|
versions,
|
|
@@ -70,15 +80,16 @@ function toAdvisory(ecosystem, name, versions, record) {
|
|
|
70
80
|
}
|
|
71
81
|
|
|
72
82
|
// OSV's batch query matches versions on its side, but answers ids only:
|
|
73
|
-
// each record is fetched once after.
|
|
74
|
-
// publishes comes back under
|
|
75
|
-
//
|
|
83
|
+
// each record is fetched once after. An advisory more than one database
|
|
84
|
+
// publishes comes back under each one's id, the records naming each other
|
|
85
|
+
// as aliases, either way round, directly or through others: one group, in
|
|
86
|
+
// which each version is reported under the first answered for it, and
|
|
87
|
+
// each record has the others' ids as aliases.
|
|
76
88
|
async function osvAdvisories(ecosystem, asked, options) {
|
|
77
|
-
const { osv, github, lookUp, covers
|
|
89
|
+
const { osv, github, lookUp, covers } = ecosystem
|
|
78
90
|
const list = [...asked].flatMap(([name, versions]) => versions.map((version) => ({ name, version })))
|
|
79
91
|
const hits = new Map() // id → name → versions, in `list` order
|
|
80
|
-
for (
|
|
81
|
-
const chunk = list.slice(i, i + QUERIES_PER_REQUEST)
|
|
92
|
+
for (const chunk of chunks(list, QUERIES_PER_REQUEST)) {
|
|
82
93
|
const body = { queries: chunk.map(({ name, version }) => ({ package: { name, ecosystem: osv }, version })) }
|
|
83
94
|
const answer = await request(QUERY_URL, { method: 'POST', body, as: 'json' })
|
|
84
95
|
assert.ok(Array.isArray(answer?.results) && answer.results.length === chunk.length, 'advisories: expected one OSV result per query')
|
|
@@ -86,21 +97,28 @@ async function osvAdvisories(ecosystem, asked, options) {
|
|
|
86
97
|
const { name, version } = chunk[j]
|
|
87
98
|
const vulns = result?.vulns ?? []
|
|
88
99
|
assert.ok(result && result.next_page_token === undefined && Array.isArray(vulns) && vulns.every((vuln) => isOsvId(vuln?.id)), `advisories: malformed OSV result for ${name}@${version}`)
|
|
89
|
-
for (const { id } of vulns
|
|
100
|
+
for (const { id } of vulns) {
|
|
90
101
|
const byName = hits.get(id) ?? hits.set(id, new Map()).get(id)
|
|
91
102
|
byName.set(name, (byName.get(name) ?? new Set()).add(version))
|
|
92
103
|
}
|
|
93
104
|
}
|
|
94
105
|
}
|
|
95
106
|
const records = (await pool([...hits.keys()], RECORDS_AT_ONCE, getVuln)).filter((record) => !record.withdrawn)
|
|
96
|
-
const
|
|
107
|
+
const groups = new Map()
|
|
108
|
+
for (const record of records) {
|
|
109
|
+
if (groups.has(record)) continue
|
|
110
|
+
const group = [record]
|
|
111
|
+
for (const member of group) group.push(...records.filter((other) => !group.includes(other) && named(member, other)))
|
|
112
|
+
for (const member of group) groups.set(member, group)
|
|
113
|
+
}
|
|
97
114
|
const rows = []
|
|
98
115
|
for (const record of records) {
|
|
116
|
+
const group = groups.get(record)
|
|
117
|
+
const above = group.filter((other) => first(other, record) < 0)
|
|
118
|
+
const aliases = [...new Set([...record.aliases, ...group.map((other) => other.id)])].filter((id) => id !== record.id)
|
|
99
119
|
for (const [name, versions] of hits.get(record.id)) {
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
const rest = [...versions].filter((version) => !shadowed.includes(version))
|
|
103
|
-
if (rest.length > 0) rows.push(toAdvisory(osv, name, rest, record))
|
|
120
|
+
const rest = [...versions].filter((version) => !above.some((other) => hits.get(other.id).get(name)?.has(version)))
|
|
121
|
+
if (rest.length > 0) rows.push(toAdvisory(osv, name, rest, { ...record, aliases }, options.details))
|
|
104
122
|
}
|
|
105
123
|
}
|
|
106
124
|
rows.sort((a, b) => order(a.name, b.name) || order(a.id, b.id))
|