@preventive/upstream 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/advisories.d.ts +52 -20
- package/github.d.ts +1 -1
- package/npm.d.ts +6 -1
- package/package.json +2 -2
- package/src/advisories/github.js +34 -15
- package/src/advisories/index.js +55 -4
- package/src/advisories/npm.js +33 -6
- package/src/advisories/osv.js +10 -5
- package/src/http.js +1 -1
- package/src/npm/tarball.js +8 -4
package/advisories.d.ts
CHANGED
|
@@ -50,23 +50,43 @@ export interface AdvisoryOptions {
|
|
|
50
50
|
// and its rows take OSV's record of their GHSA, one more request each
|
|
51
51
|
// (no GitHub client needed).
|
|
52
52
|
details?: boolean
|
|
53
|
+
// Also asks OSV of each npm package's versions, as cargo and composer
|
|
54
|
+
// packages always are, for what the registry does not have: above all
|
|
55
|
+
// the malicious packages' records (MAL-...), which it has none of, as
|
|
56
|
+
// those of npm releases taken down that GitHub's database has no
|
|
57
|
+
// advisory of. A version a registry row reports under the same advisory,
|
|
58
|
+
// its id among an OSV record's id and aliases, is left to that row,
|
|
59
|
+
// which takes the record's ids among its aliases; the versions a record
|
|
60
|
+
// of the row's own id covers beyond the registry's range join it too.
|
|
61
|
+
// Sends each npm package's name and versions to OSV, so not for a
|
|
62
|
+
// private package.
|
|
63
|
+
osv?: boolean
|
|
53
64
|
}
|
|
54
65
|
|
|
55
66
|
// One advisory on one package, with the versions asked about that it
|
|
56
|
-
// covers (never empty)
|
|
67
|
+
// covers (never empty): one row an id, however many of its sources answer
|
|
68
|
+
// it, or however often one does.
|
|
57
69
|
export interface Advisory {
|
|
58
70
|
ecosystem: Ecosystem
|
|
59
71
|
name: string
|
|
60
|
-
// `registry`: npm's, as `npm audit` has it
|
|
61
|
-
//
|
|
62
|
-
//
|
|
63
|
-
//
|
|
64
|
-
//
|
|
65
|
-
//
|
|
66
|
-
//
|
|
67
|
-
//
|
|
68
|
-
//
|
|
69
|
-
//
|
|
72
|
+
// `registry`: npm's, as `npm audit` has it. `osv`: OSV's records for
|
|
73
|
+
// cargo and composer, and with `osv` for npm, the versions they cover
|
|
74
|
+
// that no registry row reports under the same advisory; RustSec's,
|
|
75
|
+
// GitHub's and malicious packages' (MAL-...) among them; of the records
|
|
76
|
+
// naming each other as aliases, directly or through others, a version is
|
|
77
|
+
// reported under one only: RustSec's, else the GHSA, else the first by
|
|
78
|
+
// id. Each has the others' ids among its aliases, and stays a row of its
|
|
79
|
+
// own, as its source rates it.
|
|
80
|
+
// `repository`: published on the package's repository, holding only the
|
|
81
|
+
// versions the others' answer does not report under that GHSA. The only
|
|
82
|
+
// source for `soldeer` and `github`, where every range counts, whichever
|
|
83
|
+
// package it names.
|
|
84
|
+
// An id answered more than once -- by npm's registry once a range, by
|
|
85
|
+
// OSV's record of it or its repository for versions beyond the registry's
|
|
86
|
+
// range, by its repository once a range -- is one row: the registry's,
|
|
87
|
+
// else OSV's, else the repository's, with what it lacks of theirs (title,
|
|
88
|
+
// details), the worst severity of them, the highest CVSS score with its
|
|
89
|
+
// own vector, and all their ranges, CWEs and versions.
|
|
70
90
|
source: 'registry' | 'osv' | 'repository'
|
|
71
91
|
// A GHSA, RUSTSEC-..., MAL-..., DRUPAL-CORE-... or another OSV id, or
|
|
72
92
|
// npm:<id> for a registry row without a GHSA.
|
|
@@ -81,6 +101,8 @@ export interface Advisory {
|
|
|
81
101
|
// (https://osv.dev/vulnerability/RUSTSEC-...). Built, never fetched: only
|
|
82
102
|
// a registry row without a GHSA (npm:<id>) has none.
|
|
83
103
|
url?: string
|
|
104
|
+
// The advisory's other ids: an `osv` row's record's own and its group's,
|
|
105
|
+
// and with `osv`, a registry row's OSV records' of it.
|
|
84
106
|
aliases: string[]
|
|
85
107
|
title?: string
|
|
86
108
|
// With `details`, the advisory's full text, Markdown as published: an
|
|
@@ -96,11 +118,17 @@ export interface Advisory {
|
|
|
96
118
|
cvssVector?: string
|
|
97
119
|
cwe: string[]
|
|
98
120
|
// As written: npm's syntax from the registry, GitHub's
|
|
99
|
-
// (`>= 1.0.0, < 1.2.6`) from a repository; OSV
|
|
121
|
+
// (`>= 1.0.0, < 1.2.6`) from a repository; OSV's records have none. A
|
|
100
122
|
// repository range semver cannot read, or none, covers every version.
|
|
123
|
+
// Each range its id was answered with that covers an asked version,
|
|
124
|
+
// joined by ` || `; none ('') where one is none. A version only OSV
|
|
125
|
+
// reports may be outside it: hermes-engine's GHSA-f5x2-xv93-4p23 is
|
|
126
|
+
// `<=0.4.3` on npm, the last version known affected, and fixed in 0.5.2
|
|
127
|
+
// by OSV's record of it, which reports 0.5.0 too.
|
|
101
128
|
range?: string
|
|
102
129
|
// RustSec's kind for an advisory that is not a vulnerability as such:
|
|
103
|
-
// `unmaintained`, `unsound`, `notice`.
|
|
130
|
+
// `unmaintained`, `unsound`, `notice`. For an id answered more than once,
|
|
131
|
+
// only where each answer gives it.
|
|
104
132
|
informational?: string
|
|
105
133
|
versions: string[]
|
|
106
134
|
}
|
|
@@ -110,14 +138,18 @@ export interface Advisory {
|
|
|
110
138
|
// merged, and must not name two repositories. npm is asked 250 names a
|
|
111
139
|
// request, OSV 1000 versions, Soldeer one project, and with `details`, OSV
|
|
112
140
|
// one GHSA among npm's rows; a project Soldeer does not have or that names
|
|
113
|
-
// no GitHub repository,
|
|
114
|
-
//
|
|
115
|
-
//
|
|
116
|
-
//
|
|
141
|
+
// no GitHub repository, a repository gone or blocked, and one GitHub
|
|
142
|
+
// redirects anywhere but to a repository by its id, add nothing, and a
|
|
143
|
+
// GHSA OSV does not have adds no `details`. A repository renamed or
|
|
144
|
+
// transferred, which GitHub redirects to it by its id, is listed by the
|
|
145
|
+
// name it has now, as getRepo's `followRedirect` (github.js) finds it, its
|
|
146
|
+
// rows' `url` on it. Any other failure, a malformed answer, or one about
|
|
147
|
+
// something not asked, throws: nothing is left out quietly.
|
|
117
148
|
// A repository's listing is kept for 90 minutes, as one entry for every
|
|
118
|
-
// spelling of
|
|
119
|
-
//
|
|
120
|
-
//
|
|
149
|
+
// spelling of the name it is asked by, one renamed or transferred too,
|
|
150
|
+
// holding only what rows are made from, `details` included: in
|
|
151
|
+
// setCacheDir's cache gzipped, in a `cache` store as it is, or with
|
|
152
|
+
// `cache` false, nowhere. 90 minutes is how late an advisory its
|
|
121
153
|
// maintainer publishes can be seen. One gone or malformed is not kept.
|
|
122
154
|
// Sorted by ecosystem and name. Versions are matched by npm's semver, from
|
|
123
155
|
// the npm beside node, or by the semver peer where there is no npm.
|
package/github.d.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// encoded segments and has to come back out of URL parsing unchanged.
|
|
8
8
|
// Redirects are refused, so an answer is about the repo asked for, except
|
|
9
9
|
// the tarballs' and, where asked, getRepo's for a repo renamed or
|
|
10
|
-
// transferred. A response is read up to a size limit (
|
|
10
|
+
// transferred. A response is read up to a size limit (128 MiB of JSON, 128
|
|
11
11
|
// MiB of a file, 512 MiB of a tarball) and within a timeout (30 seconds, 5
|
|
12
12
|
// minutes for a tarball), and what it says goes into an error message
|
|
13
13
|
// escaped.
|
package/npm.d.ts
CHANGED
|
@@ -127,7 +127,11 @@ export interface Dist {
|
|
|
127
127
|
// `bugs` an object, of which only the strings named below are read. Each is
|
|
128
128
|
// absent where the document has none, or has another shape, or an object
|
|
129
129
|
// with none of those strings. All of it is the publisher's word, held to no
|
|
130
|
-
// repository.
|
|
130
|
+
// repository. `unpackedSize` is the document's dist.unpackedSize: the bytes
|
|
131
|
+
// of the files in the tarball, as the registry counted them when the
|
|
132
|
+
// version was published, held to no tarball. It is absent where the
|
|
133
|
+
// document has none, as for a version published before the registry
|
|
134
|
+
// counted them, or has one that is not a whole number of bytes.
|
|
131
135
|
export interface PackageMeta {
|
|
132
136
|
name: string
|
|
133
137
|
version: string
|
|
@@ -136,6 +140,7 @@ export interface PackageMeta {
|
|
|
136
140
|
repository?: string | PackageRepository
|
|
137
141
|
homepage?: string
|
|
138
142
|
bugs?: string | PackageBugs
|
|
143
|
+
unpackedSize?: number
|
|
139
144
|
}
|
|
140
145
|
|
|
141
146
|
export interface PackageRepository {
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@preventive/upstream",
|
|
3
|
-
"version": "1.0
|
|
4
|
-
"description": "Minimal npm, crates.io
|
|
3
|
+
"version": "1.2.0",
|
|
4
|
+
"description": "Minimal npm, crates.io, Soldeer, GitHub and git checkout clients",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
7
7
|
"./advisories.js": {
|
package/src/advisories/github.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import assert from 'node:assert/strict'
|
|
2
2
|
|
|
3
|
-
import { assertRepo, assertSoldeerName, assertSoldeerVersion, assertion, isGhsa, isRefName, isStrings } from '../args.js'
|
|
3
|
+
import { assertRepo, assertSoldeerName, assertSoldeerVersion, assertion, isGhsa, isRefName, isRepo, isStrings } from '../args.js'
|
|
4
4
|
import { readRecord, writeRecord } from '../cache.js'
|
|
5
5
|
import { isGone } from '../github/client.js'
|
|
6
|
-
import { recover } from '../http.js'
|
|
6
|
+
import { HttpError, recover } from '../http.js'
|
|
7
7
|
import { pool } from '../pool.js'
|
|
8
8
|
import { isExactVersion } from '../semver.js'
|
|
9
9
|
import { advisoryUrl, covered, detailsOf, isText, mayBeInRange, metrics } from './common.js'
|
|
@@ -21,7 +21,7 @@ const isRepoAdvisory = (advisory) => advisory && typeof advisory === 'object' &&
|
|
|
21
21
|
&& advisory.state === 'published' && isText(advisory.summary)
|
|
22
22
|
&& (advisory.vulnerabilities == null || Array.isArray(advisory.vulnerabilities)) && (advisory.cwe_ids == null || isStrings(advisory.cwe_ids))
|
|
23
23
|
|
|
24
|
-
export const assertClient = assertion('a GitHub client from createClient', (value) => typeof value?.listRepoAdvisories === 'function')
|
|
24
|
+
export const assertClient = assertion('a GitHub client from createClient', (value) => typeof value?.listRepoAdvisories === 'function' && typeof value.getRepo === 'function')
|
|
25
25
|
|
|
26
26
|
// A maintainer may give a vulnerable range only its lower bound, and the
|
|
27
27
|
// version that fixed it apart (`>= 5.0.0-beta.1`, patched in `5.0.0-rc.2`),
|
|
@@ -67,18 +67,37 @@ const isDigest = (advisory) => advisory && typeof advisory === 'object' && isGhs
|
|
|
67
67
|
&& (advisory.description === undefined || isText(advisory.description))
|
|
68
68
|
&& Array.isArray(advisory.ranges) && advisory.ranges.every((entry) => typeof entry?.range === 'string')
|
|
69
69
|
|
|
70
|
-
//
|
|
71
|
-
//
|
|
72
|
-
|
|
70
|
+
// GitHub answers for a repository renamed or transferred with a redirect,
|
|
71
|
+
// which getRepo follows to the repository it is now, listed by that name.
|
|
72
|
+
const isRedirect = (err) => err instanceof HttpError && err.status >= 300 && err.status < 400
|
|
73
|
+
|
|
74
|
+
async function listingOf(github, repo) {
|
|
75
|
+
try {
|
|
76
|
+
return { repo, list: await github.listRepoAdvisories({ repo }) }
|
|
77
|
+
} catch (err) {
|
|
78
|
+
if (!isRedirect(err)) throw err
|
|
79
|
+
}
|
|
80
|
+
const moved = (await github.getRepo({ repo, followRedirect: true })).full_name
|
|
81
|
+
return { repo: moved, list: await github.listRepoAdvisories({ repo: moved }) }
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// A repository's listing, digested, through the cache, and the repository
|
|
85
|
+
// it is of: GitHub's names are case-insensitive, so one entry answers every
|
|
86
|
+
// spelling, and one renamed or transferred is kept under the name asked,
|
|
87
|
+
// with `repo` the one it has now. A repository gone or blocked, or
|
|
88
|
+
// redirected anywhere getRepo does not follow, has none, and that is not
|
|
89
|
+
// kept.
|
|
73
90
|
async function listAdvisories(github, repo, cache) {
|
|
74
91
|
const name = repo.toLowerCase()
|
|
75
92
|
const entry = await readRecord(DIR, name, cache, LISTING_TTL_MS)
|
|
76
|
-
if (entry?.v === VERSION && Array.isArray(entry.advisories) && entry.advisories.every(isDigest))
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
const
|
|
80
|
-
if (
|
|
81
|
-
|
|
93
|
+
if (entry?.v === VERSION && (entry.repo === undefined || isRepo(entry.repo)) && Array.isArray(entry.advisories) && entry.advisories.every(isDigest)) {
|
|
94
|
+
return { repo: entry.repo ?? repo, advisories: entry.advisories }
|
|
95
|
+
}
|
|
96
|
+
const found = await listingOf(github, repo).catch(recover((err) => isGone(err) || isRedirect(err), null))
|
|
97
|
+
if (found === null) return { repo, advisories: [] }
|
|
98
|
+
const advisories = digest(found.repo, found.list)
|
|
99
|
+
if (advisories.every(isDigest)) await writeRecord(DIR, name, { v: VERSION, ...(found.repo !== repo && { repo: found.repo }), advisories }, cache)
|
|
100
|
+
return { repo: found.repo, advisories }
|
|
82
101
|
}
|
|
83
102
|
|
|
84
103
|
// GitHub's `>= 1.0.0, < 1.2.6` is npm's with the commas dropped.
|
|
@@ -109,10 +128,10 @@ function fromRepository(repo, name, advisory, range, asked, { covers, details })
|
|
|
109
128
|
async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers, details, cache }) {
|
|
110
129
|
// GitHub's names are case-insensitive: one spelling asks for all.
|
|
111
130
|
const namesOf = Map.groupBy([...asked.keys()].filter(repoOf), (name) => repoOf(name).toLowerCase())
|
|
112
|
-
const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({
|
|
131
|
+
const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ names, ...(await listAdvisories(github, repoOf(names[0]), cache)) }))
|
|
113
132
|
const rows = new Map()
|
|
114
|
-
for (const { repo, names,
|
|
115
|
-
for (const advisory of
|
|
133
|
+
for (const { repo, names, advisories } of listed) {
|
|
134
|
+
for (const advisory of advisories) {
|
|
116
135
|
for (const { range, ...pkg } of advisory.ranges) {
|
|
117
136
|
for (const name of names.filter((candidate) => takes(candidate, pkg))) {
|
|
118
137
|
rows.set(`${name} ${advisory.ghsa} ${range}`, fromRepository(repo, name, advisory, range, asked.get(name), { covers, details }))
|
package/src/advisories/index.js
CHANGED
|
@@ -34,8 +34,59 @@ function collect(packages) {
|
|
|
34
34
|
return byEcosystem
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
// Which of an advisory's rows stands for it: npm's registry's, then OSV's,
|
|
38
|
+
// then its repository's.
|
|
39
|
+
const SOURCES = ['registry', 'osv', 'repository']
|
|
40
|
+
const SEVERITIES = ['critical', 'high', 'moderate', 'low', 'info']
|
|
41
|
+
const worse = (a, b) => (b === undefined || (a !== undefined && SEVERITIES.indexOf(a) < SEVERITIES.indexOf(b)) ? a : b)
|
|
42
|
+
|
|
43
|
+
// An advisory's rows as one: the first's by source, what it lacks from the
|
|
44
|
+
// first of the rest that has it, the others' aliases, and of them all the
|
|
45
|
+
// worst severity, the highest CVSS score with its own vector (else the first
|
|
46
|
+
// vector), the ranges (` || `), CWEs and versions (in the order `asked` has
|
|
47
|
+
// them). RustSec's kind of advisory that is not a vulnerability as such only
|
|
48
|
+
// where each of them has that kind: a row without one is a vulnerability.
|
|
49
|
+
function joinRows(rows, asked) {
|
|
50
|
+
const [head, ...rest] = rows.toSorted((a, b) => SOURCES.indexOf(a.source) - SOURCES.indexOf(b.source))
|
|
51
|
+
const all = [head, ...rest]
|
|
52
|
+
const ranges = [...new Set(all.map((row) => row.range).filter((range) => range !== undefined))]
|
|
53
|
+
const versions = new Set(all.flatMap((row) => row.versions))
|
|
54
|
+
const joined = {
|
|
55
|
+
...head,
|
|
56
|
+
aliases: [...new Set(all.flatMap((row) => row.aliases))].filter((id) => id !== head.id),
|
|
57
|
+
cwe: [...new Set(all.flatMap((row) => row.cwe))],
|
|
58
|
+
versions: asked.filter((version) => versions.has(version)),
|
|
59
|
+
}
|
|
60
|
+
if (rest.some((row) => row.informational !== head.informational)) delete joined.informational
|
|
61
|
+
for (const key of ['title', 'details']) {
|
|
62
|
+
const from = joined[key] === undefined && rest.find((row) => row[key] !== undefined)
|
|
63
|
+
if (from) joined[key] = from[key]
|
|
64
|
+
}
|
|
65
|
+
const severity = all.map((row) => row.severity).reduce(worse)
|
|
66
|
+
if (severity !== undefined) joined.severity = severity
|
|
67
|
+
const scored = all.filter((row) => row.cvss !== undefined).toSorted((a, b) => b.cvss - a.cvss)[0] ?? all.find((row) => row.cvssVector !== undefined)
|
|
68
|
+
delete joined.cvss
|
|
69
|
+
delete joined.cvssVector
|
|
70
|
+
if (scored?.cvss !== undefined) joined.cvss = scored.cvss
|
|
71
|
+
if (scored?.cvssVector !== undefined) joined.cvssVector = scored.cvssVector
|
|
72
|
+
// A range of none ('') covers every version, as they then do together.
|
|
73
|
+
if (ranges.length > 0) joined.range = ranges.includes('') ? '' : ranges.join(' || ')
|
|
74
|
+
return joined
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// One row for each advisory id on a package, where its sources answer more
|
|
78
|
+
// than one: npm's registry one per range, and OSV's record of it, or its
|
|
79
|
+
// repository, another for the versions they cover beyond that. Records of
|
|
80
|
+
// other ids that name it as an alias stay rows of their own, each version
|
|
81
|
+
// reported under one, as their sources rate them. Rows keep the place of the
|
|
82
|
+
// first answered.
|
|
83
|
+
function oneRowPerAdvisory(rows, asked) {
|
|
84
|
+
const byId = Map.groupBy(rows, (row) => `${row.name}\0${row.id}`)
|
|
85
|
+
return [...byId.values()].map((same) => (same.length === 1 ? same[0] : joinRows(same, asked.get(same[0].name))))
|
|
86
|
+
}
|
|
87
|
+
|
|
37
88
|
export async function advisories(packages, options = {}) {
|
|
38
|
-
assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean), cache: optional(assertCache) })
|
|
89
|
+
assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean), osv: optional(assertBoolean), cache: optional(assertCache) })
|
|
39
90
|
assert.ok(!options.repoAdvisories || options.github, 'advisories: repoAdvisories needs a github client')
|
|
40
91
|
const byEcosystem = collect(packages)
|
|
41
92
|
for (const ecosystem of byEcosystem.keys()) assert.ok(!ECOSYSTEMS[ecosystem].repositoryOnly || options.github, `advisories: ${ecosystem} packages need a github client`)
|
|
@@ -44,8 +95,8 @@ export async function advisories(packages, options = {}) {
|
|
|
44
95
|
const names = [...named.keys()].toSorted()
|
|
45
96
|
const asked = new Map(names.map((name) => [name, [...named.get(name).versions].toSorted(compare)]))
|
|
46
97
|
const known = new Map(names.filter((name) => named.get(name).github).map((name) => [name, named.get(name).github]))
|
|
47
|
-
const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, known, cache: options.cache })
|
|
48
|
-
return rows.map((row) => ({ ecosystem, ...row }))
|
|
98
|
+
const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, osv: options.osv === true, known, cache: options.cache })
|
|
99
|
+
return oneRowPerAdvisory(rows.filter((row) => row.versions.length > 0), asked).map((row) => ({ ecosystem, ...row }))
|
|
49
100
|
}))
|
|
50
|
-
return found.flat().
|
|
101
|
+
return found.flat().toSorted((a, b) => order(a.ecosystem, b.ecosystem) || order(a.name, b.name))
|
|
51
102
|
}
|
package/src/advisories/npm.js
CHANGED
|
@@ -6,7 +6,7 @@ import { chunks, pool } from '../pool.js'
|
|
|
6
6
|
import { compareVersions } from '../semver.js'
|
|
7
7
|
import { advisoryUrl, covered, detailsOf, isText, metrics } from './common.js'
|
|
8
8
|
import { withRepositories } from './github.js'
|
|
9
|
-
import { RECORDS_AT_ONCE, getVuln } from './osv.js'
|
|
9
|
+
import { RECORDS_AT_ONCE, getVuln, osvRows } from './osv.js'
|
|
10
10
|
import { npmRepos } from './repos.js'
|
|
11
11
|
|
|
12
12
|
const BULK_URL = buildUrl(NPM_REGISTRY, ['-', 'npm', 'v1', 'security', 'advisories', 'bulk'])
|
|
@@ -52,21 +52,48 @@ async function registryAdvisories(asked) {
|
|
|
52
52
|
|
|
53
53
|
// The registry's rows carry no text: each GHSA's is OSV's record of it,
|
|
54
54
|
// GitHub's advisory database as published, asked once for the rows that
|
|
55
|
-
// cover an asked version
|
|
55
|
+
// cover an asked version, but where `osv` already read it. One OSV does
|
|
56
|
+
// not have yet has none.
|
|
56
57
|
async function withDetails(rows) {
|
|
57
|
-
const
|
|
58
|
+
const asks = (row) => row.source === 'registry' && row.details === undefined
|
|
59
|
+
const ghsas = [...new Set(rows.filter((row) => asks(row) && row.ghsa && row.versions.length > 0).map((row) => row.ghsa))]
|
|
58
60
|
const records = await pool(ghsas, RECORDS_AT_ONCE, (ghsa) => getVuln(ghsa).catch(recover(isNotFound, undefined)))
|
|
59
61
|
const text = new Map(ghsas.map((ghsa, i) => [ghsa, records[i]?.details]))
|
|
60
|
-
return rows.map((row) => ({ ...row, ...detailsOf(text.get(row.ghsa), row.ghsa) }))
|
|
62
|
+
return rows.map((row) => (asks(row) ? { ...row, ...detailsOf(text.get(row.ghsa), row.ghsa) } : row))
|
|
61
63
|
}
|
|
62
64
|
|
|
63
|
-
//
|
|
65
|
+
// With `osv`, OSV's records of the asked versions besides the registry's
|
|
66
|
+
// rows, the malicious packages' (MAL-...), which the registry has none
|
|
67
|
+
// of, among them. A version a registry row reports under the same
|
|
68
|
+
// advisory, the row's id one of an OSV row's ids or aliases, is left to
|
|
69
|
+
// that row, which takes the OSV row's ids among its aliases, and its text
|
|
70
|
+
// where it is the same record; an OSV row keeps the versions no such row
|
|
71
|
+
// reports, which advisories() joins to a registry row of its own id.
|
|
72
|
+
function withOsv(registry, osv) {
|
|
73
|
+
const rows = registry.map((row) => ({ ...row }))
|
|
74
|
+
const added = []
|
|
75
|
+
for (const record of osv) {
|
|
76
|
+
const ids = [record.id, ...record.aliases]
|
|
77
|
+
const same = rows.filter((row) => row.name === record.name && ids.includes(row.id) && row.versions.some((version) => record.versions.includes(version)))
|
|
78
|
+
for (const row of same) {
|
|
79
|
+
row.aliases = [...new Set([...row.aliases, ...ids])].filter((id) => id !== row.id)
|
|
80
|
+
if (record.id === row.id && record.details !== undefined) row.details = record.details
|
|
81
|
+
}
|
|
82
|
+
const versions = record.versions.filter((version) => !same.some((row) => row.versions.includes(version)))
|
|
83
|
+
if (versions.length > 0) added.push({ ...record, versions })
|
|
84
|
+
}
|
|
85
|
+
return [...rows, ...added]
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// What `npm audit` asks the registry, one row per vulnerable range, and
|
|
89
|
+
// with `osv`, what OSV has besides.
|
|
64
90
|
export const NPM = {
|
|
65
91
|
assertName: assertPackageName,
|
|
66
92
|
assertVersion: assertPackageVersion,
|
|
67
93
|
compare: compareVersions,
|
|
68
94
|
advisories: async (asked, options) => {
|
|
69
|
-
const
|
|
95
|
+
const [registry, osv] = await Promise.all([registryAdvisories(asked), options.osv ? osvRows({ osv: 'npm' }, asked, options) : []])
|
|
96
|
+
const rows = withOsv(registry, osv)
|
|
70
97
|
return await withRepositories(options.details ? await withDetails(rows) : rows, asked, options, { ecosystem: 'npm', lookUp: npmRepos })
|
|
71
98
|
},
|
|
72
99
|
}
|
package/src/advisories/osv.js
CHANGED
|
@@ -84,9 +84,9 @@ function toAdvisory(ecosystem, name, versions, record, details) {
|
|
|
84
84
|
// publishes comes back under each one's id, the records naming each other
|
|
85
85
|
// as aliases, either way round, directly or through others: one group, in
|
|
86
86
|
// which each version is reported under the first answered for it, and
|
|
87
|
-
// each record has the others' ids as aliases.
|
|
88
|
-
|
|
89
|
-
|
|
87
|
+
// each record has the others' ids as aliases. `osv` is OSV's name for the
|
|
88
|
+
// ecosystem.
|
|
89
|
+
export async function osvRows({ osv }, asked, { details }) {
|
|
90
90
|
const list = [...asked].flatMap(([name, versions]) => versions.map((version) => ({ name, version })))
|
|
91
91
|
const hits = new Map() // id -> name -> versions, in `list` order
|
|
92
92
|
for (const chunk of chunks(list, QUERIES_PER_REQUEST)) {
|
|
@@ -118,9 +118,14 @@ async function osvAdvisories(ecosystem, asked, options) {
|
|
|
118
118
|
const aliases = [...new Set([...record.aliases, ...group.map((other) => other.id)])].filter((id) => id !== record.id)
|
|
119
119
|
for (const [name, versions] of hits.get(record.id)) {
|
|
120
120
|
const rest = [...versions].filter((version) => !above.some((other) => hits.get(other.id).get(name)?.has(version)))
|
|
121
|
-
if (rest.length > 0) rows.push(toAdvisory(osv, name, rest, { ...record, aliases },
|
|
121
|
+
if (rest.length > 0) rows.push(toAdvisory(osv, name, rest, { ...record, aliases }, details))
|
|
122
122
|
}
|
|
123
123
|
}
|
|
124
124
|
rows.sort((a, b) => order(a.name, b.name) || order(a.id, b.id))
|
|
125
|
-
return
|
|
125
|
+
return rows
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
async function osvAdvisories(ecosystem, asked, options) {
|
|
129
|
+
const { github, lookUp, covers } = ecosystem
|
|
130
|
+
return await withRepositories(await osvRows(ecosystem, asked, options), asked, options, { ecosystem: github, lookUp, covers })
|
|
126
131
|
}
|
package/src/http.js
CHANGED
|
@@ -26,7 +26,7 @@ const isQuery = ([key, value]) => isQueryKey(key) && (key.endsWith('[]') ? Array
|
|
|
26
26
|
// A line break in a value would start a header of its own.
|
|
27
27
|
const isHeader = ([name, value]) => /^[A-Za-z][\w-]*$/u.test(name) && matches(/^[ -~]*$/u)(value)
|
|
28
28
|
const LIMITS = {
|
|
29
|
-
json: { bytes:
|
|
29
|
+
json: { bytes: 128 * 1024 * 1024, ms: 30_000 },
|
|
30
30
|
text: { bytes: 128 * 1024 * 1024, ms: 30_000 },
|
|
31
31
|
bytes: { bytes: MAX_BYTES, ms: 300_000 },
|
|
32
32
|
}
|
package/src/npm/tarball.js
CHANGED
|
@@ -73,24 +73,28 @@ async function npmCachedVersion(name, version, given) {
|
|
|
73
73
|
return undefined
|
|
74
74
|
}
|
|
75
75
|
|
|
76
|
+
const countOr = (value) => (Number.isSafeInteger(value) && value >= 0 ? value : undefined)
|
|
77
|
+
|
|
76
78
|
// The version's document: with the `dist` the caller has, npm's own cache's
|
|
77
79
|
// where it matches (npmCachedVersion), and neither `cache` nor the cache
|
|
78
80
|
// set is read or written; else as getVersion has it. Beside its dist, the
|
|
79
81
|
// document's gitHead where that is a full commit id, and its repository,
|
|
80
|
-
// homepage and bugs, in the shapes a package.json gives them
|
|
81
|
-
// dist is
|
|
82
|
+
// homepage and bugs, in the shapes a package.json gives them; and its
|
|
83
|
+
// dist's unpackedSize where that is a count. Each but dist is left out
|
|
84
|
+
// where there is none.
|
|
82
85
|
export async function getMeta(name, version, options = {}) {
|
|
83
86
|
assertPackage('getMeta', name, version)
|
|
84
87
|
assertArgs('getMeta', options, { dist: null, cache: optional(assertCache) })
|
|
85
88
|
const given = options.dist === undefined ? undefined : checkedDist('getMeta', name, version, options.dist)
|
|
86
89
|
const { dist, json } = (given && await npmCachedVersion(name, version, given)) ?? await getVersion('getMeta', name, version, given, options.cache)
|
|
87
|
-
const
|
|
90
|
+
const said = {
|
|
88
91
|
gitHead: isSha(own(json, 'gitHead')) ? json.gitHead : undefined,
|
|
89
92
|
repository: linkOf(own(json, 'repository'), ['type', 'url', 'directory']),
|
|
90
93
|
homepage: stringOr(own(json, 'homepage')),
|
|
91
94
|
bugs: linkOf(own(json, 'bugs'), ['url', 'email']),
|
|
95
|
+
unpackedSize: countOr(own(json.dist, 'unpackedSize')),
|
|
92
96
|
}
|
|
93
|
-
return { name, version, dist, ...Object.fromEntries(Object.entries(
|
|
97
|
+
return { name, version, dist, ...Object.fromEntries(Object.entries(said).filter(([, value]) => value !== undefined)) }
|
|
94
98
|
}
|
|
95
99
|
|
|
96
100
|
// Only the whole package's document has `time`: megabytes for some.
|