@preventive/upstream 1.0.0-alpha.8 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/advisories.d.ts +21 -7
- package/git.d.ts +2 -2
- package/npm.d.ts +117 -22
- package/npm.js +1 -1
- package/package.d.ts +5 -2
- package/package.json +11 -2
- package/src/advisories/github.js +87 -26
- package/src/advisories/index.js +4 -3
- package/src/advisories/osv.js +3 -3
- package/src/advisories/repos.js +21 -10
- package/src/args.js +1 -1
- package/src/cache.js +77 -24
- package/src/cargo/repos.js +8 -8
- package/src/download.js +8 -7
- package/src/git.js +1 -1
- package/src/npm/cacache.js +44 -0
- package/src/npm/registry.js +10 -4
- package/src/npm/repos.js +25 -13
- package/src/npm/tarball.js +99 -37
- package/src/npm/versions.js +97 -0
- package/src/package.js +24 -7
- package/src/remote.js +10 -3
- package/src/weak-edges.js +87 -0
- package/weak-edges.d.ts +36 -0
- package/weak-edges.js +1 -0
package/advisories.d.ts
CHANGED
|
@@ -2,8 +2,10 @@
|
|
|
2
2
|
// other.
|
|
3
3
|
|
|
4
4
|
import type { Client, RepoName } from './github.js'
|
|
5
|
+
import type { CacheStore } from './npm.js'
|
|
5
6
|
|
|
6
7
|
export { HttpError } from './npm.js'
|
|
8
|
+
export type { CacheStore } from './npm.js'
|
|
7
9
|
|
|
8
10
|
// As purl and stasis name them.
|
|
9
11
|
export type Ecosystem = 'npm' | 'cargo' | 'composer' | 'soldeer' | 'github'
|
|
@@ -36,9 +38,16 @@ export interface AdvisoryOptions {
|
|
|
36
38
|
// Also asks each npm, cargo and composer package's GitHub repository
|
|
37
39
|
// for its published advisories, which it has before GitHub reviews them
|
|
38
40
|
// into the databases above: the package's `github` where given, else
|
|
39
|
-
// the one npm's
|
|
40
|
-
//
|
|
41
|
+
// the one looked up. An npm package's is the one the document of its
|
|
42
|
+
// newest version asked names, kept for good as getMeta keeps it (none
|
|
43
|
+
// where the registry does not have that version); a crate's and a
|
|
44
|
+
// composer package's, the one crates.io or Packagist names, kept a
|
|
45
|
+
// month. Needs `github`.
|
|
41
46
|
repoAdvisories?: boolean
|
|
47
|
+
// Where what this call caches is kept, as CacheOptions (npm.js) has it:
|
|
48
|
+
// a repository's listing, and each package's repository looked up (an
|
|
49
|
+
// npm package's as its version's document). See `advisories`.
|
|
50
|
+
cache?: CacheStore | false
|
|
42
51
|
// Also returns each advisory's full text, as `details`. OSV's records
|
|
43
52
|
// and a repository's listing already carry it; npm's registry does not,
|
|
44
53
|
// and its rows take OSV's record of their GHSA, one more request each
|
|
@@ -53,7 +62,7 @@ export interface Advisory {
|
|
|
53
62
|
name: string
|
|
54
63
|
// `registry`: npm's, as `npm audit` has it, one row per vulnerable
|
|
55
64
|
// range. `osv`: OSV's records for cargo and composer, RustSec's, GitHub's
|
|
56
|
-
// and malicious packages' (MAL
|
|
65
|
+
// and malicious packages' (MAL-...) among them; of the records naming each
|
|
57
66
|
// other as aliases, directly or through others, a version is reported
|
|
58
67
|
// under one only: RustSec's, else the GHSA, else the first by id. Each
|
|
59
68
|
// has the others' ids among its aliases.
|
|
@@ -62,17 +71,17 @@ export interface Advisory {
|
|
|
62
71
|
// under that GHSA. The only source for `soldeer` and `github`, where
|
|
63
72
|
// every range counts, whichever package it names.
|
|
64
73
|
source: 'registry' | 'osv' | 'repository'
|
|
65
|
-
// A GHSA, RUSTSEC
|
|
74
|
+
// A GHSA, RUSTSEC-..., MAL-..., DRUPAL-CORE-... or another OSV id, or
|
|
66
75
|
// npm:<id> for a registry row without a GHSA.
|
|
67
76
|
id: string
|
|
68
77
|
// The id itself, or an OSV record's one GHSA alias.
|
|
69
78
|
ghsa?: string
|
|
70
79
|
// GitHub's page for the GHSA, with it: a `repository` row's on its
|
|
71
|
-
// repository (https://github.com/owner/name/security/advisories/GHSA
|
|
80
|
+
// repository (https://github.com/owner/name/security/advisories/GHSA-...),
|
|
72
81
|
// which has it before GitHub reviews it into its advisory database; any
|
|
73
|
-
// other's in that database (https://github.com/advisories/GHSA
|
|
82
|
+
// other's in that database (https://github.com/advisories/GHSA-...). An
|
|
74
83
|
// `osv` row without one has osv.dev's page for its record
|
|
75
|
-
// (https://osv.dev/vulnerability/RUSTSEC
|
|
84
|
+
// (https://osv.dev/vulnerability/RUSTSEC-...). Built, never fetched: only
|
|
76
85
|
// a registry row without a GHSA (npm:<id>) has none.
|
|
77
86
|
url?: string
|
|
78
87
|
aliases: string[]
|
|
@@ -108,6 +117,11 @@ export interface Advisory {
|
|
|
108
117
|
// nothing, and a GHSA OSV does not have adds no `details`. Any other
|
|
109
118
|
// failure, a malformed answer, or one about something not asked, throws:
|
|
110
119
|
// nothing is left out quietly.
|
|
120
|
+
// A repository's listing is kept for 90 minutes, as one entry for every
|
|
121
|
+
// spelling of its name, holding only what rows are made from, `details`
|
|
122
|
+
// included: in setCacheDir's cache gzipped, in a `cache` store as it is,
|
|
123
|
+
// or with `cache` false, nowhere. 90 minutes is how late an advisory its
|
|
124
|
+
// maintainer publishes can be seen. One gone or malformed is not kept.
|
|
111
125
|
// Sorted by ecosystem and name. Versions are matched by npm's semver, from
|
|
112
126
|
// the npm beside node, or by the semver peer where there is no npm.
|
|
113
127
|
export function advisories(packages: Iterable<Package>, options?: AdvisoryOptions): Promise<Advisory[]>
|
package/git.d.ts
CHANGED
|
@@ -6,8 +6,8 @@ export interface GitCheckout {
|
|
|
6
6
|
// `owner/name`, when the origin remote is a GitHub one. Never the
|
|
7
7
|
// remote's URL, which can name a private host or carry a token.
|
|
8
8
|
github?: string
|
|
9
|
-
// Where the directory sits in its checkout, `/`-separated;
|
|
10
|
-
//
|
|
9
|
+
// Where the directory sits in its checkout, `/`-separated; `''` at the
|
|
10
|
+
// root.
|
|
11
11
|
directory?: string
|
|
12
12
|
// `https://github.com/<github>`, whenever there is a `github`.
|
|
13
13
|
url?: string
|
package/npm.d.ts
CHANGED
|
@@ -7,8 +7,9 @@
|
|
|
7
7
|
// a timeout, as github.d.ts describes, and lookups for many names go to
|
|
8
8
|
// the registry eight at a time.
|
|
9
9
|
//
|
|
10
|
-
// Where
|
|
11
|
-
//
|
|
10
|
+
// Where PREVENTIVE_MEASURES_NPM_TOKEN, STASIS_NPM_TOKEN or NPM_TOKEN is
|
|
11
|
+
// set, the first of them not empty, in that order, is sent with the
|
|
12
|
+
// requests for a scoped package, so a private one can be read.
|
|
12
13
|
|
|
13
14
|
// Where a program called `name` keeps its cache for this user, as each
|
|
14
15
|
// platform has it, read from the environment when called. macOS:
|
|
@@ -32,14 +33,45 @@ export const defaultCacheDir: string | undefined
|
|
|
32
33
|
// cache: every read of it misses and every write is skipped. getTarball
|
|
33
34
|
// reads ours all the same, set or not.
|
|
34
35
|
//
|
|
35
|
-
// The directory is trusted: a package's repo
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
//
|
|
36
|
+
// The directory is trusted: a package's repo, as recorded or as a version
|
|
37
|
+
// document kept here names it, and for 90 minutes a repository's published
|
|
38
|
+
// advisories (advisories.js), are answered from it as they were written,
|
|
39
|
+
// with no request. Point it only at storage that
|
|
40
|
+
// nothing less trusted than the caller can write, never a cache shared
|
|
41
|
+
// with or restored from lower-trust jobs. Tarballs, crates (cargo.js), zips
|
|
39
42
|
// (soldeer.js) and tree tarballs (github.js) don't rely on this, since
|
|
40
|
-
// they are checked against their integrity on every call
|
|
43
|
+
// they are checked against their integrity on every call; and the version
|
|
44
|
+
// documents kept here (getMeta) never supply an integrity, only confirm
|
|
45
|
+
// one a caller gives (verifyDist, and getMeta with a `dist`). A call's `cache` option, where it has
|
|
46
|
+
// one, keeps what it caches elsewhere, or writes none of it here.
|
|
41
47
|
export function setCacheDir(dir?: string | false): void
|
|
42
48
|
|
|
49
|
+
// A store of the caller's, a database's say, for what would otherwise be
|
|
50
|
+
// kept in setCacheDir's cache: `read` answers the value last written
|
|
51
|
+
// under that `type` and `key`, or null or undefined for none. A `type` is
|
|
52
|
+
// what is kept and a `key` which one: `npm/versions` and `npm/tarballs`
|
|
53
|
+
// with `<name>@<version>` for a version's document and tarball; and for
|
|
54
|
+
// advisories.js, `github/advisories` with the repository's `owner/name` in
|
|
55
|
+
// lowercase for its listing, and `cargo/repos`, `composer/repos` and
|
|
56
|
+
// `soldeer/repos` with a package's name for the repository it names.
|
|
57
|
+
// Values are plain JSON data, which `read` may answer as written or as a
|
|
58
|
+
// copy, but a tarball's are its bytes, a Uint8Array. Whatever it answers
|
|
59
|
+
// is checked as a cached file is: one malformed, stale or kept differently
|
|
60
|
+
// is a miss, and a tarball's bytes that do not match throw. A rejection
|
|
61
|
+
// from either is the call's. It is trusted as setCacheDir's directory is.
|
|
62
|
+
export interface CacheStore {
|
|
63
|
+
read(type: string, key: string): Promise<unknown>
|
|
64
|
+
write(type: string, key: string, value: unknown): Promise<void>
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// Where a call keeps what it caches: in setCacheDir's cache where `cache`
|
|
68
|
+
// is left out; in a store of the caller's in place of it; or with `false`,
|
|
69
|
+
// nowhere, though setCacheDir's cache is read all the same. Other tools'
|
|
70
|
+
// caches and ours are read for tarballs whatever it is (getTarball).
|
|
71
|
+
export interface CacheOptions {
|
|
72
|
+
cache?: CacheStore | false
|
|
73
|
+
}
|
|
74
|
+
|
|
43
75
|
// A failed request: `status` is the HTTP status the registry answered
|
|
44
76
|
// with (404 for a package or version it does not have).
|
|
45
77
|
export class HttpError extends Error {
|
|
@@ -47,11 +79,11 @@ export class HttpError extends Error {
|
|
|
47
79
|
status: number
|
|
48
80
|
}
|
|
49
81
|
|
|
50
|
-
// A package's GitHub repo as its registry metadata names it
|
|
82
|
+
// A package's GitHub repo as its registry metadata names it -- getRepo
|
|
51
83
|
// (package.js) over its `latest` document: `github` is `owner/name`, and
|
|
52
|
-
// `directory` is where in the repo the package sits,
|
|
53
|
-
//
|
|
54
|
-
// names no GitHub repo.
|
|
84
|
+
// `directory` is where in the repo the package sits, `''` at the repo
|
|
85
|
+
// root, absent where it declares none. Throws when the package cannot be
|
|
86
|
+
// fetched or names no GitHub repo.
|
|
55
87
|
export interface GitHubLink {
|
|
56
88
|
github: string
|
|
57
89
|
directory?: string
|
|
@@ -71,7 +103,10 @@ export function resolvePackageRepos(packageNames: Iterable<string>, options?: {
|
|
|
71
103
|
// The disk cache resolvePackageRepos reads and writes, a month per entry.
|
|
72
104
|
// A read answers null for anything but a usable entry, one written by a
|
|
73
105
|
// release that resolved repos by other rules included; a write answers
|
|
74
|
-
// false where it could not write, and never throws.
|
|
106
|
+
// false where it could not write, and never throws. A write takes the
|
|
107
|
+
// `github` a lookup gives, `owner/name` with no trailing `.git`, so what
|
|
108
|
+
// it writes reads back; a `directory` left out is unknown, and `''` the
|
|
109
|
+
// repo root.
|
|
75
110
|
export function readPackageRepoCache(name: string): Promise<PackageRepo | null>
|
|
76
111
|
export function writePackageRepoCache(name: string, github: string, directory?: string): Promise<boolean>
|
|
77
112
|
|
|
@@ -84,25 +119,83 @@ export interface Dist {
|
|
|
84
119
|
integrity: string
|
|
85
120
|
}
|
|
86
121
|
|
|
122
|
+
// `gitHead` is the commit the document names, as npm reads it off the git
|
|
123
|
+
// checkout it publishes from, where it is a full commit id: 40 lowercase
|
|
124
|
+
// hex digits, or 64. `repository`, `homepage` and `bugs` are the
|
|
125
|
+
// document's, as the package.json published had them, or npm filled them
|
|
126
|
+
// in from one another: each a string as it is, or for `repository` and
|
|
127
|
+
// `bugs` an object, of which only the strings named below are read. Each is
|
|
128
|
+
// absent where the document has none, or has another shape, or an object
|
|
129
|
+
// with none of those strings. All of it is the publisher's word, held to no
|
|
130
|
+
// repository.
|
|
87
131
|
export interface PackageMeta {
|
|
88
132
|
name: string
|
|
89
133
|
version: string
|
|
90
134
|
dist: Dist
|
|
135
|
+
gitHead?: string
|
|
136
|
+
repository?: string | PackageRepository
|
|
137
|
+
homepage?: string
|
|
138
|
+
bugs?: string | PackageBugs
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export interface PackageRepository {
|
|
142
|
+
type: string | undefined
|
|
143
|
+
url: string | undefined
|
|
144
|
+
directory: string | undefined
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export interface PackageBugs {
|
|
148
|
+
url: string | undefined
|
|
149
|
+
email: string | undefined
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// getMeta's options: `cache`, and the `dist` a caller already has of that
|
|
153
|
+
// version, as a lockfile gives it, held to the rules above.
|
|
154
|
+
export interface MetaOptions extends CacheOptions {
|
|
155
|
+
dist?: Dist
|
|
91
156
|
}
|
|
92
157
|
|
|
93
158
|
// The registry's version document, refused unless it is for that name and
|
|
94
159
|
// version and its dist is held to the rules above.
|
|
95
|
-
|
|
160
|
+
//
|
|
161
|
+
// With `dist`, npm's own cache comes first, where getTarball looks for
|
|
162
|
+
// npm's tarballs: where it keeps the full packument of that name, as npm 10
|
|
163
|
+
// and later fetch it, and that version's entry there has the given dist's
|
|
164
|
+
// tarball and integrity, that entry answers, with no request, and neither
|
|
165
|
+
// `cache` nor setCacheDir's cache is read or written. An abbreviated
|
|
166
|
+
// packument, as earlier npm fetches, is passed over, as is one whose
|
|
167
|
+
// content does not match the sha512 npm filed it by. What it answers
|
|
168
|
+
// beside the dist is taken from npm's cache as it is from setCacheDir's.
|
|
169
|
+
// Else the document `cache` keeps answers, with no request, where its
|
|
170
|
+
// tarball and integrity are the given dist's; else the registry's is asked
|
|
171
|
+
// for, and throws where they are not.
|
|
172
|
+
//
|
|
173
|
+
// Without `dist`, the registry is asked on every call: a cache never
|
|
174
|
+
// supplies a dist. Kept whole, as `cache` says, and for good once it
|
|
175
|
+
// passes, as the registry never takes a version twice: in setCacheDir's
|
|
176
|
+
// cache, compressed, where `cache` is left out. One refused is not kept.
|
|
177
|
+
export function getMeta(name: string, version: string, options?: MetaOptions): Promise<PackageMeta>
|
|
178
|
+
|
|
179
|
+
// When the registry says each version of a package it lists was published,
|
|
180
|
+
// by version, as Date#toISOString writes it: from the whole package's
|
|
181
|
+
// document, the only one with `time`, megabytes for some. A version whose
|
|
182
|
+
// time is missing, not an ISO 8601 UTC time, or not one there is, as
|
|
183
|
+
// 2018-02-29, is left out. Refused unless the document is for that name.
|
|
184
|
+
export function getPublishTimes(name: string): Promise<Map<string, string>>
|
|
96
185
|
|
|
97
|
-
//
|
|
98
|
-
//
|
|
99
|
-
|
|
186
|
+
// Throws unless the integrity of that version is the one given, as for a
|
|
187
|
+
// dist read off a lockfile: by the version document `cache` keeps, with no
|
|
188
|
+
// request, else by the registry's, kept then as getMeta keeps it. One kept
|
|
189
|
+
// with another integrity, refused, or for another name or version, is
|
|
190
|
+
// passed over for the registry's, which throws where it has another.
|
|
191
|
+
export function verifyDist(name: string, version: string, dist: Dist, options?: CacheOptions): Promise<void>
|
|
100
192
|
|
|
101
193
|
// A published version's gzipped tarball, whole, in memory. Without `dist`,
|
|
102
|
-
// the version document is fetched on every call, cached tarball or not
|
|
103
|
-
// with one, it is trusted as given and
|
|
104
|
-
// for. Either way the bytes are checked
|
|
105
|
-
// they were downloaded (before they are
|
|
194
|
+
// the version document is fetched on every call, cached tarball or not,
|
|
195
|
+
// and kept as getMeta keeps it; with one, it is trusted as given and
|
|
196
|
+
// nothing but the tarball is asked for. Either way the bytes are checked
|
|
197
|
+
// against `dist.integrity`, whether they were downloaded (before they are
|
|
198
|
+
// cached) or read from a cache.
|
|
106
199
|
//
|
|
107
200
|
// Other caches are read first, and never written: npm's (cacache under
|
|
108
201
|
// npm_config_cache, else npm's default, `~/.npm` outside Windows; and npm
|
|
@@ -110,5 +203,7 @@ export function verifyDist(name: string, version: string, dist: Dist): Promise<v
|
|
|
110
203
|
// `~/.audit/cache/tgz/<org>:<name>-<version>.tgz`, then defaultCacheDir
|
|
111
204
|
// and cacheDirFor('stasis'), as setCacheDir files a tarball in either,
|
|
112
205
|
// whether set or not. A file there that does not match is passed over.
|
|
113
|
-
// Then setCacheDir's cache,
|
|
114
|
-
|
|
206
|
+
// Then setCacheDir's cache, or a `cache` store in its place, where bytes
|
|
207
|
+
// that do not match throw. A tarball downloaded is kept there, unless
|
|
208
|
+
// `cache` is false.
|
|
209
|
+
export function getTarball(name: string, version: string, dist?: Dist, options?: CacheOptions): Promise<Uint8Array>
|
package/npm.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { cacheDirFor, defaultCacheDir, setCacheDir } from './src/cache.js'
|
|
2
2
|
export { HttpError } from './src/http.js'
|
|
3
3
|
export { getGitHub, readPackageRepoCache, resolvePackageRepos, writePackageRepoCache } from './src/npm/repos.js'
|
|
4
|
-
export { getMeta, getTarball, verifyDist } from './src/npm/tarball.js'
|
|
4
|
+
export { getMeta, getPublishTimes, getTarball, verifyDist } from './src/npm/tarball.js'
|
package/package.d.ts
CHANGED
|
@@ -4,8 +4,11 @@ export interface PackageRepoLink {
|
|
|
4
4
|
// `owner/name`, by GitHub's own rules.
|
|
5
5
|
github?: string
|
|
6
6
|
// Where in that repo the package sits, `/`-separated, as the tree path
|
|
7
|
-
// itself (a homepage URL's decoded
|
|
8
|
-
// `
|
|
7
|
+
// itself (a homepage URL's decoded; a `repository.directory`'s `\` read
|
|
8
|
+
// as `/`, its empty and `.` parts dropped); `''` at its root, where
|
|
9
|
+
// `repository.directory` declares it (`./`, `/`, `/.`, `.`, `''`). Absent
|
|
10
|
+
// where unknown: neither field names a directory in it (one with a `..`
|
|
11
|
+
// part names none), and whenever `github` is absent.
|
|
9
12
|
directory?: string
|
|
10
13
|
// `https://github.com/<github>`, whenever there is a `github`.
|
|
11
14
|
url?: string
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@preventive/upstream",
|
|
3
|
-
"version": "1.0.0
|
|
3
|
+
"version": "1.0.0",
|
|
4
4
|
"description": "Minimal npm, crates.io and Soldeer registry clients, a GitHub client, and a git checkout reader: the calls we use, nothing more",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -39,6 +39,10 @@
|
|
|
39
39
|
"./soldeer.js": {
|
|
40
40
|
"types": "./soldeer.d.ts",
|
|
41
41
|
"default": "./soldeer.js"
|
|
42
|
+
},
|
|
43
|
+
"./weak-edges.js": {
|
|
44
|
+
"types": "./weak-edges.d.ts",
|
|
45
|
+
"default": "./weak-edges.js"
|
|
42
46
|
}
|
|
43
47
|
},
|
|
44
48
|
"files": [
|
|
@@ -62,6 +66,8 @@
|
|
|
62
66
|
"semver.js",
|
|
63
67
|
"soldeer.d.ts",
|
|
64
68
|
"soldeer.js",
|
|
69
|
+
"weak-edges.d.ts",
|
|
70
|
+
"weak-edges.js",
|
|
65
71
|
"src/advisories/common.js",
|
|
66
72
|
"src/advisories/github.js",
|
|
67
73
|
"src/advisories/index.js",
|
|
@@ -79,16 +85,19 @@
|
|
|
79
85
|
"src/github/read.js",
|
|
80
86
|
"src/github/write.js",
|
|
81
87
|
"src/http.js",
|
|
88
|
+
"src/npm/cacache.js",
|
|
82
89
|
"src/npm/registry.js",
|
|
83
90
|
"src/npm/repos.js",
|
|
84
91
|
"src/npm/tarball.js",
|
|
92
|
+
"src/npm/versions.js",
|
|
85
93
|
"src/package.js",
|
|
86
94
|
"src/pool.js",
|
|
87
95
|
"src/remote.js",
|
|
88
96
|
"src/semver-peer.cjs",
|
|
89
97
|
"src/semver.js",
|
|
90
98
|
"src/tree.js",
|
|
91
|
-
"src/soldeer.js"
|
|
99
|
+
"src/soldeer.js",
|
|
100
|
+
"src/weak-edges.js"
|
|
92
101
|
],
|
|
93
102
|
"engines": {
|
|
94
103
|
"node": ">=24.14.0"
|
package/src/advisories/github.js
CHANGED
|
@@ -1,13 +1,21 @@
|
|
|
1
1
|
import assert from 'node:assert/strict'
|
|
2
2
|
|
|
3
3
|
import { assertRepo, assertSoldeerName, assertSoldeerVersion, assertion, isGhsa, isRefName, isStrings } from '../args.js'
|
|
4
|
+
import { readRecord, writeRecord } from '../cache.js'
|
|
4
5
|
import { isGone } from '../github/client.js'
|
|
5
6
|
import { recover } from '../http.js'
|
|
6
7
|
import { pool } from '../pool.js'
|
|
8
|
+
import { isExactVersion } from '../semver.js'
|
|
7
9
|
import { advisoryUrl, covered, detailsOf, isText, mayBeInRange, metrics } from './common.js'
|
|
8
10
|
import { soldeerRepos } from './repos.js'
|
|
9
11
|
|
|
10
12
|
const REPOS_AT_ONCE = 4
|
|
13
|
+
const DIR = 'github/advisories'
|
|
14
|
+
// A maintainer publishes one whenever they are ready, and seeing it before
|
|
15
|
+
// GitHub reviews it is what a repository is asked for: 90 minutes, not a month.
|
|
16
|
+
const LISTING_TTL_MS = 90 * 60 * 1000
|
|
17
|
+
// Stamped on each entry, and raised when a listing is kept differently.
|
|
18
|
+
const VERSION = 2
|
|
11
19
|
|
|
12
20
|
const isRepoAdvisory = (advisory) => advisory && typeof advisory === 'object' && isGhsa(advisory.ghsa_id)
|
|
13
21
|
&& advisory.state === 'published' && isText(advisory.summary)
|
|
@@ -15,7 +23,63 @@ const isRepoAdvisory = (advisory) => advisory && typeof advisory === 'object' &&
|
|
|
15
23
|
|
|
16
24
|
export const assertClient = assertion('a GitHub client from createClient', (value) => typeof value?.listRepoAdvisories === 'function')
|
|
17
25
|
|
|
18
|
-
|
|
26
|
+
// A maintainer may give a vulnerable range only its lower bound, and the
|
|
27
|
+
// version that fixed it apart (`>= 5.0.0-beta.1`, patched in `5.0.0-rc.2`),
|
|
28
|
+
// as GitHub's page shows them side by side: such a range ends below its one
|
|
29
|
+
// patched version. A range bounded above, or patched in more than one
|
|
30
|
+
// version, is as given.
|
|
31
|
+
function boundedRange(range, patched) {
|
|
32
|
+
const fixed = typeof patched === 'string' ? patched.trim() : null
|
|
33
|
+
if (!isExactVersion(fixed) || range.includes('<')) return range
|
|
34
|
+
return range.trim() === '' ? `< ${fixed}` : `${range.trim()}, < ${fixed}`
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// What rows are made from, and all that is kept of a listing: each advisory
|
|
38
|
+
// not withdrawn, with its CVSS vector GitHub prefers, and each vulnerable
|
|
39
|
+
// range with the package it names, ended below its patched version where it
|
|
40
|
+
// has no end of its own (boundedRange). A listing that is malformed is
|
|
41
|
+
// refused whole, and never kept; one whose only fault is a description
|
|
42
|
+
// (isDigest) is answered, but not kept either.
|
|
43
|
+
function digest(repo, list) {
|
|
44
|
+
return list.flatMap((advisory) => {
|
|
45
|
+
assert.ok(isRepoAdvisory(advisory), `advisories: malformed advisory from ${repo}`)
|
|
46
|
+
if (advisory.withdrawn_at) return []
|
|
47
|
+
const cvss = [advisory.cvss_severities?.cvss_v3, advisory.cvss_severities?.cvss_v4, advisory.cvss].find((entry) => typeof entry?.vector_string === 'string')
|
|
48
|
+
const ranges = (advisory.vulnerabilities ?? []).map((vulnerability) => {
|
|
49
|
+
const range = vulnerability?.vulnerable_version_range ?? ''
|
|
50
|
+
assert.ok(typeof range === 'string', `advisories: malformed range in ${advisory.ghsa_id}`)
|
|
51
|
+
const { ecosystem, name } = vulnerability?.package ?? {}
|
|
52
|
+
return { range: boundedRange(range, vulnerability?.patched_versions), ...(typeof ecosystem === 'string' && { ecosystem }), ...(typeof name === 'string' && { name }) }
|
|
53
|
+
})
|
|
54
|
+
return [{
|
|
55
|
+
ghsa: advisory.ghsa_id,
|
|
56
|
+
title: advisory.summary,
|
|
57
|
+
...(advisory.description != null && advisory.description !== '' && { description: advisory.description }),
|
|
58
|
+
...metrics({ severity: advisory.severity, score: cvss?.score, vector: cvss?.vector_string, cwe: advisory.cwe_ids }),
|
|
59
|
+
ranges,
|
|
60
|
+
}]
|
|
61
|
+
})
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// A description is checked only for `details` (detailsOf), so a malformed
|
|
65
|
+
// one leaves the listing usable, but not kept.
|
|
66
|
+
const isDigest = (advisory) => advisory && typeof advisory === 'object' && isGhsa(advisory.ghsa) && isText(advisory.title)
|
|
67
|
+
&& (advisory.description === undefined || isText(advisory.description))
|
|
68
|
+
&& Array.isArray(advisory.ranges) && advisory.ranges.every((entry) => typeof entry?.range === 'string')
|
|
69
|
+
|
|
70
|
+
// A repository's listing, digested, through the cache: GitHub's names are
|
|
71
|
+
// case-insensitive, so one entry answers every spelling. A repository gone,
|
|
72
|
+
// renamed or blocked has none, and that is not kept.
|
|
73
|
+
async function listAdvisories(github, repo, cache) {
|
|
74
|
+
const name = repo.toLowerCase()
|
|
75
|
+
const entry = await readRecord(DIR, name, cache, LISTING_TTL_MS)
|
|
76
|
+
if (entry?.v === VERSION && Array.isArray(entry.advisories) && entry.advisories.every(isDigest)) return entry.advisories
|
|
77
|
+
const list = await github.listRepoAdvisories({ repo }).catch(recover(isGone, null))
|
|
78
|
+
if (list === null) return []
|
|
79
|
+
const advisories = digest(repo, list)
|
|
80
|
+
if (advisories.every(isDigest)) await writeRecord(DIR, name, { v: VERSION, advisories }, cache)
|
|
81
|
+
return advisories
|
|
82
|
+
}
|
|
19
83
|
|
|
20
84
|
// GitHub's `>= 1.0.0, < 1.2.6` is npm's with the commas dropped.
|
|
21
85
|
// Maintainers write these unreviewed: one without a range, or with one
|
|
@@ -23,17 +87,16 @@ const listAdvisories = (github, repo) => github.listRepoAdvisories({ repo }).cat
|
|
|
23
87
|
// may publish it long before the advisory database has one, and its
|
|
24
88
|
// `description` is the text `details` asks for: the listing has it all.
|
|
25
89
|
function fromRepository(repo, name, advisory, range, asked, { covers, details }) {
|
|
26
|
-
const cvss = [advisory.cvss_severities?.cvss_v3, advisory.cvss_severities?.cvss_v4, advisory.cvss].find((entry) => typeof entry?.vector_string === 'string')
|
|
27
90
|
return {
|
|
28
91
|
name,
|
|
29
92
|
source: 'repository',
|
|
30
|
-
id: advisory.
|
|
31
|
-
ghsa: advisory.
|
|
32
|
-
url: advisoryUrl(advisory.
|
|
93
|
+
id: advisory.ghsa,
|
|
94
|
+
ghsa: advisory.ghsa,
|
|
95
|
+
url: advisoryUrl(advisory.ghsa, repo),
|
|
33
96
|
aliases: [],
|
|
34
|
-
title: advisory.
|
|
35
|
-
...(details && detailsOf(advisory.description, advisory.
|
|
36
|
-
...metrics({ severity: advisory.severity, score: cvss
|
|
97
|
+
title: advisory.title,
|
|
98
|
+
...(details && detailsOf(advisory.description, advisory.ghsa)),
|
|
99
|
+
...metrics({ severity: advisory.severity, score: advisory.cvss, vector: advisory.cvssVector, cwe: advisory.cwe }),
|
|
37
100
|
range,
|
|
38
101
|
versions: covered(asked, range.replaceAll(',', ' '), covers),
|
|
39
102
|
}
|
|
@@ -41,21 +104,18 @@ function fromRepository(repo, name, advisory, range, asked, { covers, details })
|
|
|
41
104
|
|
|
42
105
|
// Each asked name's repository, `repoOf` it, is asked once, and its
|
|
43
106
|
// advisories' vulnerable ranges become rows for the names that `takes` a
|
|
44
|
-
//
|
|
107
|
+
// range's package, one per name, advisory and range, holding the asked
|
|
45
108
|
// versions it `covers`, and with `details`, its text.
|
|
46
|
-
async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers, details }) {
|
|
109
|
+
async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers, details, cache }) {
|
|
47
110
|
// GitHub's names are case-insensitive: one spelling asks for all.
|
|
48
111
|
const namesOf = Map.groupBy([...asked.keys()].filter(repoOf), (name) => repoOf(name).toLowerCase())
|
|
49
|
-
const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ repo: repoOf(names[0]), names, list: await listAdvisories(github, repoOf(names[0])) }))
|
|
112
|
+
const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ repo: repoOf(names[0]), names, list: await listAdvisories(github, repoOf(names[0]), cache) }))
|
|
50
113
|
const rows = new Map()
|
|
51
114
|
for (const { repo, names, list } of listed) {
|
|
52
115
|
for (const advisory of list) {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
assert.ok(typeof range === 'string', `advisories: malformed range in ${advisory.ghsa_id}`)
|
|
57
|
-
for (const name of names.filter((candidate) => takes(candidate, vulnerability?.package))) {
|
|
58
|
-
rows.set(`${name} ${advisory.ghsa_id} ${range}`, fromRepository(repo, name, advisory, range, asked.get(name), { covers, details }))
|
|
116
|
+
for (const { range, ...pkg } of advisory.ranges) {
|
|
117
|
+
for (const name of names.filter((candidate) => takes(candidate, pkg))) {
|
|
118
|
+
rows.set(`${name} ${advisory.ghsa} ${range}`, fromRepository(repo, name, advisory, range, asked.get(name), { covers, details }))
|
|
59
119
|
}
|
|
60
120
|
}
|
|
61
121
|
}
|
|
@@ -63,9 +123,10 @@ async function repositoryAdvisories(github, asked, { repoOf, takes = () => true,
|
|
|
63
123
|
return [...rows.values()]
|
|
64
124
|
}
|
|
65
125
|
|
|
66
|
-
// Each asked name's repository: as given, `known`, or else looked up
|
|
67
|
-
|
|
68
|
-
|
|
126
|
+
// Each asked name's repository: as given, `known`, or else looked up,
|
|
127
|
+
// through `cache`.
|
|
128
|
+
async function reposOf(asked, known, lookUp, cache) {
|
|
129
|
+
const found = await lookUp([...asked.keys()].filter((name) => !known.has(name)), { asked, cache })
|
|
69
130
|
return (name) => known.get(name) ?? found.get(name)
|
|
70
131
|
}
|
|
71
132
|
|
|
@@ -74,12 +135,12 @@ async function reposOf(asked, known, lookUp) {
|
|
|
74
135
|
// each package's repository, `known` or else looked up, adds what it
|
|
75
136
|
// publishes for GitHub's `ecosystem` entries naming the package, for the
|
|
76
137
|
// versions `rows` do not already report under that GHSA.
|
|
77
|
-
export async function withRepositories(rows, asked, { github, repoAdvisories, known, details }, { ecosystem, lookUp, covers }) {
|
|
138
|
+
export async function withRepositories(rows, asked, { github, repoAdvisories, known, details, cache }, { ecosystem, lookUp, covers }) {
|
|
78
139
|
if (!repoAdvisories) return rows
|
|
79
140
|
const reported = new Set(rows.flatMap((row) => [row.id, row.ghsa, ...row.aliases].filter(isGhsa).flatMap((id) => row.versions.map((version) => `${row.name} ${id} ${version}`))))
|
|
80
|
-
const repoOf = await reposOf(asked, known, lookUp)
|
|
81
|
-
const takes = (name, pkg) => pkg
|
|
82
|
-
const added = await repositoryAdvisories(github, asked, { repoOf, takes, covers, details })
|
|
141
|
+
const repoOf = await reposOf(asked, known, lookUp, cache)
|
|
142
|
+
const takes = (name, pkg) => pkg.ecosystem === ecosystem && pkg.name === name
|
|
143
|
+
const added = await repositoryAdvisories(github, asked, { repoOf, takes, covers, details, cache })
|
|
83
144
|
return [...rows, ...added.map((row) => ({ ...row, versions: row.versions.filter((version) => !reported.has(`${row.name} ${row.id} ${version}`)) }))]
|
|
84
145
|
}
|
|
85
146
|
|
|
@@ -93,7 +154,7 @@ export const GITHUB = {
|
|
|
93
154
|
repositoryOnly: true,
|
|
94
155
|
assertName: assertRepo,
|
|
95
156
|
assertVersion: assertion('a version or a branch name', isRefName),
|
|
96
|
-
advisories: (asked, { github, details }) => repositoryAdvisories(github, asked, { repoOf: (repo) => repo, covers: coversPlaceholder, details }),
|
|
157
|
+
advisories: (asked, { github, details, cache }) => repositoryAdvisories(github, asked, { repoOf: (repo) => repo, covers: coversPlaceholder, details, cache }),
|
|
97
158
|
}
|
|
98
159
|
|
|
99
160
|
// Soldeer packages, which no advisory database has: the repository each
|
|
@@ -104,5 +165,5 @@ export const SOLDEER = {
|
|
|
104
165
|
repositoryOnly: true,
|
|
105
166
|
assertName: assertSoldeerName,
|
|
106
167
|
assertVersion: assertSoldeerVersion,
|
|
107
|
-
advisories: async (asked, { github, known, details }) => await repositoryAdvisories(github, asked, { repoOf: await reposOf(asked, known, soldeerRepos), details }),
|
|
168
|
+
advisories: async (asked, { github, known, details, cache }) => await repositoryAdvisories(github, asked, { repoOf: await reposOf(asked, known, soldeerRepos, cache), details, cache }),
|
|
108
169
|
}
|
package/src/advisories/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import assert from 'node:assert/strict'
|
|
2
2
|
|
|
3
3
|
import { assertArgs, assertBoolean, assertRepo, assertion, isIterable, optional, sameName } from '../args.js'
|
|
4
|
+
import { assertCache } from '../cache.js'
|
|
4
5
|
import { order } from './common.js'
|
|
5
6
|
import { GITHUB, SOLDEER, assertClient } from './github.js'
|
|
6
7
|
import { NPM } from './npm.js'
|
|
@@ -11,7 +12,7 @@ const byNumbers = new Intl.Collator('en', { numeric: true }).compare
|
|
|
11
12
|
const assertEcosystem = assertion(`one of ${Object.keys(ECOSYSTEMS).join(', ')}`, (value) => typeof value === 'string' && Object.hasOwn(ECOSYSTEMS, value))
|
|
12
13
|
const assertVersions = assertion('a non-empty array', (value) => Array.isArray(value) && value.length > 0)
|
|
13
14
|
|
|
14
|
-
// Ecosystem
|
|
15
|
+
// Ecosystem -> name -> { versions, github? }, a name given twice merged.
|
|
15
16
|
// Every package is checked before any is used.
|
|
16
17
|
function collect(packages) {
|
|
17
18
|
assert.ok(isIterable(packages), 'advisories: packages must be an iterable of { ecosystem, name, github?, versions }')
|
|
@@ -34,7 +35,7 @@ function collect(packages) {
|
|
|
34
35
|
}
|
|
35
36
|
|
|
36
37
|
export async function advisories(packages, options = {}) {
|
|
37
|
-
assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean) })
|
|
38
|
+
assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean), cache: optional(assertCache) })
|
|
38
39
|
assert.ok(!options.repoAdvisories || options.github, 'advisories: repoAdvisories needs a github client')
|
|
39
40
|
const byEcosystem = collect(packages)
|
|
40
41
|
for (const ecosystem of byEcosystem.keys()) assert.ok(!ECOSYSTEMS[ecosystem].repositoryOnly || options.github, `advisories: ${ecosystem} packages need a github client`)
|
|
@@ -43,7 +44,7 @@ export async function advisories(packages, options = {}) {
|
|
|
43
44
|
const names = [...named.keys()].toSorted()
|
|
44
45
|
const asked = new Map(names.map((name) => [name, [...named.get(name).versions].toSorted(compare)]))
|
|
45
46
|
const known = new Map(names.filter((name) => named.get(name).github).map((name) => [name, named.get(name).github]))
|
|
46
|
-
const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, known })
|
|
47
|
+
const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, known, cache: options.cache })
|
|
47
48
|
return rows.map((row) => ({ ecosystem, ...row }))
|
|
48
49
|
}))
|
|
49
50
|
return found.flat().filter((row) => row.versions.length > 0).toSorted((a, b) => order(a.ecosystem, b.ecosystem) || order(a.name, b.name))
|
package/src/advisories/osv.js
CHANGED
|
@@ -13,12 +13,12 @@ const QUERIES_PER_REQUEST = 1000
|
|
|
13
13
|
export const RECORDS_AT_ONCE = 8
|
|
14
14
|
const isOsvId = matches(/^(?=.{1,128}$)[A-Z][\dA-Z]*(?:-[\dA-Za-z]+)+$/u)
|
|
15
15
|
// osv.dev's page for one of its records. An id that is only another's
|
|
16
|
-
// alias (PKSA
|
|
16
|
+
// alias (PKSA-...) has none, and redirects to a search.
|
|
17
17
|
const osvPage = (id) => `https://osv.dev/vulnerability/${id}`
|
|
18
18
|
const INFORMATIONAL = new Set(['unmaintained', 'unsound', 'notice'])
|
|
19
19
|
// Of the records that publish one advisory, the first answered for a
|
|
20
20
|
// version stands for the rest on it: RustSec's, what `cargo audit` reads
|
|
21
|
-
// and GitHub mirrors, then the GHSA, then any other (MAL
|
|
21
|
+
// and GitHub mirrors, then the GHSA, then any other (MAL-...), then by id.
|
|
22
22
|
const rank = (id) => (id.startsWith('RUSTSEC-') ? 0 : (isGhsa(id) ? 1 : 2))
|
|
23
23
|
const first = (a, b) => rank(a.id) - rank(b.id) || order(a.id, b.id)
|
|
24
24
|
const named = (a, b) => a.aliases.includes(b.id) || b.aliases.includes(a.id)
|
|
@@ -88,7 +88,7 @@ function toAdvisory(ecosystem, name, versions, record, details) {
|
|
|
88
88
|
async function osvAdvisories(ecosystem, asked, options) {
|
|
89
89
|
const { osv, github, lookUp, covers } = ecosystem
|
|
90
90
|
const list = [...asked].flatMap(([name, versions]) => versions.map((version) => ({ name, version })))
|
|
91
|
-
const hits = new Map() // id
|
|
91
|
+
const hits = new Map() // id -> name -> versions, in `list` order
|
|
92
92
|
for (const chunk of chunks(list, QUERIES_PER_REQUEST)) {
|
|
93
93
|
const body = { queries: chunk.map(({ name, version }) => ({ package: { name, ecosystem: osv }, version })) }
|
|
94
94
|
const answer = await request(QUERY_URL, { method: 'POST', body, as: 'json' })
|