@preventive/upstream 1.0.0-alpha.10 → 1.0.0-alpha.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/advisories.d.ts +21 -0
- package/git.d.ts +2 -2
- package/npm.d.ts +10 -8
- package/package.d.ts +4 -1
- package/package.json +1 -1
- package/src/advisories/github.js +70 -22
- package/src/advisories/index.js +3 -3
- package/src/cache.js +32 -9
- package/src/git.js +1 -1
- package/src/npm/repos.js +14 -12
- package/src/package.js +13 -5
package/advisories.d.ts
CHANGED
|
@@ -27,6 +27,19 @@ export interface Package {
|
|
|
27
27
|
versions: string[]
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
+
// A store of the caller's, a database's say, for what would otherwise be
|
|
31
|
+
// kept in setCacheDir's cache (npm.js): `read` answers the value last
|
|
32
|
+
// written under `key`, or null or undefined for none. Keys are
|
|
33
|
+
// `<kind>/<name>`, `github/advisories/owner/name` for a repository's
|
|
34
|
+
// listing, and values plain JSON data, which `read` may answer as written
|
|
35
|
+
// or as a copy. Whatever it answers is checked as a cached file is: one
|
|
36
|
+
// malformed, stale or kept differently is a miss. A rejection from
|
|
37
|
+
// either is the call's. It is trusted as setCacheDir's directory is.
|
|
38
|
+
export interface CacheStore {
|
|
39
|
+
read(key: string): Promise<unknown>
|
|
40
|
+
write(key: string, value: unknown): Promise<void>
|
|
41
|
+
}
|
|
42
|
+
|
|
30
43
|
export interface AdvisoryOptions {
|
|
31
44
|
// The client every GitHub request goes through. Required for `soldeer`
|
|
32
45
|
// and `github` packages, whose repository's published advisories are
|
|
@@ -39,6 +52,9 @@ export interface AdvisoryOptions {
|
|
|
39
52
|
// the one npm's metadata, crates.io or Packagist names, looked up
|
|
40
53
|
// through the cache. Needs `github`.
|
|
41
54
|
repoAdvisories?: boolean
|
|
55
|
+
// Where a repository's listing is kept, in place of setCacheDir's cache:
|
|
56
|
+
// see `advisories`. Repository lookups still go through that cache.
|
|
57
|
+
cache?: CacheStore
|
|
42
58
|
// Also returns each advisory's full text, as `details`. OSV's records
|
|
43
59
|
// and a repository's listing already carry it; npm's registry does not,
|
|
44
60
|
// and its rows take OSV's record of their GHSA, one more request each
|
|
@@ -108,6 +124,11 @@ export interface Advisory {
|
|
|
108
124
|
// nothing, and a GHSA OSV does not have adds no `details`. Any other
|
|
109
125
|
// failure, a malformed answer, or one about something not asked, throws:
|
|
110
126
|
// nothing is left out quietly.
|
|
127
|
+
// A repository's listing is kept for 90 minutes, as one entry for every
|
|
128
|
+
// spelling of its name, holding only what rows are made from, `details`
|
|
129
|
+
// included: in setCacheDir's cache brotli-compressed at quality 9, or in
|
|
130
|
+
// `cache` as it is. 90 minutes is how late an advisory its maintainer
|
|
131
|
+
// publishes can be seen. One gone or malformed is not kept.
|
|
111
132
|
// Sorted by ecosystem and name. Versions are matched by npm's semver, from
|
|
112
133
|
// the npm beside node, or by the semver peer where there is no npm.
|
|
113
134
|
export function advisories(packages: Iterable<Package>, options?: AdvisoryOptions): Promise<Advisory[]>
|
package/git.d.ts
CHANGED
|
@@ -6,8 +6,8 @@ export interface GitCheckout {
|
|
|
6
6
|
// `owner/name`, when the origin remote is a GitHub one. Never the
|
|
7
7
|
// remote's URL, which can name a private host or carry a token.
|
|
8
8
|
github?: string
|
|
9
|
-
// Where the directory sits in its checkout, `/`-separated;
|
|
10
|
-
//
|
|
9
|
+
// Where the directory sits in its checkout, `/`-separated; `''` at the
|
|
10
|
+
// root.
|
|
11
11
|
directory?: string
|
|
12
12
|
// `https://github.com/<github>`, whenever there is a `github`.
|
|
13
13
|
url?: string
|
package/npm.d.ts
CHANGED
|
@@ -32,10 +32,11 @@ export const defaultCacheDir: string | undefined
|
|
|
32
32
|
// cache: every read of it misses and every write is skipped. getTarball
|
|
33
33
|
// reads ours all the same, set or not.
|
|
34
34
|
//
|
|
35
|
-
// The directory is trusted: a package's repo
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
//
|
|
35
|
+
// The directory is trusted: a package's repo, and for 90 minutes a
|
|
36
|
+
// repository's published advisories (advisories.js), are answered from it
|
|
37
|
+
// as they were written, with no request. Point it only at storage that
|
|
38
|
+
// nothing less trusted than the caller can write, never a cache shared
|
|
39
|
+
// with or restored from lower-trust jobs. Tarballs, crates (cargo.js), zips
|
|
39
40
|
// (soldeer.js) and tree tarballs (github.js) don't rely on this, since
|
|
40
41
|
// they are checked against their integrity on every call.
|
|
41
42
|
export function setCacheDir(dir?: string | false): void
|
|
@@ -49,9 +50,9 @@ export class HttpError extends Error {
|
|
|
49
50
|
|
|
50
51
|
// A package's GitHub repo as its registry metadata names it — getRepo
|
|
51
52
|
// (package.js) over its `latest` document: `github` is `owner/name`, and
|
|
52
|
-
// `directory` is where in the repo the package sits,
|
|
53
|
-
//
|
|
54
|
-
// names no GitHub repo.
|
|
53
|
+
// `directory` is where in the repo the package sits, `''` at the repo
|
|
54
|
+
// root, absent where it declares none. Throws when the package cannot be
|
|
55
|
+
// fetched or names no GitHub repo.
|
|
55
56
|
export interface GitHubLink {
|
|
56
57
|
github: string
|
|
57
58
|
directory?: string
|
|
@@ -73,7 +74,8 @@ export function resolvePackageRepos(packageNames: Iterable<string>, options?: {
|
|
|
73
74
|
// release that resolved repos by other rules included; a write answers
|
|
74
75
|
// false where it could not write, and never throws. A write takes the
|
|
75
76
|
// `github` a lookup gives, `owner/name` with no trailing `.git`, so what
|
|
76
|
-
// it writes reads back
|
|
77
|
+
// it writes reads back; a `directory` left out is unknown, and `''` the
|
|
78
|
+
// repo root.
|
|
77
79
|
export function readPackageRepoCache(name: string): Promise<PackageRepo | null>
|
|
78
80
|
export function writePackageRepoCache(name: string, github: string, directory?: string): Promise<boolean>
|
|
79
81
|
|
package/package.d.ts
CHANGED
|
@@ -5,7 +5,10 @@ export interface PackageRepoLink {
|
|
|
5
5
|
github?: string
|
|
6
6
|
// Where in that repo the package sits, `/`-separated, as the tree path
|
|
7
7
|
// itself (a homepage URL's decoded; a `repository.directory`'s `\` read
|
|
8
|
-
// as
|
|
8
|
+
// as `/`, its empty and `.` parts dropped); `''` at its root, where
|
|
9
|
+
// `repository.directory` declares it (`./`, `/`, `/.`, `.`, `''`). Absent
|
|
10
|
+
// where unknown: neither field names a directory in it (one with a `..`
|
|
11
|
+
// part names none), and whenever `github` is absent.
|
|
9
12
|
directory?: string
|
|
10
13
|
// `https://github.com/<github>`, whenever there is a `github`.
|
|
11
14
|
url?: string
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@preventive/upstream",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.11",
|
|
4
4
|
"description": "Minimal npm, crates.io and Soldeer registry clients, a GitHub client, and a git checkout reader: the calls we use, nothing more",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
package/src/advisories/github.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import assert from 'node:assert/strict'
|
|
2
2
|
|
|
3
3
|
import { assertRepo, assertSoldeerName, assertSoldeerVersion, assertion, isGhsa, isRefName, isStrings } from '../args.js'
|
|
4
|
+
import { readRecord, writeRecord } from '../cache.js'
|
|
4
5
|
import { isGone } from '../github/client.js'
|
|
5
6
|
import { recover } from '../http.js'
|
|
6
7
|
import { pool } from '../pool.js'
|
|
@@ -8,14 +9,65 @@ import { advisoryUrl, covered, detailsOf, isText, mayBeInRange, metrics } from '
|
|
|
8
9
|
import { soldeerRepos } from './repos.js'
|
|
9
10
|
|
|
10
11
|
const REPOS_AT_ONCE = 4
|
|
12
|
+
const DIR = 'github/advisories'
|
|
13
|
+
// A maintainer publishes one whenever they are ready, and seeing it before
|
|
14
|
+
// GitHub reviews it is what a repository is asked for: 90 minutes, not a month.
|
|
15
|
+
const LISTING_TTL_MS = 90 * 60 * 1000
|
|
16
|
+
// Stamped on each entry, and raised when a listing is kept differently.
|
|
17
|
+
const VERSION = 1
|
|
11
18
|
|
|
12
19
|
const isRepoAdvisory = (advisory) => advisory && typeof advisory === 'object' && isGhsa(advisory.ghsa_id)
|
|
13
20
|
&& advisory.state === 'published' && isText(advisory.summary)
|
|
14
21
|
&& (advisory.vulnerabilities == null || Array.isArray(advisory.vulnerabilities)) && (advisory.cwe_ids == null || isStrings(advisory.cwe_ids))
|
|
15
22
|
|
|
16
23
|
export const assertClient = assertion('a GitHub client from createClient', (value) => typeof value?.listRepoAdvisories === 'function')
|
|
24
|
+
export const assertCacheStore = assertion('a store with read and write', (value) => typeof value?.read === 'function' && typeof value?.write === 'function')
|
|
17
25
|
|
|
18
|
-
|
|
26
|
+
// What rows are made from, and all that is kept of a listing: each advisory
|
|
27
|
+
// not withdrawn, with its CVSS vector GitHub prefers, and each vulnerable
|
|
28
|
+
// range with the package it names. A listing that is malformed is refused
|
|
29
|
+
// whole, and never kept; one whose only fault is a description (isDigest)
|
|
30
|
+
// is answered, but not kept either.
|
|
31
|
+
function digest(repo, list) {
|
|
32
|
+
return list.flatMap((advisory) => {
|
|
33
|
+
assert.ok(isRepoAdvisory(advisory), `advisories: malformed advisory from ${repo}`)
|
|
34
|
+
if (advisory.withdrawn_at) return []
|
|
35
|
+
const cvss = [advisory.cvss_severities?.cvss_v3, advisory.cvss_severities?.cvss_v4, advisory.cvss].find((entry) => typeof entry?.vector_string === 'string')
|
|
36
|
+
const ranges = (advisory.vulnerabilities ?? []).map((vulnerability) => {
|
|
37
|
+
const range = vulnerability?.vulnerable_version_range ?? ''
|
|
38
|
+
assert.ok(typeof range === 'string', `advisories: malformed range in ${advisory.ghsa_id}`)
|
|
39
|
+
const { ecosystem, name } = vulnerability?.package ?? {}
|
|
40
|
+
return { range, ...(typeof ecosystem === 'string' && { ecosystem }), ...(typeof name === 'string' && { name }) }
|
|
41
|
+
})
|
|
42
|
+
return [{
|
|
43
|
+
ghsa: advisory.ghsa_id,
|
|
44
|
+
title: advisory.summary,
|
|
45
|
+
...(advisory.description != null && advisory.description !== '' && { description: advisory.description }),
|
|
46
|
+
...metrics({ severity: advisory.severity, score: cvss?.score, vector: cvss?.vector_string, cwe: advisory.cwe_ids }),
|
|
47
|
+
ranges,
|
|
48
|
+
}]
|
|
49
|
+
})
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// A description is checked only for `details` (detailsOf), so a malformed
|
|
53
|
+
// one leaves the listing usable, but not kept.
|
|
54
|
+
const isDigest = (advisory) => advisory && typeof advisory === 'object' && isGhsa(advisory.ghsa) && isText(advisory.title)
|
|
55
|
+
&& (advisory.description === undefined || isText(advisory.description))
|
|
56
|
+
&& Array.isArray(advisory.ranges) && advisory.ranges.every((entry) => typeof entry?.range === 'string')
|
|
57
|
+
|
|
58
|
+
// A repository's listing, digested, through the cache: GitHub's names are
|
|
59
|
+
// case-insensitive, so one entry answers every spelling. A repository gone,
|
|
60
|
+
// renamed or blocked has none, and that is not kept.
|
|
61
|
+
async function listAdvisories(github, repo, store) {
|
|
62
|
+
const name = repo.toLowerCase()
|
|
63
|
+
const entry = await readRecord(DIR, name, { ttl: LISTING_TTL_MS, store })
|
|
64
|
+
if (entry?.v === VERSION && Array.isArray(entry.advisories) && entry.advisories.every(isDigest)) return entry.advisories
|
|
65
|
+
const list = await github.listRepoAdvisories({ repo }).catch(recover(isGone, null))
|
|
66
|
+
if (list === null) return []
|
|
67
|
+
const advisories = digest(repo, list)
|
|
68
|
+
if (advisories.every(isDigest)) await writeRecord(DIR, name, { v: VERSION, advisories }, { store })
|
|
69
|
+
return advisories
|
|
70
|
+
}
|
|
19
71
|
|
|
20
72
|
// GitHub's `>= 1.0.0, < 1.2.6` is npm's with the commas dropped.
|
|
21
73
|
// Maintainers write these unreviewed: one without a range, or with one
|
|
@@ -23,17 +75,16 @@ const listAdvisories = (github, repo) => github.listRepoAdvisories({ repo }).cat
|
|
|
23
75
|
// may publish it long before the advisory database has one, and its
|
|
24
76
|
// `description` is the text `details` asks for: the listing has it all.
|
|
25
77
|
function fromRepository(repo, name, advisory, range, asked, { covers, details }) {
|
|
26
|
-
const cvss = [advisory.cvss_severities?.cvss_v3, advisory.cvss_severities?.cvss_v4, advisory.cvss].find((entry) => typeof entry?.vector_string === 'string')
|
|
27
78
|
return {
|
|
28
79
|
name,
|
|
29
80
|
source: 'repository',
|
|
30
|
-
id: advisory.
|
|
31
|
-
ghsa: advisory.
|
|
32
|
-
url: advisoryUrl(advisory.
|
|
81
|
+
id: advisory.ghsa,
|
|
82
|
+
ghsa: advisory.ghsa,
|
|
83
|
+
url: advisoryUrl(advisory.ghsa, repo),
|
|
33
84
|
aliases: [],
|
|
34
|
-
title: advisory.
|
|
35
|
-
...(details && detailsOf(advisory.description, advisory.
|
|
36
|
-
...metrics({ severity: advisory.severity, score: cvss
|
|
85
|
+
title: advisory.title,
|
|
86
|
+
...(details && detailsOf(advisory.description, advisory.ghsa)),
|
|
87
|
+
...metrics({ severity: advisory.severity, score: advisory.cvss, vector: advisory.cvssVector, cwe: advisory.cwe }),
|
|
37
88
|
range,
|
|
38
89
|
versions: covered(asked, range.replaceAll(',', ' '), covers),
|
|
39
90
|
}
|
|
@@ -41,21 +92,18 @@ function fromRepository(repo, name, advisory, range, asked, { covers, details })
|
|
|
41
92
|
|
|
42
93
|
// Each asked name's repository, `repoOf` it, is asked once, and its
|
|
43
94
|
// advisories' vulnerable ranges become rows for the names that `takes` a
|
|
44
|
-
//
|
|
95
|
+
// range's package, one per name, advisory and range, holding the asked
|
|
45
96
|
// versions it `covers`, and with `details`, its text.
|
|
46
|
-
async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers, details }) {
|
|
97
|
+
async function repositoryAdvisories(github, asked, { repoOf, takes = () => true, covers, details, cache }) {
|
|
47
98
|
// GitHub's names are case-insensitive: one spelling asks for all.
|
|
48
99
|
const namesOf = Map.groupBy([...asked.keys()].filter(repoOf), (name) => repoOf(name).toLowerCase())
|
|
49
|
-
const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ repo: repoOf(names[0]), names, list: await listAdvisories(github, repoOf(names[0])) }))
|
|
100
|
+
const listed = await pool([...namesOf.values()], REPOS_AT_ONCE, async (names) => ({ repo: repoOf(names[0]), names, list: await listAdvisories(github, repoOf(names[0]), cache) }))
|
|
50
101
|
const rows = new Map()
|
|
51
102
|
for (const { repo, names, list } of listed) {
|
|
52
103
|
for (const advisory of list) {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
assert.ok(typeof range === 'string', `advisories: malformed range in ${advisory.ghsa_id}`)
|
|
57
|
-
for (const name of names.filter((candidate) => takes(candidate, vulnerability?.package))) {
|
|
58
|
-
rows.set(`${name} ${advisory.ghsa_id} ${range}`, fromRepository(repo, name, advisory, range, asked.get(name), { covers, details }))
|
|
104
|
+
for (const { range, ...pkg } of advisory.ranges) {
|
|
105
|
+
for (const name of names.filter((candidate) => takes(candidate, pkg))) {
|
|
106
|
+
rows.set(`${name} ${advisory.ghsa} ${range}`, fromRepository(repo, name, advisory, range, asked.get(name), { covers, details }))
|
|
59
107
|
}
|
|
60
108
|
}
|
|
61
109
|
}
|
|
@@ -74,12 +122,12 @@ async function reposOf(asked, known, lookUp) {
|
|
|
74
122
|
// each package's repository, `known` or else looked up, adds what it
|
|
75
123
|
// publishes for GitHub's `ecosystem` entries naming the package, for the
|
|
76
124
|
// versions `rows` do not already report under that GHSA.
|
|
77
|
-
export async function withRepositories(rows, asked, { github, repoAdvisories, known, details }, { ecosystem, lookUp, covers }) {
|
|
125
|
+
export async function withRepositories(rows, asked, { github, repoAdvisories, known, details, cache }, { ecosystem, lookUp, covers }) {
|
|
78
126
|
if (!repoAdvisories) return rows
|
|
79
127
|
const reported = new Set(rows.flatMap((row) => [row.id, row.ghsa, ...row.aliases].filter(isGhsa).flatMap((id) => row.versions.map((version) => `${row.name} ${id} ${version}`))))
|
|
80
128
|
const repoOf = await reposOf(asked, known, lookUp)
|
|
81
|
-
const takes = (name, pkg) => pkg
|
|
82
|
-
const added = await repositoryAdvisories(github, asked, { repoOf, takes, covers, details })
|
|
129
|
+
const takes = (name, pkg) => pkg.ecosystem === ecosystem && pkg.name === name
|
|
130
|
+
const added = await repositoryAdvisories(github, asked, { repoOf, takes, covers, details, cache })
|
|
83
131
|
return [...rows, ...added.map((row) => ({ ...row, versions: row.versions.filter((version) => !reported.has(`${row.name} ${row.id} ${version}`)) }))]
|
|
84
132
|
}
|
|
85
133
|
|
|
@@ -93,7 +141,7 @@ export const GITHUB = {
|
|
|
93
141
|
repositoryOnly: true,
|
|
94
142
|
assertName: assertRepo,
|
|
95
143
|
assertVersion: assertion('a version or a branch name', isRefName),
|
|
96
|
-
advisories: (asked, { github, details }) => repositoryAdvisories(github, asked, { repoOf: (repo) => repo, covers: coversPlaceholder, details }),
|
|
144
|
+
advisories: (asked, { github, details, cache }) => repositoryAdvisories(github, asked, { repoOf: (repo) => repo, covers: coversPlaceholder, details, cache }),
|
|
97
145
|
}
|
|
98
146
|
|
|
99
147
|
// Soldeer packages, which no advisory database has: the repository each
|
|
@@ -104,5 +152,5 @@ export const SOLDEER = {
|
|
|
104
152
|
repositoryOnly: true,
|
|
105
153
|
assertName: assertSoldeerName,
|
|
106
154
|
assertVersion: assertSoldeerVersion,
|
|
107
|
-
advisories: async (asked, { github, known, details }) => await repositoryAdvisories(github, asked, { repoOf: await reposOf(asked, known, soldeerRepos), details }),
|
|
155
|
+
advisories: async (asked, { github, known, details, cache }) => await repositoryAdvisories(github, asked, { repoOf: await reposOf(asked, known, soldeerRepos), details, cache }),
|
|
108
156
|
}
|
package/src/advisories/index.js
CHANGED
|
@@ -2,7 +2,7 @@ import assert from 'node:assert/strict'
|
|
|
2
2
|
|
|
3
3
|
import { assertArgs, assertBoolean, assertRepo, assertion, isIterable, optional, sameName } from '../args.js'
|
|
4
4
|
import { order } from './common.js'
|
|
5
|
-
import { GITHUB, SOLDEER, assertClient } from './github.js'
|
|
5
|
+
import { GITHUB, SOLDEER, assertCacheStore, assertClient } from './github.js'
|
|
6
6
|
import { NPM } from './npm.js'
|
|
7
7
|
import { CARGO, COMPOSER } from './osv.js'
|
|
8
8
|
|
|
@@ -34,7 +34,7 @@ function collect(packages) {
|
|
|
34
34
|
}
|
|
35
35
|
|
|
36
36
|
export async function advisories(packages, options = {}) {
|
|
37
|
-
assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean) })
|
|
37
|
+
assertArgs('advisories', options, { github: optional(assertClient), repoAdvisories: optional(assertBoolean), details: optional(assertBoolean), cache: optional(assertCacheStore) })
|
|
38
38
|
assert.ok(!options.repoAdvisories || options.github, 'advisories: repoAdvisories needs a github client')
|
|
39
39
|
const byEcosystem = collect(packages)
|
|
40
40
|
for (const ecosystem of byEcosystem.keys()) assert.ok(!ECOSYSTEMS[ecosystem].repositoryOnly || options.github, `advisories: ${ecosystem} packages need a github client`)
|
|
@@ -43,7 +43,7 @@ export async function advisories(packages, options = {}) {
|
|
|
43
43
|
const names = [...named.keys()].toSorted()
|
|
44
44
|
const asked = new Map(names.map((name) => [name, [...named.get(name).versions].toSorted(compare)]))
|
|
45
45
|
const known = new Map(names.filter((name) => named.get(name).github).map((name) => [name, named.get(name).github]))
|
|
46
|
-
const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, known })
|
|
46
|
+
const rows = await find(asked, { github: options.github, repoAdvisories: options.repoAdvisories === true, details: options.details === true, known, cache: options.cache })
|
|
47
47
|
return rows.map((row) => ({ ecosystem, ...row }))
|
|
48
48
|
}))
|
|
49
49
|
return found.flat().filter((row) => row.versions.length > 0).toSorted((a, b) => order(a.ecosystem, b.ecosystem) || order(a.name, b.name))
|
package/src/cache.js
CHANGED
|
@@ -3,12 +3,20 @@ import { constants } from 'node:fs'
|
|
|
3
3
|
import { mkdir, open, rename, rm, writeFile } from 'node:fs/promises'
|
|
4
4
|
import { homedir } from 'node:os'
|
|
5
5
|
import { dirname, isAbsolute, join, resolve } from 'node:path'
|
|
6
|
+
import { promisify } from 'node:util'
|
|
7
|
+
import { brotliCompress, brotliDecompress, constants as zlib } from 'node:zlib'
|
|
6
8
|
|
|
7
9
|
import { assertCacheName, assertDirectoryPath, isRepo } from './args.js'
|
|
8
10
|
import { MAX_BYTES, decode } from './http.js'
|
|
9
11
|
|
|
10
|
-
const DIRS = new Set(['npm/repos', 'npm/tarballs', 'cargo/repos', 'cargo/crates', 'composer/repos', 'soldeer/repos', 'soldeer/zips', 'github/trees'])
|
|
12
|
+
const DIRS = new Set(['npm/repos', 'npm/tarballs', 'cargo/repos', 'cargo/crates', 'composer/repos', 'soldeer/repos', 'soldeer/zips', 'github/trees', 'github/advisories'])
|
|
11
13
|
const RECORD_TTL_MS = 30 * 24 * 60 * 60 * 1000 // A link only moves on a transfer or rename, and GitHub redirects those.
|
|
14
|
+
// Records filed brotli'd, at quality 9, as `<name>.json.br`: a listing
|
|
15
|
+
// carries each advisory's full text, Markdown that compresses well.
|
|
16
|
+
const BROTLI_DIRS = new Set(['github/advisories'])
|
|
17
|
+
const BROTLI = { params: { [zlib.BROTLI_PARAM_QUALITY]: 9 } }
|
|
18
|
+
const compress = promisify(brotliCompress)
|
|
19
|
+
const decompress = promisify(brotliDecompress)
|
|
12
20
|
|
|
13
21
|
// An environment variable may hold anything: only an absolute path is one.
|
|
14
22
|
const absolute = (path) => (typeof path === 'string' && isAbsolute(path) ? path : undefined)
|
|
@@ -95,10 +103,12 @@ export async function readCache(dir, key) {
|
|
|
95
103
|
return path && await readRegularFile(path)
|
|
96
104
|
}
|
|
97
105
|
|
|
98
|
-
|
|
106
|
+
// With `brotli`, the file is decompressed first, to no more than a file
|
|
107
|
+
// read; one that is not brotli, or runs past that, is a miss.
|
|
108
|
+
export async function readCacheJSON(dir, key, { brotli = false } = {}) {
|
|
99
109
|
const bytes = await readCache(dir, key)
|
|
100
110
|
try {
|
|
101
|
-
return bytes && JSON.parse(decode(bytes, key))
|
|
111
|
+
return bytes && JSON.parse(decode(brotli ? await decompress(bytes, { maxOutputLength: MAX_BYTES }) : bytes, key))
|
|
102
112
|
} catch {
|
|
103
113
|
return null
|
|
104
114
|
}
|
|
@@ -119,18 +129,31 @@ export async function writeCache(dir, key, data) {
|
|
|
119
129
|
}
|
|
120
130
|
}
|
|
121
131
|
|
|
122
|
-
export async function writeCacheJSON(dir, key, value) {
|
|
123
|
-
|
|
132
|
+
export async function writeCacheJSON(dir, key, value, { brotli = false } = {}) {
|
|
133
|
+
const json = JSON.stringify(value)
|
|
134
|
+
return await writeCache(dir, key, brotli ? await compress(json, BROTLI) : json)
|
|
124
135
|
}
|
|
125
136
|
|
|
126
|
-
|
|
127
|
-
|
|
137
|
+
// A record is kept for `ttl`, in the cache set, or with `store`, in the
|
|
138
|
+
// caller's store instead, under `<dir>/<name>`, as it is: only the files
|
|
139
|
+
// are compressed.
|
|
140
|
+
const recordFile = (dir, name) => (BROTLI_DIRS.has(dir) ? `${name}.json.br` : `${name}.json`)
|
|
141
|
+
|
|
142
|
+
export async function readRecord(dir, name, { ttl = RECORD_TTL_MS, store } = {}) {
|
|
143
|
+
assert.ok(DIRS.has(dir), `Unexpected cache entry: ${dir}`)
|
|
144
|
+
const record = store ? await store.read(`${dir}/${name}`) : await readCacheJSON(dir, recordFile(dir, name), { brotli: BROTLI_DIRS.has(dir) })
|
|
128
145
|
const age = typeof record?.at === 'number' ? Date.now() - record.at : Number.NaN
|
|
129
|
-
const fresh = age >= 0 && age <=
|
|
146
|
+
const fresh = age >= 0 && age <= ttl // An entry from the future is not fresh forever.
|
|
130
147
|
return fresh && record.name === name ? record : null
|
|
131
148
|
}
|
|
132
149
|
|
|
133
|
-
export
|
|
150
|
+
export async function writeRecord(dir, name, value, { store } = {}) {
|
|
151
|
+
assert.ok(DIRS.has(dir), `Unexpected cache entry: ${dir}`)
|
|
152
|
+
const record = { at: Date.now(), name, ...value }
|
|
153
|
+
if (!store) return await writeCacheJSON(dir, recordFile(dir, name), record, { brotli: BROTLI_DIRS.has(dir) })
|
|
154
|
+
await store.write(`${dir}/${name}`, record)
|
|
155
|
+
return true
|
|
156
|
+
}
|
|
134
157
|
|
|
135
158
|
// Name → the GitHub repo cached for it, for those of `names` that have one.
|
|
136
159
|
export async function readRepos(dir, names) {
|
package/src/git.js
CHANGED
|
@@ -54,5 +54,5 @@ export async function findGitCheckout(dir) {
|
|
|
54
54
|
const directory = relative(dirs.root, start).split(sep).join('/')
|
|
55
55
|
const github = await originGitHub(dirs.commonDir)
|
|
56
56
|
const commit = await headCommit(dirs.gitDir, dirs.commonDir)
|
|
57
|
-
return { ...(github && { github }),
|
|
57
|
+
return { ...(github && { github }), directory, ...(github && { url: `https://github.com/${github}` }), ...(commit && { commit }) }
|
|
58
58
|
}
|
package/src/npm/repos.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import assert from 'node:assert/strict'
|
|
2
2
|
|
|
3
|
-
import { assertNames, assertPackageName, assertion, isRepo } from '../args.js'
|
|
3
|
+
import { assertNames, assertPackageName, assertion, isRepo, optional } from '../args.js'
|
|
4
4
|
import { readRecord, writeRecord } from '../cache.js'
|
|
5
5
|
import { isNotFound, recover } from '../http.js'
|
|
6
6
|
import { assertRepoDirectory, getRepo, isRepoDirectory } from '../package.js'
|
|
@@ -12,8 +12,10 @@ const CONCURRENCY = 8
|
|
|
12
12
|
// Stamped on each entry, and raised when getRepo would answer differently:
|
|
13
13
|
// 2 is `repository` taking precedence over `bugs`; 3 the shorthand dropping
|
|
14
14
|
// a `.git`, URLs read past whitespace, and `repository.directory`'s `\` read
|
|
15
|
-
// as
|
|
16
|
-
|
|
15
|
+
// as `/`; 4 a declared root answered as `''`, no directory declared left
|
|
16
|
+
// out (unknown) rather than read as the root, and a declared directory's
|
|
17
|
+
// empty and `.` parts dropped.
|
|
18
|
+
const VERSION = 4
|
|
17
19
|
|
|
18
20
|
// A slug a lookup gives: getRepo answers no repo ending in `.git`, the
|
|
19
21
|
// suffix it drops (withoutDotGit; `.GIT` it keeps, and so does this).
|
|
@@ -33,23 +35,23 @@ export async function getGitHub(name) {
|
|
|
33
35
|
|
|
34
36
|
// An entry under another VERSION was resolved by other rules, and may
|
|
35
37
|
// name a repo getRepo no longer would (a stale tracker's, or a shorthand's
|
|
36
|
-
// with its `.git
|
|
37
|
-
//
|
|
38
|
-
//
|
|
38
|
+
// with its `.git`, or the root where a package declared no directory): a
|
|
39
|
+
// miss, as is one naming a repo or a directory no lookup gives. One without
|
|
40
|
+
// `directory` is a package whose place in its repo is unknown.
|
|
39
41
|
export async function readPackageRepoCache(name) {
|
|
40
42
|
assertPackageName('readPackageRepoCache', 'name', name)
|
|
41
43
|
const entry = await readRecord(DIR, name)
|
|
42
|
-
if (entry?.v !== VERSION || !isLookedUpRepo(entry.github) || !isRepoDirectory(entry.directory)) return null
|
|
43
|
-
return { github: entry.github, ...(entry.directory && { directory: entry.directory }) }
|
|
44
|
+
if (entry?.v !== VERSION || !isLookedUpRepo(entry.github) || (entry.directory !== undefined && !isRepoDirectory(entry.directory))) return null
|
|
45
|
+
return { github: entry.github, ...(entry.directory !== undefined && { directory: entry.directory }) }
|
|
44
46
|
}
|
|
45
47
|
|
|
46
48
|
// Only resolved repos are cached: a 404, a rate limit and a package with
|
|
47
49
|
// no repo link fail alike.
|
|
48
|
-
export async function writePackageRepoCache(name, github, directory
|
|
50
|
+
export async function writePackageRepoCache(name, github, directory) {
|
|
49
51
|
assertPackageName('writePackageRepoCache', 'name', name)
|
|
50
52
|
assertLookedUpRepo('writePackageRepoCache', 'github', github)
|
|
51
|
-
assertRepoDirectory('writePackageRepoCache', 'directory', directory)
|
|
52
|
-
return await writeRecord(DIR, name, { v: VERSION, github, directory })
|
|
53
|
+
optional(assertRepoDirectory)('writePackageRepoCache', 'directory', directory)
|
|
54
|
+
return await writeRecord(DIR, name, { v: VERSION, github, ...(directory !== undefined && { directory }) })
|
|
53
55
|
}
|
|
54
56
|
|
|
55
57
|
export async function resolvePackageRepos(packageNames, options = {}) {
|
|
@@ -66,5 +68,5 @@ export async function lookUpPackageRepo(name) {
|
|
|
66
68
|
const { github, directory } = await fetchRepo('lookUpPackageRepo', name).catch(recover(isNotFound, {}))
|
|
67
69
|
if (!github) return null
|
|
68
70
|
await writePackageRepoCache(name, github, directory)
|
|
69
|
-
return { github, ...(directory && { directory }) }
|
|
71
|
+
return { github, ...(directory !== undefined && { directory }) }
|
|
70
72
|
}
|
package/src/package.js
CHANGED
|
@@ -38,8 +38,16 @@ function repoSubdirectory(value) {
|
|
|
38
38
|
|
|
39
39
|
// `repository.directory` is a path its author may write with Windows' `\`.
|
|
40
40
|
// A tree path is `/`-separated: a `\` is a name's own only in a homepage's
|
|
41
|
-
// (`%5C`), which is the path itself.
|
|
42
|
-
|
|
41
|
+
// (`%5C`), which is the path itself. Its empty and `.` parts are dropped
|
|
42
|
+
// (`./a//b/.` is `a/b`), and one of nothing else (`./`, `/`, `/.`, `.`, `''`)
|
|
43
|
+
// declares the repo's root, `''`; one with a `..` part names none, whatever
|
|
44
|
+
// it would come to. A package.json that declares none says nothing of where
|
|
45
|
+
// in the repo it sits.
|
|
46
|
+
function declaredDirectory(value) {
|
|
47
|
+
if (typeof value !== 'string') return undefined
|
|
48
|
+
const parts = value.replaceAll('\\', '/').split('/').filter((part) => part !== '' && part !== '.')
|
|
49
|
+
return parts.length === 0 ? '' : repoSubdirectory(parts.join('/'))
|
|
50
|
+
}
|
|
43
51
|
|
|
44
52
|
export const isRepoDirectory = (value) => typeof value === 'string' && (value === '' || repoSubdirectory(value) === value)
|
|
45
53
|
export const assertRepoDirectory = assertion('a path inside the repository', isRepoDirectory)
|
|
@@ -54,7 +62,7 @@ export function getRepo(pkg) {
|
|
|
54
62
|
// `bugs` tracker can be left pointing at a former owner or misspelt.
|
|
55
63
|
const github = [declared, repoIn(bugsRegex, urlOf(bugs)), homepageRepo].find(isRepo)
|
|
56
64
|
if (github === undefined) return {}
|
|
57
|
-
const directory = (sameName(declared, github)
|
|
58
|
-
|
|
59
|
-
return { github, ...(directory && { directory }), url: `https://github.com/${github}` }
|
|
65
|
+
const directory = (sameName(declared, github) ? declaredDirectory(repository?.directory) : undefined)
|
|
66
|
+
?? (sameName(homepageRepo, github) ? repoSubdirectory(decodePath(homepage.directory)) : undefined)
|
|
67
|
+
return { github, ...(directory !== undefined && { directory }), url: `https://github.com/${github}` }
|
|
60
68
|
}
|