cve-lite-cli 1.40.0 → 1.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -11
- package/dist/advisory/local-advisory-source.js +9 -5
- package/dist/advisory/local-db.js +91 -13
- package/dist/advisory/osv-sync.js +187 -56
- package/dist/advisory/staging-directory.js +112 -0
- package/dist/cli/advisories-sync.js +16 -2
- package/dist/cli/args.js +46 -35
- package/dist/cli/config.js +4 -1
- package/dist/cli/help.js +4 -3
- package/dist/cli/validate.js +23 -1
- package/dist/constants.js +11 -25
- package/dist/index.js +8 -3
- package/dist/output/cyclonedx.js +23 -14
- package/dist/output/debug.js +68 -18
- package/dist/output/formatters.js +33 -5
- package/dist/output/html-reporter.js +55 -5
- package/dist/output/multi-folder-html-reporter.js +59 -3
- package/dist/output/output-dir.js +193 -7
- package/dist/output/printers.js +16 -5
- package/dist/output/sarif.js +131 -48
- package/dist/output/sbom-dependency-edges.js +16 -10
- package/dist/output/sbom-file.js +14 -6
- package/dist/output/scan-json.js +21 -3
- package/dist/output/spdx.js +61 -23
- package/dist/output/spinner.js +2 -1
- package/dist/output/write-outputs.js +142 -52
- package/dist/overrides/context-builder.js +23 -70
- package/dist/parsers/bun-lock.js +1 -2
- package/dist/parsers/index.js +22 -1
- package/dist/parsers/npm-lock-graph.js +39 -1
- package/dist/parsers/package-lock.js +25 -5
- package/dist/parsers/pnpm-lock.js +3 -2
- package/dist/parsers/utils.js +5 -3
- package/dist/parsers/yarn-lock-graph.js +5 -2
- package/dist/parsers/yarn-lock.js +1 -2
- package/dist/remediation/fix-commands.js +105 -44
- package/dist/remediation/npm-transitive-graph.js +30 -72
- package/dist/remediation/npm-transitive-resolution.js +24 -77
- package/dist/remediation/parent-upgrade-blockers.js +51 -0
- package/dist/remediation/parent-upgrade.js +5 -79
- package/dist/remediation/registry-parent-ranges.js +21 -0
- package/dist/remediation/transitive-chain-resolver.js +3 -3
- package/dist/scan/completeness.js +10 -4
- package/dist/scan/multi-folder-scan.js +133 -58
- package/dist/scan/single-scan.js +41 -25
- package/dist/scanner.js +257 -58
- package/dist/usage/scanner.js +19 -8
- package/dist/utils/array.js +11 -3
- package/dist/utils/debug-bundle.js +118 -0
- package/dist/utils/debug-project.js +102 -0
- package/dist/utils/debug-redaction.js +117 -0
- package/dist/utils/file.js +2 -1
- package/dist/utils/fix-runner.js +95 -13
- package/dist/utils/import-type-queries.js +322 -0
- package/dist/utils/network.js +13 -9
- package/dist/utils/npm-lockfile.js +12 -0
- package/dist/utils/osv-fetch.js +70 -0
- package/dist/utils/package-identity.js +43 -0
- package/dist/utils/package-manager-source.js +24 -0
- package/dist/utils/purl.js +37 -0
- package/dist/utils/registry-config.js +1 -16
- package/dist/utils/string.js +8 -0
- package/dist/utils/validate-url.js +1 -1
- package/dist/utils/version.js +30 -7
- package/dist/utils/versions/osv-range.js +191 -0
- package/dist/utils/versions/pep440.js +246 -0
- package/dist/utils/versions/scheme.js +10 -0
- package/package.json +3 -2
package/README.md
CHANGED
|
@@ -57,12 +57,12 @@
|
|
|
57
57
|
</p>
|
|
58
58
|
|
|
59
59
|
<p>
|
|
60
|
-
<sub>
|
|
60
|
+
<sub>Used by
|
|
61
61
|
<a href="https://github.com/solidjs/solid/blob/main/.github/workflows/cve-lite.yml"><strong>SolidJS</strong></a>,
|
|
62
|
-
<a href="https://github.com/
|
|
63
|
-
<a href="https://github.com/
|
|
64
|
-
<a href="https://github.com/
|
|
65
|
-
In
|
|
62
|
+
<a href="https://github.com/open-circle/valibot/blob/main/.github/workflows/security.yml"><strong>Valibot</strong></a>,
|
|
63
|
+
<a href="https://github.com/bcgov/nr-waste-plus/blob/main/.github/workflows/analysis.yml"><strong>the Government of British Columbia</strong></a> as a required merge gate, and
|
|
64
|
+
<a href="https://github.com/refugies-info/karfur/blob/dev/package.json"><strong>the Government of France</strong></a>.<br/>
|
|
65
|
+
In public repositories in France, Canada, Germany, Thailand and China. <a href="https://owasp.org/cve-lite-cli/who-uses">See who uses it</a>.</sub>
|
|
66
66
|
</p>
|
|
67
67
|
|
|
68
68
|
<br/>
|
|
@@ -370,16 +370,22 @@ CVE Lite CLI complements these tools. It is not a replacement for continuous mon
|
|
|
370
370
|
|
|
371
371
|
## Community and adoption
|
|
372
372
|
|
|
373
|
-
CVE Lite CLI
|
|
373
|
+
CVE Lite CLI runs in public repositories across governments, open-source libraries, companies, and independent developers. The [Who uses CVE Lite CLI?](https://owasp.org/cve-lite-cli/who-uses) page links every adopter to the public file that proves it.
|
|
374
374
|
|
|
375
375
|
Notable adopters include:
|
|
376
376
|
|
|
377
|
-
- **SolidJS** - the framework's creator, Ryan Carniato, added CVE Lite CLI
|
|
378
|
-
- **
|
|
379
|
-
- **
|
|
380
|
-
- **Government of
|
|
381
|
-
- **Valibot** and **Formisch** - integrated as a GitHub Action and dev script with branch protection
|
|
377
|
+
- **SolidJS** - the framework's creator, Ryan Carniato, added CVE Lite CLI to its CI in his own commit. It scans every push and pull request to the main branch and reports results to GitHub Code Scanning
|
|
378
|
+
- **Valibot** and **Formisch** - security workflows that run the scan and publish results to GitHub Code Scanning, merged by their creator, Fabian Hiller. Valibot has about 14 million npm downloads a week
|
|
379
|
+
- **Government of British Columbia** (Canada) - a required check on every push and pull request in its Waste Plus natural resources application, alongside Trivy and npm audit
|
|
380
|
+
- **Government of France** - pre-push scans that block high and critical vulnerabilities in Réfugiés.info, the government's information portal for refugees, and in IA dans l'État, the state's interministerial AI programme
|
|
382
381
|
|
|
382
|
+
## Teaching and learning
|
|
383
|
+
|
|
384
|
+
Teach dependency security with CVE Lite Classroom: free, open-source, hands-on exercises covering vulnerability remediation, critical review of AI-assisted fixes, and CI security gates. Includes runnable applications, instructor guides, worked solutions, and grading rubrics, with tracks for npm, pnpm, Yarn, and Bun.
|
|
385
|
+
|
|
386
|
+
[Explore the classroom exercises](https://sonukapoor.github.io/cve-lite-classroom/)
|
|
387
|
+
|
|
388
|
+
CVE Lite Classroom is a companion project maintained by the author of CVE Lite CLI, separate from the OWASP project.
|
|
383
389
|
|
|
384
390
|
## Current limitations
|
|
385
391
|
|
|
@@ -4,11 +4,15 @@ export class LocalAdvisorySource {
|
|
|
4
4
|
this.db = db;
|
|
5
5
|
}
|
|
6
6
|
queryBatch(packages, _meta) {
|
|
7
|
-
const results = packages.map(pkg =>
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
7
|
+
const results = packages.map(pkg => {
|
|
8
|
+
const match = this.db.matchPackage(pkg);
|
|
9
|
+
return {
|
|
10
|
+
package: pkg.name,
|
|
11
|
+
version: pkg.version,
|
|
12
|
+
vulnerabilities: match.ids.map(id => ({ id })),
|
|
13
|
+
...(match.unreadable ? { unreadableBoundary: true } : {}),
|
|
14
|
+
};
|
|
15
|
+
});
|
|
12
16
|
return Promise.resolve(results);
|
|
13
17
|
}
|
|
14
18
|
getVuln(id) {
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import Database from "better-sqlite3";
|
|
4
|
-
import { compareVersions, isPreReleaseVersion, looksLikeVersion, normalizeAdvisoryVersion } from "../utils/version.js";
|
|
4
|
+
import { compareVersions, isPep440Ecosystem, isPreReleaseVersion, looksLikeVersion, normalizeAdvisoryVersion } from "../utils/version.js";
|
|
5
|
+
import { readPep440Range, versionInPep440Window } from "../utils/versions/osv-range.js";
|
|
6
|
+
import { pep440Scheme } from "../utils/versions/pep440.js";
|
|
5
7
|
const SCHEMA_SQL = `
|
|
6
8
|
CREATE TABLE IF NOT EXISTS advisories (
|
|
7
9
|
id TEXT PRIMARY KEY,
|
|
@@ -164,17 +166,35 @@ export class LocalAdvisoryDatabase {
|
|
|
164
166
|
}
|
|
165
167
|
}
|
|
166
168
|
findMatchingVulnerabilityIds(pkg) {
|
|
167
|
-
|
|
168
|
-
|
|
169
|
+
return this.matchPackage(pkg).ids;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Ids whose stored window contains `pkg`, plus whether a PyPI boundary that
|
|
173
|
+
* could still contain it could not be read. An unreadable boundary does not
|
|
174
|
+
* abort the lookup and is not reported as a clean miss.
|
|
175
|
+
*/
|
|
176
|
+
matchPackage(pkg) {
|
|
177
|
+
if (!looksLikeVersion(pkg.version, pkg.ecosystem)) {
|
|
178
|
+
// Semver returns no matches for a tag or a range. A PyPI version that
|
|
179
|
+
// cannot be read is inconclusive, not unaffected.
|
|
180
|
+
return { ids: [], unreadable: isPep440Ecosystem(pkg.ecosystem) };
|
|
169
181
|
}
|
|
170
182
|
const rows = this.findMatchingIdsStatement.all(pkg.ecosystem, pkg.name);
|
|
171
183
|
const ids = new Set();
|
|
184
|
+
let unreadable = false;
|
|
172
185
|
for (const row of rows) {
|
|
173
|
-
if (
|
|
186
|
+
if (isPep440Ecosystem(pkg.ecosystem)) {
|
|
187
|
+
const verdict = pep440RowVerdict(pkg.version, row);
|
|
188
|
+
if (verdict === "unreadable")
|
|
189
|
+
unreadable = true;
|
|
190
|
+
else if (verdict === "match")
|
|
191
|
+
ids.add(row.advisory_id);
|
|
192
|
+
}
|
|
193
|
+
else if (versionMatchesRange(pkg.version, row, pkg.ecosystem)) {
|
|
174
194
|
ids.add(row.advisory_id);
|
|
175
195
|
}
|
|
176
196
|
}
|
|
177
|
-
return [...ids];
|
|
197
|
+
return { ids: [...ids], unreadable };
|
|
178
198
|
}
|
|
179
199
|
}
|
|
180
200
|
/**
|
|
@@ -201,32 +221,61 @@ function releaseCore(version) {
|
|
|
201
221
|
* negative for a false positive. No guard is needed for that: a release core
|
|
202
222
|
* never contains "-", so it can never equal a pre-release `introduced`.
|
|
203
223
|
*/
|
|
204
|
-
function versionAtOrAboveIntroduced(version, introduced) {
|
|
205
|
-
|
|
224
|
+
function versionAtOrAboveIntroduced(version, introduced, ecosystem) {
|
|
225
|
+
// PEP 440 orders dev and pre-releases strictly, below the final release.
|
|
226
|
+
if (isPep440Ecosystem(ecosystem))
|
|
227
|
+
return pep440Scheme.atOrAboveIntroduced(version, introduced);
|
|
228
|
+
if (compareVersions(version, introduced, ecosystem) >= 0)
|
|
206
229
|
return true;
|
|
230
|
+
// The pre-release relaxation is a semver rule.
|
|
207
231
|
return isPreReleaseVersion(version) && releaseCore(version) === introduced;
|
|
208
232
|
}
|
|
209
233
|
/** OSV range types whose boundaries are versions we can order. GIT uses commit hashes. */
|
|
210
234
|
const COMPARABLE_RANGE_TYPES = new Set(["ECOSYSTEM", "SEMVER"]);
|
|
211
|
-
function
|
|
235
|
+
function pep440RowVerdict(version, row) {
|
|
236
|
+
const introducedSentinel = row.introduced === "0";
|
|
237
|
+
const introduced = row.introduced === null || introducedSentinel
|
|
238
|
+
? row.introduced
|
|
239
|
+
: normalizeAdvisoryVersion(row.introduced, "PyPI");
|
|
240
|
+
if (row.introduced !== null && !introducedSentinel) {
|
|
241
|
+
if (!introduced) {
|
|
242
|
+
const readableFixed = row.fixed === null ? null : normalizeAdvisoryVersion(row.fixed, "PyPI");
|
|
243
|
+
if (readableFixed && compareVersions(version, readableFixed, "PyPI") >= 0)
|
|
244
|
+
return "miss";
|
|
245
|
+
return "unreadable";
|
|
246
|
+
}
|
|
247
|
+
if (!versionAtOrAboveIntroduced(version, introduced, "PyPI"))
|
|
248
|
+
return "miss";
|
|
249
|
+
}
|
|
250
|
+
const fixed = row.fixed === null ? null : normalizeAdvisoryVersion(row.fixed, "PyPI");
|
|
251
|
+
const lastAffected = row.last_affected === null ? null : normalizeAdvisoryVersion(row.last_affected, "PyPI");
|
|
252
|
+
if ((row.fixed !== null && !fixed) || (row.last_affected !== null && !lastAffected))
|
|
253
|
+
return "unreadable";
|
|
254
|
+
return versionInPep440Window(version, {
|
|
255
|
+
introduced: introducedSentinel ? "0" : introduced,
|
|
256
|
+
fixed,
|
|
257
|
+
lastAffected,
|
|
258
|
+
}) ? "match" : "miss";
|
|
259
|
+
}
|
|
260
|
+
function versionMatchesRange(version, row, ecosystem) {
|
|
212
261
|
const introduced = row.introduced;
|
|
213
262
|
// Defence in depth behind the range-type filter, which stops these rows being
|
|
214
263
|
// written in the first place. A database built by an earlier version still
|
|
215
264
|
// holds them, and a bound that is not a version must never decide the answer
|
|
216
265
|
// by silently failing to exclude (#1191).
|
|
217
|
-
const fixed = normalizeAdvisoryVersion(row.fixed);
|
|
218
|
-
const lastAffected = normalizeAdvisoryVersion(row.last_affected);
|
|
266
|
+
const fixed = normalizeAdvisoryVersion(row.fixed, ecosystem);
|
|
267
|
+
const lastAffected = normalizeAdvisoryVersion(row.last_affected, ecosystem);
|
|
219
268
|
if (row.fixed && !fixed)
|
|
220
269
|
return false;
|
|
221
270
|
if (row.last_affected && !lastAffected)
|
|
222
271
|
return false;
|
|
223
|
-
if (introduced && introduced !== "0" && !versionAtOrAboveIntroduced(version, introduced)) {
|
|
272
|
+
if (introduced && introduced !== "0" && !versionAtOrAboveIntroduced(version, introduced, ecosystem)) {
|
|
224
273
|
return false;
|
|
225
274
|
}
|
|
226
|
-
if (fixed && compareVersions(version, fixed) >= 0) {
|
|
275
|
+
if (fixed && compareVersions(version, fixed, ecosystem) >= 0) {
|
|
227
276
|
return false;
|
|
228
277
|
}
|
|
229
|
-
if (lastAffected && compareVersions(version, lastAffected) > 0) {
|
|
278
|
+
if (lastAffected && compareVersions(version, lastAffected, ecosystem) > 0) {
|
|
230
279
|
return false;
|
|
231
280
|
}
|
|
232
281
|
return true;
|
|
@@ -250,6 +299,10 @@ function deriveAdvisoryPackageRows(vuln) {
|
|
|
250
299
|
if (range.type && !COMPARABLE_RANGE_TYPES.has(range.type))
|
|
251
300
|
continue;
|
|
252
301
|
const events = range.events ?? [];
|
|
302
|
+
if (isPep440Ecosystem(ecosystem)) {
|
|
303
|
+
rows.push(...pep440PackageRows(vuln.id, ecosystem, packageName, events));
|
|
304
|
+
continue;
|
|
305
|
+
}
|
|
253
306
|
let introduced = null;
|
|
254
307
|
for (const event of events) {
|
|
255
308
|
if (event.introduced) {
|
|
@@ -291,3 +344,28 @@ function deriveAdvisoryPackageRows(vuln) {
|
|
|
291
344
|
}
|
|
292
345
|
return rows;
|
|
293
346
|
}
|
|
347
|
+
function pep440PackageRows(advisoryId, ecosystem, packageName, events) {
|
|
348
|
+
const read = readPep440Range(events);
|
|
349
|
+
const rows = [];
|
|
350
|
+
for (const window of read.windows) {
|
|
351
|
+
rows.push({
|
|
352
|
+
advisory_id: advisoryId,
|
|
353
|
+
ecosystem,
|
|
354
|
+
package_name: packageName,
|
|
355
|
+
introduced: window.introduced,
|
|
356
|
+
fixed: window.fixed,
|
|
357
|
+
last_affected: window.lastAffected,
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
for (const boundary of read.unreadable) {
|
|
361
|
+
rows.push({
|
|
362
|
+
advisory_id: advisoryId,
|
|
363
|
+
ecosystem,
|
|
364
|
+
package_name: packageName,
|
|
365
|
+
introduced: boundary.field === "introduced" ? boundary.value : boundary.introduced,
|
|
366
|
+
fixed: boundary.field === "fixed" ? boundary.value : null,
|
|
367
|
+
last_affected: boundary.field === "last_affected" ? boundary.value : null,
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
return rows;
|
|
371
|
+
}
|
|
@@ -1,11 +1,21 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import os from "node:os";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
4
5
|
import { unzipSync } from "fflate";
|
|
5
6
|
import { LocalAdvisoryDatabase } from "./local-db.js";
|
|
6
|
-
import { pluralize } from "../utils/string.js";
|
|
7
|
+
import { ensureSentence, pluralize } from "../utils/string.js";
|
|
8
|
+
import { cleanupAbandonedStagingDirectories, recordStagingOwner, stagingDirectoryPrefix } from "./staging-directory.js";
|
|
7
9
|
const DEFAULT_OSV_NPM_DUMP_URL = "https://storage.googleapis.com/osv-vulnerabilities/npm/all.zip";
|
|
8
10
|
export const ADVISORY_DB_STALE_AFTER_MS = 7 * 24 * 60 * 60 * 1000;
|
|
11
|
+
export class AdvisorySyncError extends Error {
|
|
12
|
+
databaseState;
|
|
13
|
+
constructor(message, databaseState, cause) {
|
|
14
|
+
super(message, { cause });
|
|
15
|
+
this.databaseState = databaseState;
|
|
16
|
+
this.name = "AdvisorySyncError";
|
|
17
|
+
}
|
|
18
|
+
}
|
|
9
19
|
export function getDefaultAdvisoryDbPath(outputPath) {
|
|
10
20
|
if (outputPath) {
|
|
11
21
|
return path.resolve(outputPath);
|
|
@@ -16,21 +26,121 @@ export function getDefaultAdvisoryDbPath(outputPath) {
|
|
|
16
26
|
}
|
|
17
27
|
return path.join(os.homedir(), ".cache", "cve-lite", "advisories.db");
|
|
18
28
|
}
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
29
|
+
/**
|
|
30
|
+
* Replaces the advisory database after a successful sync. Node's rename replaces
|
|
31
|
+
* existing files on Windows too, but an open or locked target can cause EPERM.
|
|
32
|
+
* If replacement is refused, move the old database aside only after the new one
|
|
33
|
+
* is complete and restore it if the second rename fails.
|
|
34
|
+
*/
|
|
35
|
+
function replaceAdvisoryDatabase(tempPath, dbPath) {
|
|
36
|
+
try {
|
|
37
|
+
fs.renameSync(tempPath, dbPath);
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
const code = error.code;
|
|
42
|
+
if (code !== "EEXIST" && code !== "EPERM")
|
|
43
|
+
throw error;
|
|
44
|
+
}
|
|
45
|
+
const backupPath = `${dbPath}.backup-${process.pid}-${randomUUID()}`;
|
|
46
|
+
let movedExisting = false;
|
|
47
|
+
try {
|
|
48
|
+
fs.renameSync(dbPath, backupPath);
|
|
49
|
+
movedExisting = true;
|
|
50
|
+
fs.renameSync(tempPath, dbPath);
|
|
51
|
+
}
|
|
52
|
+
catch (error) {
|
|
53
|
+
if (!movedExisting)
|
|
54
|
+
throw error;
|
|
55
|
+
restoreAdvisoryDatabase(backupPath, dbPath, error);
|
|
56
|
+
}
|
|
57
|
+
return cleanupBackup(backupPath, "The refreshed database was installed.");
|
|
58
|
+
}
|
|
59
|
+
function cleanupBackup(backupPath, databaseMessage) {
|
|
60
|
+
try {
|
|
61
|
+
fs.rmSync(backupPath, { force: true });
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
65
|
+
return `Could not remove advisory database backup ${backupPath}: ${ensureSentence(message)} `
|
|
66
|
+
+ `${databaseMessage} Resolve the filesystem error and remove ${backupPath} manually.`;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
function restoreAdvisoryDatabase(backupPath, dbPath, installError) {
|
|
70
|
+
const originalMessage = installError instanceof Error ? installError.message : String(installError);
|
|
71
|
+
try {
|
|
72
|
+
// Hardlink creation is atomic and refuses an existing target. An exists-check
|
|
73
|
+
// followed by rename can overwrite a database installed by another sync.
|
|
74
|
+
fs.linkSync(backupPath, dbPath);
|
|
75
|
+
}
|
|
76
|
+
catch (restoreError) {
|
|
77
|
+
const restoreMessage = restoreError instanceof Error ? restoreError.message : String(restoreError);
|
|
78
|
+
throw new AdvisorySyncError(`Failed to install refreshed advisory database: ${ensureSentence(originalMessage)} `
|
|
79
|
+
+ `The previous database could not be restored and remains at ${backupPath}: ${ensureSentence(restoreMessage)} `
|
|
80
|
+
+ `No existing database at ${dbPath} was overwritten. Close any program using these files and inspect both databases. `
|
|
81
|
+
+ `After resolving the filesystem error, rename ${backupPath} to ${dbPath} only if the target is absent; do not overwrite a newer database.`, "recovery-required", restoreError);
|
|
82
|
+
}
|
|
83
|
+
const warning = cleanupBackup(backupPath, "The previous database was restored.");
|
|
84
|
+
if (warning)
|
|
85
|
+
throw new AdvisorySyncError(`${ensureSentence(originalMessage)} ${warning}`, "unchanged", installError);
|
|
86
|
+
throw installError;
|
|
87
|
+
}
|
|
88
|
+
function rejectDirectoryOutput(dbPath) {
|
|
89
|
+
try {
|
|
90
|
+
if (!fs.lstatSync(dbPath).isDirectory())
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
if (error.code === "ENOENT")
|
|
95
|
+
return;
|
|
96
|
+
throw error;
|
|
97
|
+
}
|
|
98
|
+
throw new Error(`Advisory database output ${dbPath} is a directory. Choose a database file path, for example ${path.join(dbPath, "advisories.db")}.`);
|
|
99
|
+
}
|
|
100
|
+
async function ingestAdvisories(db, advisoryEntries, sourceUrl, onProgress) {
|
|
101
|
+
const parsedVulns = [];
|
|
102
|
+
let advisoryCount = 0;
|
|
103
|
+
const progressInterval = 250;
|
|
104
|
+
for (const [, bytes] of advisoryEntries) {
|
|
105
|
+
const text = Buffer.from(bytes).toString("utf8");
|
|
106
|
+
const vuln = JSON.parse(text);
|
|
107
|
+
if (!vuln.id)
|
|
108
|
+
continue;
|
|
109
|
+
if (!Array.isArray(vuln.affected) || vuln.affected.length === 0)
|
|
110
|
+
continue;
|
|
111
|
+
// OSV's /v1/querybatch endpoint excludes withdrawn advisories at query
|
|
112
|
+
// time; mirror that here so offline scans don't surface false positives
|
|
113
|
+
// from advisories that have been retracted.
|
|
114
|
+
if (vuln.withdrawn)
|
|
115
|
+
continue;
|
|
116
|
+
parsedVulns.push(vuln);
|
|
117
|
+
advisoryCount += 1;
|
|
118
|
+
if (advisoryCount === 1 ||
|
|
119
|
+
advisoryCount % progressInterval === 0 ||
|
|
120
|
+
advisoryCount === advisoryEntries.length) {
|
|
121
|
+
onProgress?.({
|
|
122
|
+
phase: "ingest",
|
|
123
|
+
processedAdvisories: advisoryCount,
|
|
124
|
+
totalAdvisories: advisoryEntries.length,
|
|
125
|
+
message: `Processing advisory records: ${advisoryCount} / ${advisoryEntries.length}`,
|
|
126
|
+
});
|
|
127
|
+
await yieldToEventLoop();
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
db.bulkUpsertVulnerabilities(parsedVulns);
|
|
131
|
+
db.setMetadata({
|
|
132
|
+
lastSyncAt: new Date().toISOString(),
|
|
133
|
+
sourceUrl,
|
|
134
|
+
});
|
|
135
|
+
return advisoryCount;
|
|
136
|
+
}
|
|
137
|
+
async function downloadAdvisoryArchive(sourceUrl, dbPath, fetchImpl, onProgress) {
|
|
24
138
|
onProgress?.({
|
|
25
139
|
phase: "init",
|
|
26
140
|
sourceUrl,
|
|
27
141
|
dbPath,
|
|
28
142
|
message: `Sync initiated. Downloading advisory dump from ${sourceUrl}`,
|
|
29
143
|
});
|
|
30
|
-
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
31
|
-
if (fs.existsSync(dbPath)) {
|
|
32
|
-
fs.rmSync(dbPath, { force: true });
|
|
33
|
-
}
|
|
34
144
|
const response = await fetchImpl(sourceUrl);
|
|
35
145
|
if (!response.ok) {
|
|
36
146
|
throw new Error(`OSV dump download failed: ${response.status} ${response.statusText}`);
|
|
@@ -56,6 +166,9 @@ export async function syncOsvAdvisories(options = {}) {
|
|
|
56
166
|
: `Download complete: ${formatBytes(zippedBytes.byteLength)}`,
|
|
57
167
|
});
|
|
58
168
|
await yieldToEventLoop();
|
|
169
|
+
return zippedBytes;
|
|
170
|
+
}
|
|
171
|
+
async function buildAdvisoryDatabase(zippedBytes, tempDbPath, sourceUrl, onProgress) {
|
|
59
172
|
const archiveEntries = unzipSync(zippedBytes);
|
|
60
173
|
const advisoryEntries = Object.entries(archiveEntries).filter(([entryName]) => entryName.endsWith(".json"));
|
|
61
174
|
onProgress?.({
|
|
@@ -65,64 +178,82 @@ export async function syncOsvAdvisories(options = {}) {
|
|
|
65
178
|
message: `Archive loaded. Processing ${advisoryEntries.length} advisory ${pluralize(advisoryEntries.length, "record")}`,
|
|
66
179
|
});
|
|
67
180
|
await yieldToEventLoop();
|
|
68
|
-
const db = new LocalAdvisoryDatabase(
|
|
181
|
+
const db = new LocalAdvisoryDatabase(tempDbPath);
|
|
182
|
+
let advisoryCount;
|
|
69
183
|
try {
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
184
|
+
advisoryCount = await ingestAdvisories(db, advisoryEntries, sourceUrl, onProgress);
|
|
185
|
+
}
|
|
186
|
+
finally {
|
|
187
|
+
db.close();
|
|
188
|
+
}
|
|
189
|
+
if (advisoryCount === 0) {
|
|
190
|
+
throw new Error(`OSV dump from ${sourceUrl} contained no usable advisory records. `
|
|
191
|
+
+ `Check the source URL, then run "cve-lite advisories sync" again.`);
|
|
192
|
+
}
|
|
193
|
+
return advisoryCount;
|
|
194
|
+
}
|
|
195
|
+
/** Removes staged files, returning a warning for the caller to report. */
|
|
196
|
+
function cleanupTempDirectory(tempDir) {
|
|
197
|
+
try {
|
|
198
|
+
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
199
|
+
}
|
|
200
|
+
catch (error) {
|
|
201
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
202
|
+
return `Could not remove temporary advisory sync directory ${tempDir}: ${ensureSentence(message)} `
|
|
203
|
+
+ `Resolve the filesystem error and remove ${tempDir} manually.`;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
function buildSyncError(refreshError, databaseInstalled, cleanupWarning) {
|
|
207
|
+
const message = refreshError instanceof Error ? refreshError.message : String(refreshError);
|
|
208
|
+
const databaseState = refreshError instanceof AdvisorySyncError
|
|
209
|
+
? refreshError.databaseState
|
|
210
|
+
: databaseInstalled ? "replaced" : "unchanged";
|
|
211
|
+
return new AdvisorySyncError(cleanupWarning ? `${ensureSentence(message)} ${cleanupWarning}` : message, databaseState, refreshError);
|
|
212
|
+
}
|
|
213
|
+
export async function syncOsvAdvisories(options = {}) {
|
|
214
|
+
const onProgress = options.onProgress;
|
|
215
|
+
const sourceUrl = options.sourceUrl ?? DEFAULT_OSV_NPM_DUMP_URL;
|
|
216
|
+
const dbPath = getDefaultAdvisoryDbPath(options.outputPath);
|
|
217
|
+
rejectDirectoryOutput(dbPath);
|
|
218
|
+
let tempDir;
|
|
219
|
+
let databaseInstalled = false;
|
|
220
|
+
let refreshError;
|
|
221
|
+
let cleanupWarning;
|
|
222
|
+
let replacementWarning;
|
|
223
|
+
let syncResult;
|
|
224
|
+
const startupWarnings = [];
|
|
225
|
+
try {
|
|
226
|
+
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
227
|
+
startupWarnings.push(...cleanupAbandonedStagingDirectories(dbPath));
|
|
228
|
+
tempDir = fs.mkdtempSync(stagingDirectoryPrefix(dbPath));
|
|
229
|
+
recordStagingOwner(tempDir, dbPath);
|
|
230
|
+
const tempDbPath = path.join(tempDir, path.basename(dbPath));
|
|
231
|
+
const zippedBytes = await downloadAdvisoryArchive(sourceUrl, dbPath, options.fetchImpl ?? fetch, onProgress);
|
|
232
|
+
const advisoryCount = await buildAdvisoryDatabase(zippedBytes, tempDbPath, sourceUrl, onProgress);
|
|
233
|
+
replacementWarning = replaceAdvisoryDatabase(tempDbPath, dbPath);
|
|
234
|
+
databaseInstalled = true;
|
|
104
235
|
onProgress?.({
|
|
105
236
|
phase: "complete",
|
|
106
237
|
advisoryCount,
|
|
107
238
|
dbPath,
|
|
108
239
|
message: `Sync complete. Stored ${advisoryCount} advisory ${pluralize(advisoryCount, "record")} in ${dbPath}`,
|
|
109
240
|
});
|
|
110
|
-
|
|
241
|
+
syncResult = { advisoryCount, dbPath, sourceUrl };
|
|
111
242
|
}
|
|
112
243
|
catch (error) {
|
|
113
|
-
|
|
114
|
-
fs.rmSync(dbPath, { force: true });
|
|
115
|
-
}
|
|
116
|
-
throw error;
|
|
244
|
+
refreshError = error;
|
|
117
245
|
}
|
|
118
246
|
finally {
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
247
|
+
if (tempDir !== undefined)
|
|
248
|
+
cleanupWarning = cleanupTempDirectory(tempDir);
|
|
249
|
+
}
|
|
250
|
+
const warnings = [...startupWarnings, replacementWarning, cleanupWarning].filter((warning) => warning !== undefined);
|
|
251
|
+
if (syncResult === undefined) {
|
|
252
|
+
throw buildSyncError(refreshError, databaseInstalled, warnings.join(" ") || undefined);
|
|
125
253
|
}
|
|
254
|
+
if (warnings.length > 0)
|
|
255
|
+
syncResult.warnings = warnings;
|
|
256
|
+
return syncResult;
|
|
126
257
|
}
|
|
127
258
|
async function readResponseBytes(response, onChunk) {
|
|
128
259
|
if (!response.body) {
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import { ensureSentence } from "../utils/string.js";
|
|
6
|
+
const OWNER_FILE = "owner.json";
|
|
7
|
+
export function stagingDirectoryPrefix(dbPath) {
|
|
8
|
+
return path.join(path.dirname(dbPath), `.${path.basename(dbPath)}.tmp-${process.pid}-`);
|
|
9
|
+
}
|
|
10
|
+
/** Called after the caller records the directory, so finally also covers write failures. */
|
|
11
|
+
export function recordStagingOwner(tempDir, dbPath) {
|
|
12
|
+
const owner = {
|
|
13
|
+
version: 1,
|
|
14
|
+
pid: process.pid,
|
|
15
|
+
hostname: os.hostname(),
|
|
16
|
+
dbPath,
|
|
17
|
+
directoryName: path.basename(tempDir),
|
|
18
|
+
};
|
|
19
|
+
fs.writeFileSync(path.join(tempDir, OWNER_FILE), JSON.stringify(owner), { flag: "wx", mode: 0o600 });
|
|
20
|
+
}
|
|
21
|
+
function readOwner(directory, dbPath) {
|
|
22
|
+
const entry = fs.lstatSync(directory);
|
|
23
|
+
if (!entry.isDirectory() || entry.isSymbolicLink())
|
|
24
|
+
return null;
|
|
25
|
+
const metadata = fs.lstatSync(path.join(directory, OWNER_FILE));
|
|
26
|
+
if (!metadata.isFile() || metadata.isSymbolicLink())
|
|
27
|
+
return null;
|
|
28
|
+
const parsed = JSON.parse(fs.readFileSync(path.join(directory, OWNER_FILE), "utf8"));
|
|
29
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed))
|
|
30
|
+
return null;
|
|
31
|
+
const owner = parsed;
|
|
32
|
+
if (owner.version !== 1 || !Number.isSafeInteger(owner.pid) || owner.pid <= 0 || owner.pid > 2147483647)
|
|
33
|
+
return null;
|
|
34
|
+
if (owner.hostname !== os.hostname() || owner.dbPath !== dbPath || typeof owner.directoryName !== "string")
|
|
35
|
+
return null;
|
|
36
|
+
const prefix = `.${path.basename(dbPath)}.tmp-${owner.pid}-`;
|
|
37
|
+
if (!owner.directoryName.startsWith(prefix) || path.basename(owner.directoryName) !== owner.directoryName)
|
|
38
|
+
return null;
|
|
39
|
+
const name = path.basename(directory);
|
|
40
|
+
if (name !== owner.directoryName && !name.startsWith(`${owner.directoryName}.cleanup-`))
|
|
41
|
+
return null;
|
|
42
|
+
// On POSIX, do not sweep another user's staging directory, even with matching metadata.
|
|
43
|
+
if (process.getuid && (entry.uid !== process.getuid() || metadata.uid !== process.getuid()))
|
|
44
|
+
return null;
|
|
45
|
+
return owner;
|
|
46
|
+
}
|
|
47
|
+
function ownerHasExited(pid) {
|
|
48
|
+
try {
|
|
49
|
+
process.kill(pid, 0);
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
catch (error) {
|
|
53
|
+
// EPERM and any unfamiliar error mean the process may still be active.
|
|
54
|
+
return error.code === "ESRCH";
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
function removeAbandonedDirectory(directory, dbPath) {
|
|
58
|
+
const before = fs.lstatSync(directory);
|
|
59
|
+
const owner = readOwner(directory, dbPath);
|
|
60
|
+
if (owner === null || !ownerHasExited(owner.pid))
|
|
61
|
+
return;
|
|
62
|
+
if (path.basename(directory) !== owner.directoryName) {
|
|
63
|
+
const claim = path.basename(directory).match(/\.cleanup-([1-9]\d*)-[\da-f-]{36}$/i);
|
|
64
|
+
if (claim === null || !ownerHasExited(Number(claim[1])))
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
const current = fs.lstatSync(directory);
|
|
68
|
+
if (current.dev !== before.dev || current.ino !== before.ino)
|
|
69
|
+
return;
|
|
70
|
+
const claimedPath = path.join(path.dirname(directory), `${owner.directoryName}.cleanup-${process.pid}-${randomUUID()}`);
|
|
71
|
+
// A claim avoids two scavengers recursively removing the same directory.
|
|
72
|
+
fs.renameSync(directory, claimedPath);
|
|
73
|
+
const claimed = fs.lstatSync(claimedPath);
|
|
74
|
+
const claimedOwner = readOwner(claimedPath, dbPath);
|
|
75
|
+
if (claimed.dev !== before.dev || claimed.ino !== before.ino || claimedOwner === null || !ownerHasExited(claimedOwner.pid))
|
|
76
|
+
return;
|
|
77
|
+
try {
|
|
78
|
+
fs.rmSync(claimedPath, { recursive: true, force: true });
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
throw new Error(`Could not remove abandoned advisory sync directory ${claimedPath}: ${ensureSentence(error instanceof Error ? error.message : String(error))} Resolve the filesystem error and remove it manually once no sync is using it.`, { cause: error });
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
/** Never infer ownership from a glob or age: legacy/unverifiable directories are left alone. */
|
|
85
|
+
export function cleanupAbandonedStagingDirectories(dbPath) {
|
|
86
|
+
const warnings = [];
|
|
87
|
+
const parent = path.dirname(dbPath);
|
|
88
|
+
const prefix = `.${path.basename(dbPath)}.tmp-`;
|
|
89
|
+
let entries;
|
|
90
|
+
try {
|
|
91
|
+
entries = fs.readdirSync(parent, { withFileTypes: true });
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
return [`Could not inspect abandoned advisory sync directories in ${parent}: ${ensureSentence(error instanceof Error ? error.message : String(error))} Check directory permissions and remove abandoned staging files manually once no sync is using them.`];
|
|
95
|
+
}
|
|
96
|
+
for (const entry of entries) {
|
|
97
|
+
if (!entry.isDirectory() || entry.isSymbolicLink() || !entry.name.startsWith(prefix))
|
|
98
|
+
continue;
|
|
99
|
+
const directory = path.join(parent, entry.name);
|
|
100
|
+
try {
|
|
101
|
+
removeAbandonedDirectory(directory, dbPath);
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
if (error.code === "ENOENT" || error instanceof SyntaxError)
|
|
105
|
+
continue;
|
|
106
|
+
warnings.push(error instanceof Error && error.message.startsWith("Could not remove abandoned advisory sync directory")
|
|
107
|
+
? error.message
|
|
108
|
+
: `Could not inspect abandoned advisory sync directory ${directory}: ${ensureSentence(error instanceof Error ? error.message : String(error))} Check this directory and remove it manually only when no sync is using it.`);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return warnings;
|
|
112
|
+
}
|