cve-lite-cli 1.40.0 → 1.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +17 -11
  2. package/dist/advisory/local-advisory-source.js +9 -5
  3. package/dist/advisory/local-db.js +91 -13
  4. package/dist/advisory/osv-sync.js +187 -56
  5. package/dist/advisory/staging-directory.js +112 -0
  6. package/dist/cli/advisories-sync.js +16 -2
  7. package/dist/cli/args.js +46 -35
  8. package/dist/cli/config.js +4 -1
  9. package/dist/cli/help.js +4 -3
  10. package/dist/cli/validate.js +23 -1
  11. package/dist/constants.js +11 -25
  12. package/dist/index.js +8 -3
  13. package/dist/output/cyclonedx.js +23 -14
  14. package/dist/output/debug.js +68 -18
  15. package/dist/output/formatters.js +33 -5
  16. package/dist/output/html-reporter.js +55 -5
  17. package/dist/output/multi-folder-html-reporter.js +59 -3
  18. package/dist/output/output-dir.js +193 -7
  19. package/dist/output/printers.js +16 -5
  20. package/dist/output/sarif.js +131 -48
  21. package/dist/output/sbom-dependency-edges.js +16 -10
  22. package/dist/output/sbom-file.js +14 -6
  23. package/dist/output/scan-json.js +21 -3
  24. package/dist/output/spdx.js +61 -23
  25. package/dist/output/spinner.js +2 -1
  26. package/dist/output/write-outputs.js +142 -52
  27. package/dist/overrides/context-builder.js +23 -70
  28. package/dist/parsers/bun-lock.js +1 -2
  29. package/dist/parsers/index.js +22 -1
  30. package/dist/parsers/npm-lock-graph.js +39 -1
  31. package/dist/parsers/package-lock.js +25 -5
  32. package/dist/parsers/pnpm-lock.js +3 -2
  33. package/dist/parsers/utils.js +5 -3
  34. package/dist/parsers/yarn-lock-graph.js +5 -2
  35. package/dist/parsers/yarn-lock.js +1 -2
  36. package/dist/remediation/fix-commands.js +105 -44
  37. package/dist/remediation/npm-transitive-graph.js +30 -72
  38. package/dist/remediation/npm-transitive-resolution.js +24 -77
  39. package/dist/remediation/parent-upgrade-blockers.js +51 -0
  40. package/dist/remediation/parent-upgrade.js +5 -79
  41. package/dist/remediation/registry-parent-ranges.js +21 -0
  42. package/dist/remediation/transitive-chain-resolver.js +3 -3
  43. package/dist/scan/completeness.js +10 -4
  44. package/dist/scan/multi-folder-scan.js +133 -58
  45. package/dist/scan/single-scan.js +41 -25
  46. package/dist/scanner.js +257 -58
  47. package/dist/usage/scanner.js +19 -8
  48. package/dist/utils/array.js +11 -3
  49. package/dist/utils/debug-bundle.js +118 -0
  50. package/dist/utils/debug-project.js +102 -0
  51. package/dist/utils/debug-redaction.js +117 -0
  52. package/dist/utils/file.js +2 -1
  53. package/dist/utils/fix-runner.js +95 -13
  54. package/dist/utils/import-type-queries.js +322 -0
  55. package/dist/utils/network.js +13 -9
  56. package/dist/utils/npm-lockfile.js +12 -0
  57. package/dist/utils/osv-fetch.js +70 -0
  58. package/dist/utils/package-identity.js +43 -0
  59. package/dist/utils/package-manager-source.js +24 -0
  60. package/dist/utils/purl.js +37 -0
  61. package/dist/utils/registry-config.js +1 -16
  62. package/dist/utils/string.js +8 -0
  63. package/dist/utils/validate-url.js +1 -1
  64. package/dist/utils/version.js +30 -7
  65. package/dist/utils/versions/osv-range.js +191 -0
  66. package/dist/utils/versions/pep440.js +246 -0
  67. package/dist/utils/versions/scheme.js +10 -0
  68. package/package.json +3 -2
package/README.md CHANGED
@@ -57,12 +57,12 @@
57
57
  </p>
58
58
 
59
59
  <p>
60
- <sub>Running in CI at
60
+ <sub>Used by
61
61
  <a href="https://github.com/solidjs/solid/blob/main/.github/workflows/cve-lite.yml"><strong>SolidJS</strong></a>,
62
- <a href="https://github.com/bcgov/nr-waste-plus/blob/main/.github/workflows/analysis.yml"><strong>the Government of British Columbia</strong></a> as a required merge gate,
63
- <a href="https://github.com/etalab-ia/parcours-rag/blob/main/package.json">French government digital services (<strong>DINUM</strong>)</a>, and
64
- <a href="https://github.com/fabian-hiller/valibot"><strong>Valibot</strong></a>.<br/>
65
- In production use in France, Canada, Germany, Thailand and China.</sub>
62
+ <a href="https://github.com/open-circle/valibot/blob/main/.github/workflows/security.yml"><strong>Valibot</strong></a>,
63
+ <a href="https://github.com/bcgov/nr-waste-plus/blob/main/.github/workflows/analysis.yml"><strong>the Government of British Columbia</strong></a> as a required merge gate, and
64
+ <a href="https://github.com/refugies-info/karfur/blob/dev/package.json"><strong>the Government of France</strong></a>.<br/>
65
+ In public repositories in France, Canada, Germany, Thailand and China. <a href="https://owasp.org/cve-lite-cli/who-uses">See who uses it</a>.</sub>
66
66
  </p>
67
67
 
68
68
  <br/>
@@ -370,16 +370,22 @@ CVE Lite CLI complements these tools. It is not a replacement for continuous mon
370
370
 
371
371
  ## Community and adoption
372
372
 
373
- CVE Lite CLI is used in production across organizations and independent developers in multiple countries. Confirmed users span government digital services, enterprise engineering platforms, open-source libraries, and security education.
373
+ CVE Lite CLI runs in public repositories across governments, open-source libraries, companies, and independent developers. The [Who uses CVE Lite CLI?](https://owasp.org/cve-lite-cli/who-uses) page links every adopter to the public file that proves it.
374
374
 
375
375
  Notable adopters include:
376
376
 
377
- - **SolidJS** - the framework's creator, Ryan Carniato, added CVE Lite CLI directly to its CI, scanning every change with results reported to GitHub Code Scanning
378
- - **DINUM** (France) - integrated across four French government digital service repositories, including official AI training infrastructure for civil servants
379
- - **DGAFP** (France) - Ministry for Civil Service, used as a pre-commit hook on a public-sector HR chatbot
380
- - **Government of British Columbia** (Canada) - running as a required merge gate alongside Trivy at the Ministry of Citizens' Services
381
- - **Valibot** and **Formisch** - integrated as a GitHub Action and dev script with branch protection
377
+ - **SolidJS** - the framework's creator, Ryan Carniato, added CVE Lite CLI to its CI in his own commit. It scans every push and pull request to the main branch and reports results to GitHub Code Scanning
378
+ - **Valibot** and **Formisch** - security workflows that run the scan and publish results to GitHub Code Scanning, merged by their creator, Fabian Hiller. Valibot has about 14 million npm downloads a week
379
+ - **Government of British Columbia** (Canada) - a required check on every push and pull request in its Waste Plus natural resources application, alongside Trivy and npm audit
380
+ - **Government of France** - pre-push scans that block high and critical vulnerabilities in Réfugiés.info, the government's information portal for refugees, and in IA dans l'État, the state's interministerial AI programme
382
381
 
382
+ ## Teaching and learning
383
+
384
+ Teach dependency security with CVE Lite Classroom: free, open-source, hands-on exercises covering vulnerability remediation, critical review of AI-assisted fixes, and CI security gates. Includes runnable applications, instructor guides, worked solutions, and grading rubrics, with tracks for npm, pnpm, Yarn, and Bun.
385
+
386
+ [Explore the classroom exercises](https://sonukapoor.github.io/cve-lite-classroom/)
387
+
388
+ CVE Lite Classroom is a companion project maintained by the author of CVE Lite CLI, separate from the OWASP project.
383
389
 
384
390
  ## Current limitations
385
391
 
@@ -4,11 +4,15 @@ export class LocalAdvisorySource {
4
4
  this.db = db;
5
5
  }
6
6
  queryBatch(packages, _meta) {
7
- const results = packages.map(pkg => ({
8
- package: pkg.name,
9
- version: pkg.version,
10
- vulnerabilities: this.db.findMatchingVulnerabilityIds(pkg).map(id => ({ id })),
11
- }));
7
+ const results = packages.map(pkg => {
8
+ const match = this.db.matchPackage(pkg);
9
+ return {
10
+ package: pkg.name,
11
+ version: pkg.version,
12
+ vulnerabilities: match.ids.map(id => ({ id })),
13
+ ...(match.unreadable ? { unreadableBoundary: true } : {}),
14
+ };
15
+ });
12
16
  return Promise.resolve(results);
13
17
  }
14
18
  getVuln(id) {
@@ -1,7 +1,9 @@
1
1
  import fs from "node:fs";
2
2
  import path from "node:path";
3
3
  import Database from "better-sqlite3";
4
- import { compareVersions, isPreReleaseVersion, looksLikeVersion, normalizeAdvisoryVersion } from "../utils/version.js";
4
+ import { compareVersions, isPep440Ecosystem, isPreReleaseVersion, looksLikeVersion, normalizeAdvisoryVersion } from "../utils/version.js";
5
+ import { readPep440Range, versionInPep440Window } from "../utils/versions/osv-range.js";
6
+ import { pep440Scheme } from "../utils/versions/pep440.js";
5
7
  const SCHEMA_SQL = `
6
8
  CREATE TABLE IF NOT EXISTS advisories (
7
9
  id TEXT PRIMARY KEY,
@@ -164,17 +166,35 @@ export class LocalAdvisoryDatabase {
164
166
  }
165
167
  }
166
168
  findMatchingVulnerabilityIds(pkg) {
167
- if (!looksLikeVersion(pkg.version)) {
168
- return [];
169
+ return this.matchPackage(pkg).ids;
170
+ }
171
+ /**
172
+ * Ids whose stored window contains `pkg`, plus whether a PyPI boundary that
173
+ * could still contain it could not be read. An unreadable boundary does not
174
+ * abort the lookup and is not reported as a clean miss.
175
+ */
176
+ matchPackage(pkg) {
177
+ if (!looksLikeVersion(pkg.version, pkg.ecosystem)) {
178
+ // Semver returns no matches for a tag or a range. A PyPI version that
179
+ // cannot be read is inconclusive, not unaffected.
180
+ return { ids: [], unreadable: isPep440Ecosystem(pkg.ecosystem) };
169
181
  }
170
182
  const rows = this.findMatchingIdsStatement.all(pkg.ecosystem, pkg.name);
171
183
  const ids = new Set();
184
+ let unreadable = false;
172
185
  for (const row of rows) {
173
- if (versionMatchesRange(pkg.version, row)) {
186
+ if (isPep440Ecosystem(pkg.ecosystem)) {
187
+ const verdict = pep440RowVerdict(pkg.version, row);
188
+ if (verdict === "unreadable")
189
+ unreadable = true;
190
+ else if (verdict === "match")
191
+ ids.add(row.advisory_id);
192
+ }
193
+ else if (versionMatchesRange(pkg.version, row, pkg.ecosystem)) {
174
194
  ids.add(row.advisory_id);
175
195
  }
176
196
  }
177
- return [...ids];
197
+ return { ids: [...ids], unreadable };
178
198
  }
179
199
  }
180
200
  /**
@@ -201,32 +221,61 @@ function releaseCore(version) {
201
221
  * negative for a false positive. No guard is needed for that: a release core
202
222
  * never contains "-", so it can never equal a pre-release `introduced`.
203
223
  */
204
- function versionAtOrAboveIntroduced(version, introduced) {
205
- if (compareVersions(version, introduced) >= 0)
224
+ function versionAtOrAboveIntroduced(version, introduced, ecosystem) {
225
+ // PEP 440 orders dev and pre-releases strictly, below the final release.
226
+ if (isPep440Ecosystem(ecosystem))
227
+ return pep440Scheme.atOrAboveIntroduced(version, introduced);
228
+ if (compareVersions(version, introduced, ecosystem) >= 0)
206
229
  return true;
230
+ // The pre-release relaxation is a semver rule.
207
231
  return isPreReleaseVersion(version) && releaseCore(version) === introduced;
208
232
  }
209
233
  /** OSV range types whose boundaries are versions we can order. GIT uses commit hashes. */
210
234
  const COMPARABLE_RANGE_TYPES = new Set(["ECOSYSTEM", "SEMVER"]);
211
- function versionMatchesRange(version, row) {
235
+ function pep440RowVerdict(version, row) {
236
+ const introducedSentinel = row.introduced === "0";
237
+ const introduced = row.introduced === null || introducedSentinel
238
+ ? row.introduced
239
+ : normalizeAdvisoryVersion(row.introduced, "PyPI");
240
+ if (row.introduced !== null && !introducedSentinel) {
241
+ if (!introduced) {
242
+ const readableFixed = row.fixed === null ? null : normalizeAdvisoryVersion(row.fixed, "PyPI");
243
+ if (readableFixed && compareVersions(version, readableFixed, "PyPI") >= 0)
244
+ return "miss";
245
+ return "unreadable";
246
+ }
247
+ if (!versionAtOrAboveIntroduced(version, introduced, "PyPI"))
248
+ return "miss";
249
+ }
250
+ const fixed = row.fixed === null ? null : normalizeAdvisoryVersion(row.fixed, "PyPI");
251
+ const lastAffected = row.last_affected === null ? null : normalizeAdvisoryVersion(row.last_affected, "PyPI");
252
+ if ((row.fixed !== null && !fixed) || (row.last_affected !== null && !lastAffected))
253
+ return "unreadable";
254
+ return versionInPep440Window(version, {
255
+ introduced: introducedSentinel ? "0" : introduced,
256
+ fixed,
257
+ lastAffected,
258
+ }) ? "match" : "miss";
259
+ }
260
+ function versionMatchesRange(version, row, ecosystem) {
212
261
  const introduced = row.introduced;
213
262
  // Defence in depth behind the range-type filter, which stops these rows being
214
263
  // written in the first place. A database built by an earlier version still
215
264
  // holds them, and a bound that is not a version must never decide the answer
216
265
  // by silently failing to exclude (#1191).
217
- const fixed = normalizeAdvisoryVersion(row.fixed);
218
- const lastAffected = normalizeAdvisoryVersion(row.last_affected);
266
+ const fixed = normalizeAdvisoryVersion(row.fixed, ecosystem);
267
+ const lastAffected = normalizeAdvisoryVersion(row.last_affected, ecosystem);
219
268
  if (row.fixed && !fixed)
220
269
  return false;
221
270
  if (row.last_affected && !lastAffected)
222
271
  return false;
223
- if (introduced && introduced !== "0" && !versionAtOrAboveIntroduced(version, introduced)) {
272
+ if (introduced && introduced !== "0" && !versionAtOrAboveIntroduced(version, introduced, ecosystem)) {
224
273
  return false;
225
274
  }
226
- if (fixed && compareVersions(version, fixed) >= 0) {
275
+ if (fixed && compareVersions(version, fixed, ecosystem) >= 0) {
227
276
  return false;
228
277
  }
229
- if (lastAffected && compareVersions(version, lastAffected) > 0) {
278
+ if (lastAffected && compareVersions(version, lastAffected, ecosystem) > 0) {
230
279
  return false;
231
280
  }
232
281
  return true;
@@ -250,6 +299,10 @@ function deriveAdvisoryPackageRows(vuln) {
250
299
  if (range.type && !COMPARABLE_RANGE_TYPES.has(range.type))
251
300
  continue;
252
301
  const events = range.events ?? [];
302
+ if (isPep440Ecosystem(ecosystem)) {
303
+ rows.push(...pep440PackageRows(vuln.id, ecosystem, packageName, events));
304
+ continue;
305
+ }
253
306
  let introduced = null;
254
307
  for (const event of events) {
255
308
  if (event.introduced) {
@@ -291,3 +344,28 @@ function deriveAdvisoryPackageRows(vuln) {
291
344
  }
292
345
  return rows;
293
346
  }
347
+ function pep440PackageRows(advisoryId, ecosystem, packageName, events) {
348
+ const read = readPep440Range(events);
349
+ const rows = [];
350
+ for (const window of read.windows) {
351
+ rows.push({
352
+ advisory_id: advisoryId,
353
+ ecosystem,
354
+ package_name: packageName,
355
+ introduced: window.introduced,
356
+ fixed: window.fixed,
357
+ last_affected: window.lastAffected,
358
+ });
359
+ }
360
+ for (const boundary of read.unreadable) {
361
+ rows.push({
362
+ advisory_id: advisoryId,
363
+ ecosystem,
364
+ package_name: packageName,
365
+ introduced: boundary.field === "introduced" ? boundary.value : boundary.introduced,
366
+ fixed: boundary.field === "fixed" ? boundary.value : null,
367
+ last_affected: boundary.field === "last_affected" ? boundary.value : null,
368
+ });
369
+ }
370
+ return rows;
371
+ }
@@ -1,11 +1,21 @@
1
1
  import fs from "node:fs";
2
2
  import os from "node:os";
3
3
  import path from "node:path";
4
+ import { randomUUID } from "node:crypto";
4
5
  import { unzipSync } from "fflate";
5
6
  import { LocalAdvisoryDatabase } from "./local-db.js";
6
- import { pluralize } from "../utils/string.js";
7
+ import { ensureSentence, pluralize } from "../utils/string.js";
8
+ import { cleanupAbandonedStagingDirectories, recordStagingOwner, stagingDirectoryPrefix } from "./staging-directory.js";
7
9
  const DEFAULT_OSV_NPM_DUMP_URL = "https://storage.googleapis.com/osv-vulnerabilities/npm/all.zip";
8
10
  export const ADVISORY_DB_STALE_AFTER_MS = 7 * 24 * 60 * 60 * 1000;
11
+ export class AdvisorySyncError extends Error {
12
+ databaseState;
13
+ constructor(message, databaseState, cause) {
14
+ super(message, { cause });
15
+ this.databaseState = databaseState;
16
+ this.name = "AdvisorySyncError";
17
+ }
18
+ }
9
19
  export function getDefaultAdvisoryDbPath(outputPath) {
10
20
  if (outputPath) {
11
21
  return path.resolve(outputPath);
@@ -16,21 +26,121 @@ export function getDefaultAdvisoryDbPath(outputPath) {
16
26
  }
17
27
  return path.join(os.homedir(), ".cache", "cve-lite", "advisories.db");
18
28
  }
19
- export async function syncOsvAdvisories(options = {}) {
20
- const fetchImpl = options.fetchImpl ?? fetch;
21
- const onProgress = options.onProgress;
22
- const sourceUrl = options.sourceUrl ?? DEFAULT_OSV_NPM_DUMP_URL;
23
- const dbPath = getDefaultAdvisoryDbPath(options.outputPath);
29
+ /**
30
+ * Replaces the advisory database after a successful sync. Node's rename replaces
31
+ * existing files on Windows too, but an open or locked target can cause EPERM.
32
+ * If replacement is refused, move the old database aside only after the new one
33
+ * is complete and restore it if the second rename fails.
34
+ */
35
+ function replaceAdvisoryDatabase(tempPath, dbPath) {
36
+ try {
37
+ fs.renameSync(tempPath, dbPath);
38
+ return;
39
+ }
40
+ catch (error) {
41
+ const code = error.code;
42
+ if (code !== "EEXIST" && code !== "EPERM")
43
+ throw error;
44
+ }
45
+ const backupPath = `${dbPath}.backup-${process.pid}-${randomUUID()}`;
46
+ let movedExisting = false;
47
+ try {
48
+ fs.renameSync(dbPath, backupPath);
49
+ movedExisting = true;
50
+ fs.renameSync(tempPath, dbPath);
51
+ }
52
+ catch (error) {
53
+ if (!movedExisting)
54
+ throw error;
55
+ restoreAdvisoryDatabase(backupPath, dbPath, error);
56
+ }
57
+ return cleanupBackup(backupPath, "The refreshed database was installed.");
58
+ }
59
+ function cleanupBackup(backupPath, databaseMessage) {
60
+ try {
61
+ fs.rmSync(backupPath, { force: true });
62
+ }
63
+ catch (error) {
64
+ const message = error instanceof Error ? error.message : String(error);
65
+ return `Could not remove advisory database backup ${backupPath}: ${ensureSentence(message)} `
66
+ + `${databaseMessage} Resolve the filesystem error and remove ${backupPath} manually.`;
67
+ }
68
+ }
69
+ function restoreAdvisoryDatabase(backupPath, dbPath, installError) {
70
+ const originalMessage = installError instanceof Error ? installError.message : String(installError);
71
+ try {
72
+ // Hardlink creation is atomic and refuses an existing target. An exists-check
73
+ // followed by rename can overwrite a database installed by another sync.
74
+ fs.linkSync(backupPath, dbPath);
75
+ }
76
+ catch (restoreError) {
77
+ const restoreMessage = restoreError instanceof Error ? restoreError.message : String(restoreError);
78
+ throw new AdvisorySyncError(`Failed to install refreshed advisory database: ${ensureSentence(originalMessage)} `
79
+ + `The previous database could not be restored and remains at ${backupPath}: ${ensureSentence(restoreMessage)} `
80
+ + `No existing database at ${dbPath} was overwritten. Close any program using these files and inspect both databases. `
81
+ + `After resolving the filesystem error, rename ${backupPath} to ${dbPath} only if the target is absent; do not overwrite a newer database.`, "recovery-required", restoreError);
82
+ }
83
+ const warning = cleanupBackup(backupPath, "The previous database was restored.");
84
+ if (warning)
85
+ throw new AdvisorySyncError(`${ensureSentence(originalMessage)} ${warning}`, "unchanged", installError);
86
+ throw installError;
87
+ }
88
+ function rejectDirectoryOutput(dbPath) {
89
+ try {
90
+ if (!fs.lstatSync(dbPath).isDirectory())
91
+ return;
92
+ }
93
+ catch (error) {
94
+ if (error.code === "ENOENT")
95
+ return;
96
+ throw error;
97
+ }
98
+ throw new Error(`Advisory database output ${dbPath} is a directory. Choose a database file path, for example ${path.join(dbPath, "advisories.db")}.`);
99
+ }
100
+ async function ingestAdvisories(db, advisoryEntries, sourceUrl, onProgress) {
101
+ const parsedVulns = [];
102
+ let advisoryCount = 0;
103
+ const progressInterval = 250;
104
+ for (const [, bytes] of advisoryEntries) {
105
+ const text = Buffer.from(bytes).toString("utf8");
106
+ const vuln = JSON.parse(text);
107
+ if (!vuln.id)
108
+ continue;
109
+ if (!Array.isArray(vuln.affected) || vuln.affected.length === 0)
110
+ continue;
111
+ // OSV's /v1/querybatch endpoint excludes withdrawn advisories at query
112
+ // time; mirror that here so offline scans don't surface false positives
113
+ // from advisories that have been retracted.
114
+ if (vuln.withdrawn)
115
+ continue;
116
+ parsedVulns.push(vuln);
117
+ advisoryCount += 1;
118
+ if (advisoryCount === 1 ||
119
+ advisoryCount % progressInterval === 0 ||
120
+ advisoryCount === advisoryEntries.length) {
121
+ onProgress?.({
122
+ phase: "ingest",
123
+ processedAdvisories: advisoryCount,
124
+ totalAdvisories: advisoryEntries.length,
125
+ message: `Processing advisory records: ${advisoryCount} / ${advisoryEntries.length}`,
126
+ });
127
+ await yieldToEventLoop();
128
+ }
129
+ }
130
+ db.bulkUpsertVulnerabilities(parsedVulns);
131
+ db.setMetadata({
132
+ lastSyncAt: new Date().toISOString(),
133
+ sourceUrl,
134
+ });
135
+ return advisoryCount;
136
+ }
137
+ async function downloadAdvisoryArchive(sourceUrl, dbPath, fetchImpl, onProgress) {
24
138
  onProgress?.({
25
139
  phase: "init",
26
140
  sourceUrl,
27
141
  dbPath,
28
142
  message: `Sync initiated. Downloading advisory dump from ${sourceUrl}`,
29
143
  });
30
- fs.mkdirSync(path.dirname(dbPath), { recursive: true });
31
- if (fs.existsSync(dbPath)) {
32
- fs.rmSync(dbPath, { force: true });
33
- }
34
144
  const response = await fetchImpl(sourceUrl);
35
145
  if (!response.ok) {
36
146
  throw new Error(`OSV dump download failed: ${response.status} ${response.statusText}`);
@@ -56,6 +166,9 @@ export async function syncOsvAdvisories(options = {}) {
56
166
  : `Download complete: ${formatBytes(zippedBytes.byteLength)}`,
57
167
  });
58
168
  await yieldToEventLoop();
169
+ return zippedBytes;
170
+ }
171
+ async function buildAdvisoryDatabase(zippedBytes, tempDbPath, sourceUrl, onProgress) {
59
172
  const archiveEntries = unzipSync(zippedBytes);
60
173
  const advisoryEntries = Object.entries(archiveEntries).filter(([entryName]) => entryName.endsWith(".json"));
61
174
  onProgress?.({
@@ -65,64 +178,82 @@ export async function syncOsvAdvisories(options = {}) {
65
178
  message: `Archive loaded. Processing ${advisoryEntries.length} advisory ${pluralize(advisoryEntries.length, "record")}`,
66
179
  });
67
180
  await yieldToEventLoop();
68
- const db = new LocalAdvisoryDatabase(dbPath);
181
+ const db = new LocalAdvisoryDatabase(tempDbPath);
182
+ let advisoryCount;
69
183
  try {
70
- const parsedVulns = [];
71
- let advisoryCount = 0;
72
- const progressInterval = 250;
73
- for (const [, bytes] of advisoryEntries) {
74
- const text = Buffer.from(bytes).toString("utf8");
75
- const vuln = JSON.parse(text);
76
- if (!vuln.id) {
77
- continue;
78
- }
79
- // OSV's /v1/querybatch endpoint excludes withdrawn advisories at query
80
- // time; mirror that here so offline scans don't surface false positives
81
- // from advisories that have been retracted.
82
- if (vuln.withdrawn) {
83
- continue;
84
- }
85
- parsedVulns.push(vuln);
86
- advisoryCount += 1;
87
- if (advisoryCount === 1 ||
88
- advisoryCount % progressInterval === 0 ||
89
- advisoryCount === advisoryEntries.length) {
90
- onProgress?.({
91
- phase: "ingest",
92
- processedAdvisories: advisoryCount,
93
- totalAdvisories: advisoryEntries.length,
94
- message: `Processing advisory records: ${advisoryCount} / ${advisoryEntries.length}`,
95
- });
96
- await yieldToEventLoop();
97
- }
98
- }
99
- db.bulkUpsertVulnerabilities(parsedVulns);
100
- db.setMetadata({
101
- lastSyncAt: new Date().toISOString(),
102
- sourceUrl,
103
- });
184
+ advisoryCount = await ingestAdvisories(db, advisoryEntries, sourceUrl, onProgress);
185
+ }
186
+ finally {
187
+ db.close();
188
+ }
189
+ if (advisoryCount === 0) {
190
+ throw new Error(`OSV dump from ${sourceUrl} contained no usable advisory records. `
191
+ + `Check the source URL, then run "cve-lite advisories sync" again.`);
192
+ }
193
+ return advisoryCount;
194
+ }
195
+ /** Removes staged files, returning a warning for the caller to report. */
196
+ function cleanupTempDirectory(tempDir) {
197
+ try {
198
+ fs.rmSync(tempDir, { recursive: true, force: true });
199
+ }
200
+ catch (error) {
201
+ const message = error instanceof Error ? error.message : String(error);
202
+ return `Could not remove temporary advisory sync directory ${tempDir}: ${ensureSentence(message)} `
203
+ + `Resolve the filesystem error and remove ${tempDir} manually.`;
204
+ }
205
+ }
206
+ function buildSyncError(refreshError, databaseInstalled, cleanupWarning) {
207
+ const message = refreshError instanceof Error ? refreshError.message : String(refreshError);
208
+ const databaseState = refreshError instanceof AdvisorySyncError
209
+ ? refreshError.databaseState
210
+ : databaseInstalled ? "replaced" : "unchanged";
211
+ return new AdvisorySyncError(cleanupWarning ? `${ensureSentence(message)} ${cleanupWarning}` : message, databaseState, refreshError);
212
+ }
213
+ export async function syncOsvAdvisories(options = {}) {
214
+ const onProgress = options.onProgress;
215
+ const sourceUrl = options.sourceUrl ?? DEFAULT_OSV_NPM_DUMP_URL;
216
+ const dbPath = getDefaultAdvisoryDbPath(options.outputPath);
217
+ rejectDirectoryOutput(dbPath);
218
+ let tempDir;
219
+ let databaseInstalled = false;
220
+ let refreshError;
221
+ let cleanupWarning;
222
+ let replacementWarning;
223
+ let syncResult;
224
+ const startupWarnings = [];
225
+ try {
226
+ fs.mkdirSync(path.dirname(dbPath), { recursive: true });
227
+ startupWarnings.push(...cleanupAbandonedStagingDirectories(dbPath));
228
+ tempDir = fs.mkdtempSync(stagingDirectoryPrefix(dbPath));
229
+ recordStagingOwner(tempDir, dbPath);
230
+ const tempDbPath = path.join(tempDir, path.basename(dbPath));
231
+ const zippedBytes = await downloadAdvisoryArchive(sourceUrl, dbPath, options.fetchImpl ?? fetch, onProgress);
232
+ const advisoryCount = await buildAdvisoryDatabase(zippedBytes, tempDbPath, sourceUrl, onProgress);
233
+ replacementWarning = replaceAdvisoryDatabase(tempDbPath, dbPath);
234
+ databaseInstalled = true;
104
235
  onProgress?.({
105
236
  phase: "complete",
106
237
  advisoryCount,
107
238
  dbPath,
108
239
  message: `Sync complete. Stored ${advisoryCount} advisory ${pluralize(advisoryCount, "record")} in ${dbPath}`,
109
240
  });
110
- return { advisoryCount, dbPath, sourceUrl };
241
+ syncResult = { advisoryCount, dbPath, sourceUrl };
111
242
  }
112
243
  catch (error) {
113
- if (fs.existsSync(dbPath)) {
114
- fs.rmSync(dbPath, { force: true });
115
- }
116
- throw error;
244
+ refreshError = error;
117
245
  }
118
246
  finally {
119
- try {
120
- db.close();
121
- }
122
- catch {
123
- // ignore close errors during sync cleanup
124
- }
247
+ if (tempDir !== undefined)
248
+ cleanupWarning = cleanupTempDirectory(tempDir);
249
+ }
250
+ const warnings = [...startupWarnings, replacementWarning, cleanupWarning].filter((warning) => warning !== undefined);
251
+ if (syncResult === undefined) {
252
+ throw buildSyncError(refreshError, databaseInstalled, warnings.join(" ") || undefined);
125
253
  }
254
+ if (warnings.length > 0)
255
+ syncResult.warnings = warnings;
256
+ return syncResult;
126
257
  }
127
258
  async function readResponseBytes(response, onChunk) {
128
259
  if (!response.body) {
@@ -0,0 +1,112 @@
1
+ import fs from "node:fs";
2
+ import os from "node:os";
3
+ import path from "node:path";
4
+ import { randomUUID } from "node:crypto";
5
+ import { ensureSentence } from "../utils/string.js";
6
+ const OWNER_FILE = "owner.json";
7
+ export function stagingDirectoryPrefix(dbPath) {
8
+ return path.join(path.dirname(dbPath), `.${path.basename(dbPath)}.tmp-${process.pid}-`);
9
+ }
10
+ /** Called after the caller records the directory, so finally also covers write failures. */
11
+ export function recordStagingOwner(tempDir, dbPath) {
12
+ const owner = {
13
+ version: 1,
14
+ pid: process.pid,
15
+ hostname: os.hostname(),
16
+ dbPath,
17
+ directoryName: path.basename(tempDir),
18
+ };
19
+ fs.writeFileSync(path.join(tempDir, OWNER_FILE), JSON.stringify(owner), { flag: "wx", mode: 0o600 });
20
+ }
21
+ function readOwner(directory, dbPath) {
22
+ const entry = fs.lstatSync(directory);
23
+ if (!entry.isDirectory() || entry.isSymbolicLink())
24
+ return null;
25
+ const metadata = fs.lstatSync(path.join(directory, OWNER_FILE));
26
+ if (!metadata.isFile() || metadata.isSymbolicLink())
27
+ return null;
28
+ const parsed = JSON.parse(fs.readFileSync(path.join(directory, OWNER_FILE), "utf8"));
29
+ if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed))
30
+ return null;
31
+ const owner = parsed;
32
+ if (owner.version !== 1 || !Number.isSafeInteger(owner.pid) || owner.pid <= 0 || owner.pid > 2147483647)
33
+ return null;
34
+ if (owner.hostname !== os.hostname() || owner.dbPath !== dbPath || typeof owner.directoryName !== "string")
35
+ return null;
36
+ const prefix = `.${path.basename(dbPath)}.tmp-${owner.pid}-`;
37
+ if (!owner.directoryName.startsWith(prefix) || path.basename(owner.directoryName) !== owner.directoryName)
38
+ return null;
39
+ const name = path.basename(directory);
40
+ if (name !== owner.directoryName && !name.startsWith(`${owner.directoryName}.cleanup-`))
41
+ return null;
42
+ // On POSIX, do not sweep another user's staging directory, even with matching metadata.
43
+ if (process.getuid && (entry.uid !== process.getuid() || metadata.uid !== process.getuid()))
44
+ return null;
45
+ return owner;
46
+ }
47
+ function ownerHasExited(pid) {
48
+ try {
49
+ process.kill(pid, 0);
50
+ return false;
51
+ }
52
+ catch (error) {
53
+ // EPERM and any unfamiliar error mean the process may still be active.
54
+ return error.code === "ESRCH";
55
+ }
56
+ }
57
+ function removeAbandonedDirectory(directory, dbPath) {
58
+ const before = fs.lstatSync(directory);
59
+ const owner = readOwner(directory, dbPath);
60
+ if (owner === null || !ownerHasExited(owner.pid))
61
+ return;
62
+ if (path.basename(directory) !== owner.directoryName) {
63
+ const claim = path.basename(directory).match(/\.cleanup-([1-9]\d*)-[\da-f-]{36}$/i);
64
+ if (claim === null || !ownerHasExited(Number(claim[1])))
65
+ return;
66
+ }
67
+ const current = fs.lstatSync(directory);
68
+ if (current.dev !== before.dev || current.ino !== before.ino)
69
+ return;
70
+ const claimedPath = path.join(path.dirname(directory), `${owner.directoryName}.cleanup-${process.pid}-${randomUUID()}`);
71
+ // A claim avoids two scavengers recursively removing the same directory.
72
+ fs.renameSync(directory, claimedPath);
73
+ const claimed = fs.lstatSync(claimedPath);
74
+ const claimedOwner = readOwner(claimedPath, dbPath);
75
+ if (claimed.dev !== before.dev || claimed.ino !== before.ino || claimedOwner === null || !ownerHasExited(claimedOwner.pid))
76
+ return;
77
+ try {
78
+ fs.rmSync(claimedPath, { recursive: true, force: true });
79
+ }
80
+ catch (error) {
81
+ throw new Error(`Could not remove abandoned advisory sync directory ${claimedPath}: ${ensureSentence(error instanceof Error ? error.message : String(error))} Resolve the filesystem error and remove it manually once no sync is using it.`, { cause: error });
82
+ }
83
+ }
84
+ /** Never infer ownership from a glob or age: legacy/unverifiable directories are left alone. */
85
+ export function cleanupAbandonedStagingDirectories(dbPath) {
86
+ const warnings = [];
87
+ const parent = path.dirname(dbPath);
88
+ const prefix = `.${path.basename(dbPath)}.tmp-`;
89
+ let entries;
90
+ try {
91
+ entries = fs.readdirSync(parent, { withFileTypes: true });
92
+ }
93
+ catch (error) {
94
+ return [`Could not inspect abandoned advisory sync directories in ${parent}: ${ensureSentence(error instanceof Error ? error.message : String(error))} Check directory permissions and remove abandoned staging files manually once no sync is using them.`];
95
+ }
96
+ for (const entry of entries) {
97
+ if (!entry.isDirectory() || entry.isSymbolicLink() || !entry.name.startsWith(prefix))
98
+ continue;
99
+ const directory = path.join(parent, entry.name);
100
+ try {
101
+ removeAbandonedDirectory(directory, dbPath);
102
+ }
103
+ catch (error) {
104
+ if (error.code === "ENOENT" || error instanceof SyntaxError)
105
+ continue;
106
+ warnings.push(error instanceof Error && error.message.startsWith("Could not remove abandoned advisory sync directory")
107
+ ? error.message
108
+ : `Could not inspect abandoned advisory sync directory ${directory}: ${ensureSentence(error instanceof Error ? error.message : String(error))} Check this directory and remove it manually only when no sync is using it.`);
109
+ }
110
+ }
111
+ return warnings;
112
+ }