cve-lite-cli 1.41.4 → 1.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +22 -19
  2. package/dist/advisory/detail-resolution.js +62 -0
  3. package/dist/advisory/local-db.js +90 -15
  4. package/dist/advisory/osv-advisory-source.js +113 -62
  5. package/dist/advisory/osv-sync.js +18 -16
  6. package/dist/advisory/sqlite.js +50 -0
  7. package/dist/audit-log/ndjson-writer.js +6 -3
  8. package/dist/audit-log/open-error.js +22 -0
  9. package/dist/cli/args.js +145 -87
  10. package/dist/cli/flags.js +2 -0
  11. package/dist/cli/help.js +6 -2
  12. package/dist/cli/overrides.js +5 -2
  13. package/dist/cli/validate.js +45 -5
  14. package/dist/constants.js +10 -0
  15. package/dist/index.js +15 -6
  16. package/dist/licenses/license-check.js +17 -4
  17. package/dist/osv/cache.js +7 -13
  18. package/dist/output/cyclonedx.js +57 -18
  19. package/dist/output/finding-display.js +8 -1
  20. package/dist/output/formatters.js +52 -11
  21. package/dist/output/html-reporter.js +40 -8
  22. package/dist/output/override-findings-terminal.js +21 -13
  23. package/dist/output/printers.js +185 -76
  24. package/dist/output/sarif.js +182 -16
  25. package/dist/output/spdx.js +15 -1
  26. package/dist/output/write-outputs.js +29 -20
  27. package/dist/overrides/context-builder.js +1 -7
  28. package/dist/overrides/detectors/oa005-nested-ineffective.js +6 -1
  29. package/dist/overrides/detectors/oa008-materialized.js +33 -10
  30. package/dist/overrides/detectors/oa009-stale-floor.js +2 -2
  31. package/dist/overrides/detectors/oa010-vulnerable-floor.js +19 -13
  32. package/dist/overrides/detectors/phantom-utils.js +2 -10
  33. package/dist/overrides/fixer.js +13 -2
  34. package/dist/overrides/index.js +1 -0
  35. package/dist/overrides/types.js +15 -1
  36. package/dist/parsers/bun-lock.js +6 -0
  37. package/dist/parsers/index.js +177 -40
  38. package/dist/parsers/multi-package.js +4 -0
  39. package/dist/parsers/package-lock.js +6 -2
  40. package/dist/parsers/pnpm-lock.js +4 -7
  41. package/dist/parsers/requirements-txt.js +185 -0
  42. package/dist/parsers/utils.js +11 -6
  43. package/dist/parsers/yarn-lock.js +1 -5
  44. package/dist/remediation/fix-commands.js +60 -36
  45. package/dist/remediation/parent-update-child-note.js +3 -2
  46. package/dist/scan/completeness.js +57 -4
  47. package/dist/scan/multi-folder-scan.js +27 -13
  48. package/dist/scan/single-scan.js +118 -37
  49. package/dist/scanner.js +59 -36
  50. package/dist/types.js +1 -1
  51. package/dist/usage/scanner.js +345 -45
  52. package/dist/utils/array.js +5 -0
  53. package/dist/utils/baseline.js +8 -2
  54. package/dist/utils/chalk.js +1 -3
  55. package/dist/utils/create-pr.js +2 -2
  56. package/dist/utils/debug-bundle.js +37 -22
  57. package/dist/utils/debug-project.js +6 -5
  58. package/dist/utils/debug-redaction.js +1 -1
  59. package/dist/utils/effective-options.js +5 -1
  60. package/dist/utils/finding.js +4 -0
  61. package/dist/utils/fix-runner.js +110 -36
  62. package/dist/utils/import-type-queries.js +5 -1
  63. package/dist/utils/install-command.js +6 -0
  64. package/dist/utils/malicious.js +11 -0
  65. package/dist/utils/package-identity.js +12 -2
  66. package/dist/utils/path-coverage.js +1 -5
  67. package/dist/utils/sanitize.js +59 -0
  68. package/dist/utils/sbom-format.js +17 -8
  69. package/dist/utils/severity.js +61 -7
  70. package/dist/utils/url.js +175 -0
  71. package/dist/utils/version.js +44 -0
  72. package/dist/utils/vuln.js +11 -14
  73. package/package.json +2 -4
package/README.md CHANGED
@@ -34,7 +34,7 @@
34
34
  <tr>
35
35
  <td align="center" width="33%" valign="top"><strong>OWASP Lab Project</strong><br/><sub>Peer-reviewed by the org behind the OWASP Top 10:<br/>the security standard followed by millions of developers</sub></td>
36
36
  <td align="center" width="33%" valign="top"><strong>Remediation-first</strong><br/><sub>Validated fix commands + parent-aware<br/>transitive guidance: not just CVE IDs</sub></td>
37
- <td align="center" width="33%" valign="top"><strong>Runs locally</strong><br/><sub>Nothing leaves your machine: not your<br/>code, not your dependency tree</sub></td>
37
+ <td align="center" width="33%" valign="top"><strong>Runs locally</strong><br/><sub>Package names and versions are checked online, or fully offline with <code>--offline</code>.</sub></td>
38
38
  </tr>
39
39
  </table>
40
40
 
@@ -68,7 +68,7 @@
68
68
  <br/>
69
69
 
70
70
  <p>
71
- ⚡ <strong>One command. No account, no API key, nothing leaves your machine.</strong>
71
+ ⚡ <strong>One command. No account, no API key, no source code uploaded.</strong>
72
72
  &nbsp;<code>npx cve-lite-cli .</code>&nbsp;
73
73
  <a href="#quick-start"><strong>→ Quick Start</strong></a>
74
74
  </p>
@@ -121,7 +121,9 @@ npx cve-lite-cli /path/to/project
121
121
 
122
122
  No account. No configuration. No source code leaves your machine.
123
123
 
124
- > **Note:** CVE Lite CLI includes [`better-sqlite3`](https://github.com/WiseLibs/better-sqlite3), a native SQLite binding used for the [local advisory database](https://owasp.org/cve-lite-cli/docs/offline-advisory-db). During installation, npm may download a prebuilt binary or compile one from source. This can produce extra console output: it is expected and is not CVE Lite CLI itself running an install script. See [Dependency footprint](#dependency-footprint) for details.
124
+ CVE Lite CLI never uploads source code or lockfiles. In online mode, it sends package names and versions to advisory services, including OSV and the public npm registry advisory API. OSV and npm advisory APIs are not the only destinations receiving data during online scans: fix validation looks up package names (not versions) on the configured package registry, which is public npm by default, and EPSS enrichment sends the CVE IDs of findings to FIRST's API. Configuring an internal OSV endpoint only redirects OSV requests; npm advisory requests still use the public npm registry. Use `--offline` to scan against a local advisory database when dependency metadata must not leave your machine.
125
+
126
+ > Requires Node.js 22.13.0 or later on the 22.x line, or 23.4.0 and later. Installation works with npm scripts disabled.
125
127
 
126
128
  ## Usage
127
129
 
@@ -138,6 +140,7 @@ cve-lite /path/to/project --report # interactive HTML dashboard
138
140
  cve-lite /path/to/project --check-overrides # audit override hygiene alongside the CVE scan
139
141
  cve-lite /path/to/project --check-maintenance # flag CVE-blocking version drag and npm-deprecated direct deps
140
142
  cve-lite /path/to/project --check-licenses # detect copyleft and unknown licenses alongside the CVE scan
143
+ cve-lite /path/to/project --python # scan pinned Python requirements.txt (experimental)
141
144
  cve-lite advisories sync # sync advisory DB for offline use
142
145
  cve-lite advisories init # create an empty advisory DB to populate yourself
143
146
  cve-lite /path/to/project --offline # scan with no runtime API calls
@@ -322,6 +325,19 @@ jobs:
322
325
 
323
326
  [Full Action documentation ->](https://owasp.org/cve-lite-cli/docs/github-action) - for scan-only CI gating see the [CI and Workflow Integration guide](https://owasp.org/cve-lite-cli/docs/workflow-integration).
324
327
 
328
+ ## GitLab CI/CD
329
+
330
+ Add the scan job to `.gitlab-ci.yml`. The URL references the `v1.42.0` release tag of the template.
331
+
332
+ ```yaml
333
+ include:
334
+ - remote: 'https://raw.githubusercontent.com/OWASP/cve-lite-cli/v1.42.0/templates/cve-lite.yml'
335
+ inputs:
336
+ fail-on: high
337
+ ```
338
+
339
+ Set `path` to select one project when the repository has multiple lockfiles. Omit `inputs` for an informational scan of the current directory. Both modes publish the JSON report and CycloneDX SBOM as job artifacts. [GitLab CI inputs and offline setup](https://owasp.org/cve-lite-cli/docs/gitlab-ci).
340
+
325
341
  ## Real-world validation
326
342
 
327
343
  CVE Lite CLI has been evaluated against real open-source projects to verify that it surfaces meaningful issues - including non-obvious transitive vulnerabilities and complex upgrade paths - not just low-signal advisory matches.
@@ -399,26 +415,13 @@ CVE Lite Classroom is a companion project maintained by the author of CVE Lite C
399
415
 
400
416
  ## Dependency footprint
401
417
 
402
- **Runtime:** `yaml` · `yarn-lockfile` · `better-sqlite3` · `fflate` · `semver`
418
+ **Runtime:** `yaml` · `yarn-lockfile` · `fflate` · `semver`
403
419
 
404
420
  **Dev only:** `@types/node` · `tsx` · `typescript`
405
421
 
406
422
  This is intentional. Because CVE Lite CLI is a security-oriented tool, runtime dependencies are kept minimal and reviewable.
407
423
 
408
- ### Native dependency: `better-sqlite3`
409
-
410
- `better-sqlite3` is a native Node.js addon that provides the SQLite binding for CVE Lite CLI's local advisory database. This database powers offline scanning (`--offline`, `--offline-db`), advisory sync (`cve-lite advisories sync`), and air-gapped workflows (`cve-lite advisories init`).
411
-
412
- Because it includes compiled C/C++ code, npm must run an install script when installing `better-sqlite3`. During installation:
413
-
414
- - **Prebuilt binary available:** `prebuild-install` downloads a prebuilt binary for your platform: fast, though npm may still print a deprecation warning for `prebuild-install` itself ([#519](https://github.com/OWASP/cve-lite-cli/issues/519)).
415
- - **No prebuilt available:** the package falls back to compiling from source using `node-gyp`, which requires a C/C++ toolchain and produces more console output.
416
-
417
- This install step belongs to `better-sqlite3`, not to CVE Lite CLI: the project itself does not define any `preinstall`, `install`, or `postinstall` hooks.
418
-
419
- **npm install-script permissions:** Depending on your npm version and configuration, npm may prompt you to approve the install script before it runs. This is an npm security feature. Online scanning works even when the install script is blocked: the native binding is loaded lazily and only used by `--offline`, `--offline-db`, `cve-lite advisories sync`, and `cve-lite advisories init`. If you use any of those features, the `better-sqlite3` install script must be allowed to complete.
420
-
421
- **npx behavior:** When using `npx cve-lite-cli`, a quiet run does not necessarily mean the install script was skipped. If npm's cache already contains the required prebuilt binary, installation is silent. On a cold cache, the same native install step runs.
424
+ The [local advisory database](https://owasp.org/cve-lite-cli/docs/offline-advisory-db) uses the SQLite driver bundled with Node.js. Existing database files continue to work.
422
425
 
423
426
  ## More documentation
424
427
 
@@ -446,7 +449,7 @@ For governance details - roles, decision-making, and dispute resolution - see [C
446
449
 
447
450
  Feedback on output clarity, remediation guidance, ecosystem coverage, and CI usage is especially valuable.
448
451
 
449
- See [CONTRIBUTING.md](https://github.com/OWASP/cve-lite-cli/blob/main/src/docs/CONTRIBUTING.md) to get started.
452
+ See [CONTRIBUTING.md](https://github.com/OWASP/cve-lite-cli/blob/main/src/docs/CONTRIBUTING.md) to get started. Every pull request must close an issue assigned to you, or one you opened that nobody has claimed.
450
453
 
451
454
  ## Add a badge to your project
452
455
 
@@ -0,0 +1,62 @@
1
+ // Pauses between attempts at one advisory detail lookup, so a lookup is tried
2
+ // once plus once per entry here. Short on purpose: lookups run ten at a time,
3
+ // and the worst case adds about a second per failing advisory (#1443).
4
+ export const OSV_DETAIL_RETRY_DELAYS_MS = [250, 750];
5
+ // The HTTP status of a failed lookup, as the OSV source attaches it to the
6
+ // error. The message is never read: a network error or a proxy's reason text
7
+ // can name a status the server did not send, and a 404 read from text would
8
+ // replace a cached record with a negative one.
9
+ function httpStatusOf(error) {
10
+ const attached = error.status;
11
+ return typeof attached === "number" ? attached : null;
12
+ }
13
+ // A 404 or 410 means OSV has no such record, which no retry will change.
14
+ // Everything else, network errors, rate limits and server errors included,
15
+ // may succeed on another attempt.
16
+ export function isTransientDetailFailure(error) {
17
+ if (!(error instanceof Error))
18
+ return true;
19
+ const status = httpStatusOf(error);
20
+ return status !== 404 && status !== 410;
21
+ }
22
+ // A client error other than a timeout or rate limit, such as a 400 or a 403
23
+ // from a proxy, gets the same answer however often it is asked. It is not
24
+ // retried, but it is not evidence the record is missing either, so it stays
25
+ // a failed lookup.
26
+ export function isRetryableDetailFailure(error) {
27
+ if (!(error instanceof Error))
28
+ return true;
29
+ const status = httpStatusOf(error);
30
+ if (status === null || status < 400 || status >= 500)
31
+ return true;
32
+ return status === 408 || status === 429;
33
+ }
34
+ const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
35
+ // Fetches one advisory's details, retrying transient failures. When every
36
+ // attempt fails, a cached copy older than the TTL is used if one exists,
37
+ // because it carries the advisory's real id and aliases and keeps the finding
38
+ // intact; the caller discloses that it is cached data. A negative cache entry
39
+ // is never used as a fallback: it would turn a failed lookup into a
40
+ // confirmed-missing one on stale evidence.
41
+ export async function resolveAdvisoryDetail(params) {
42
+ const sleep = params.sleep ?? defaultSleep;
43
+ let lastError;
44
+ for (let attempt = 0; attempt <= OSV_DETAIL_RETRY_DELAYS_MS.length; attempt++) {
45
+ if (attempt > 0)
46
+ await sleep(OSV_DETAIL_RETRY_DELAYS_MS[attempt - 1]);
47
+ try {
48
+ return { kind: "fresh", vuln: await params.fetchDetail(params.id) };
49
+ }
50
+ catch (error) {
51
+ if (!isTransientDetailFailure(error))
52
+ return { kind: "confirmed-missing" };
53
+ lastError = error;
54
+ if (!isRetryableDetailFailure(error))
55
+ break;
56
+ }
57
+ }
58
+ if (params.staleEntry?.vuln) {
59
+ return { kind: "stale", vuln: params.staleEntry.vuln, cachedAt: params.staleEntry.cachedAt };
60
+ }
61
+ return { kind: "transient", error: lastError };
62
+ }
@@ -1,9 +1,10 @@
1
1
  import fs from "node:fs";
2
2
  import path from "node:path";
3
- import Database from "better-sqlite3";
3
+ import { openDatabase, inTransaction } from "./sqlite.js";
4
4
  import { compareVersions, isPep440Ecosystem, isPreReleaseVersion, looksLikeVersion, normalizeAdvisoryVersion } from "../utils/version.js";
5
5
  import { readPep440Range, versionInPep440Window } from "../utils/versions/osv-range.js";
6
6
  import { pep440Scheme } from "../utils/versions/pep440.js";
7
+ import { quoteShellPath } from "../utils/shell-quote.js";
7
8
  const SCHEMA_SQL = `
8
9
  CREATE TABLE IF NOT EXISTS advisories (
9
10
  id TEXT PRIMARY KEY,
@@ -37,6 +38,7 @@ const SCHEMA_SQL = `
37
38
  `;
38
39
  export class LocalAdvisoryDatabase {
39
40
  dbPath;
41
+ static warnedTemporaryIndexes = new Set();
40
42
  db;
41
43
  upsertAdvisoryStatement;
42
44
  deleteRangesStatement;
@@ -45,14 +47,18 @@ export class LocalAdvisoryDatabase {
45
47
  getMetadataStatement;
46
48
  getVulnerabilityStatement;
47
49
  findMatchingIdsStatement;
50
+ deleteVersionsStatement;
51
+ insertVersionStatement;
52
+ findExactIdsStatement;
48
53
  constructor(dbPath, options) {
49
54
  this.dbPath = dbPath;
50
55
  const readonly = !!options?.readonly;
51
56
  if (!readonly) {
52
57
  fs.mkdirSync(path.dirname(dbPath), { recursive: true });
53
58
  }
54
- this.db = new Database(dbPath, readonly ? { readonly: true, fileMustExist: true } : undefined);
55
- this.db.pragma("foreign_keys = ON");
59
+ this.db = openDatabase(dbPath, readonly);
60
+ // Refuse app-defined functions and unsafe virtual tables inside the schema of a database we did not build (--offline-db).
61
+ this.db.exec("PRAGMA trusted_schema = OFF");
56
62
  this.db.exec(SCHEMA_SQL);
57
63
  this.upsertAdvisoryStatement = this.db.prepare(`
58
64
  INSERT INTO advisories (id, modified_at, aliases_json, summary, osv_json)
@@ -100,6 +106,16 @@ export class LocalAdvisoryDatabase {
100
106
  SELECT osv_json
101
107
  FROM advisories
102
108
  WHERE id = ?
109
+ `);
110
+ this.ensureVersionsIndex(readonly);
111
+ this.deleteVersionsStatement = this.db.prepare("DELETE FROM advisory_versions WHERE advisory_id = ?");
112
+ this.insertVersionStatement = this.db.prepare(`
113
+ INSERT INTO advisory_versions (advisory_id, ecosystem, package_name, versions_json)
114
+ VALUES (@advisory_id, @ecosystem, @package_name, @versions_json)
115
+ `);
116
+ this.findExactIdsStatement = this.db.prepare(`
117
+ SELECT v.advisory_id FROM advisory_versions v, json_each(v.versions_json) version
118
+ WHERE v.ecosystem = ? AND v.package_name = ? AND version.type = 'text' AND version.value = ?
103
119
  `);
104
120
  this.findMatchingIdsStatement = this.db.prepare(`
105
121
  SELECT advisory_id, introduced, fixed, last_affected
@@ -107,6 +123,50 @@ export class LocalAdvisoryDatabase {
107
123
  WHERE ecosystem = ? AND package_name = ?
108
124
  `);
109
125
  }
126
+ ensureVersionsIndex(readonly) {
127
+ // Older databases retain the full OSV document but have no exact-version
128
+ // index. Backfill it once; read-only scans use a temporary index so the
129
+ // supplied file remains untouched and does not need to be synced again.
130
+ // Index each version list by package rather than materializing a separate
131
+ // row for every release in a large advisory dump.
132
+ const hasVersionsIndex = !!this.db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'advisory_versions'").get();
133
+ if (!hasVersionsIndex) {
134
+ inTransaction(this.db, () => {
135
+ this.db.exec(`CREATE ${readonly ? "TEMP " : ""}TABLE advisory_versions (
136
+ advisory_id TEXT NOT NULL,
137
+ ecosystem TEXT NOT NULL,
138
+ package_name TEXT NOT NULL,
139
+ versions_json TEXT NOT NULL
140
+ ${readonly ? "" : ", FOREIGN KEY (advisory_id) REFERENCES advisories(id) ON DELETE CASCADE"}
141
+ )`);
142
+ this.db.exec("CREATE INDEX idx_advisory_versions_lookup ON advisory_versions (ecosystem, package_name)");
143
+ this.db.exec("CREATE INDEX idx_advisory_versions_advisory ON advisory_versions (advisory_id)");
144
+ this.db.exec(`
145
+ INSERT INTO advisory_versions (advisory_id, ecosystem, package_name, versions_json)
146
+ WITH affected_entries AS (
147
+ SELECT a.id, CASE WHEN affected.type = 'object' THEN affected.value ELSE '{}' END AS entry
148
+ FROM advisories a,
149
+ json_each(CASE WHEN json_valid(a.osv_json) THEN a.osv_json ELSE '{}' END, '$.affected') affected
150
+ )
151
+ SELECT id, json_extract(entry, '$.package.ecosystem'),
152
+ json_extract(entry, '$.package.name'), json_extract(entry, '$.versions')
153
+ FROM affected_entries
154
+ WHERE json_type(entry, '$.versions') = 'array'
155
+ AND json_array_length(entry, '$.versions') > 0
156
+ AND json_extract(entry, '$.package.ecosystem') IS NOT NULL
157
+ AND json_extract(entry, '$.package.name') IS NOT NULL
158
+ `);
159
+ });
160
+ // Source metadata and package scanning can open the same file twice.
161
+ // Keep the resync remedy to one line per database in this process.
162
+ const warningKey = path.resolve(this.dbPath);
163
+ if (readonly && !LocalAdvisoryDatabase.warnedTemporaryIndexes.has(warningKey)) {
164
+ LocalAdvisoryDatabase.warnedTemporaryIndexes.add(warningKey);
165
+ console.warn("Warning: Built a temporary affected-version index for this older advisory database; " +
166
+ `run \`cve-lite advisories sync --output ${quoteShellPath(warningKey)}\` to persist the index and avoid rebuilding it on every offline scan.`);
167
+ }
168
+ }
169
+ }
110
170
  close() {
111
171
  this.db.close();
112
172
  }
@@ -124,18 +184,14 @@ export class LocalAdvisoryDatabase {
124
184
  };
125
185
  }
126
186
  upsertVulnerability(vuln) {
127
- const transaction = this.db.transaction((item) => {
128
- this.upsertVulnerabilityInternal(item);
129
- });
130
- transaction(vuln);
187
+ inTransaction(this.db, () => this.upsertVulnerabilityInternal(vuln));
131
188
  }
132
189
  bulkUpsertVulnerabilities(vulns) {
133
- const transaction = this.db.transaction((items) => {
134
- for (const item of items) {
190
+ const items = [...vulns];
191
+ inTransaction(this.db, () => {
192
+ for (const item of items)
135
193
  this.upsertVulnerabilityInternal(item);
136
- }
137
194
  });
138
- transaction([...vulns]);
139
195
  }
140
196
  upsertVulnerabilityInternal(vuln) {
141
197
  const advisoryRows = deriveAdvisoryPackageRows(vuln);
@@ -148,6 +204,9 @@ export class LocalAdvisoryDatabase {
148
204
  summary: vuln.summary ?? null,
149
205
  osv_json: advisoryJson,
150
206
  });
207
+ this.deleteVersionsStatement.run(vuln.id);
208
+ for (const row of deriveAdvisoryVersionRows(vuln))
209
+ this.insertVersionStatement.run(row);
151
210
  this.deleteRangesStatement.run(vuln.id);
152
211
  for (const row of advisoryRows) {
153
212
  this.insertRangeStatement.run(row);
@@ -174,13 +233,15 @@ export class LocalAdvisoryDatabase {
174
233
  * abort the lookup and is not reported as a clean miss.
175
234
  */
176
235
  matchPackage(pkg) {
236
+ // Explicit membership does not require an orderable version string.
237
+ const exactRows = this.findExactIdsStatement.all(pkg.ecosystem, pkg.name, pkg.version);
238
+ const ids = new Set(exactRows.map(row => row.advisory_id));
177
239
  if (!looksLikeVersion(pkg.version, pkg.ecosystem)) {
178
- // Semver returns no matches for a tag or a range. A PyPI version that
179
- // cannot be read is inconclusive, not unaffected.
180
- return { ids: [], unreadable: isPep440Ecosystem(pkg.ecosystem) };
240
+ // Skip range comparisons for an unreadable version, preserving any exact
241
+ // matches. An unreadable PyPI version remains inconclusive.
242
+ return { ids: [...ids], unreadable: isPep440Ecosystem(pkg.ecosystem) };
181
243
  }
182
244
  const rows = this.findMatchingIdsStatement.all(pkg.ecosystem, pkg.name);
183
- const ids = new Set();
184
245
  let unreadable = false;
185
246
  for (const row of rows) {
186
247
  if (isPep440Ecosystem(pkg.ecosystem)) {
@@ -369,3 +430,17 @@ function pep440PackageRows(advisoryId, ecosystem, packageName, events) {
369
430
  }
370
431
  return rows;
371
432
  }
433
+ function deriveAdvisoryVersionRows(vuln) {
434
+ const affectedEntries = Array.isArray(vuln.affected) ? vuln.affected : [];
435
+ return affectedEntries.flatMap(affected => {
436
+ const ecosystem = affected.package?.ecosystem;
437
+ const packageName = affected.package?.name;
438
+ if (!ecosystem || !packageName)
439
+ return [];
440
+ if (!Array.isArray(affected.versions) || affected.versions.length === 0)
441
+ return [];
442
+ return [{
443
+ advisory_id: vuln.id, ecosystem, package_name: packageName, versions_json: JSON.stringify(affected.versions),
444
+ }];
445
+ });
446
+ }
@@ -1,74 +1,49 @@
1
1
  import { extractErrorMessage } from "../utils/network.js";
2
+ import { DEFAULT_OSV_URL } from "../constants.js";
3
+ import { isPep440Ecosystem } from "../utils/version.js";
4
+ // Bound even a faulty server that never repeats a continuation token.
5
+ const MAX_OSV_QUERY_PAGES = 100;
2
6
  export class OsvAdvisorySource {
3
7
  baseUrl;
4
8
  debugLog;
5
9
  fetchImpl;
6
- constructor(baseUrl = "https://api.osv.dev", debugLog, fetchImpl) {
10
+ constructor(baseUrl = DEFAULT_OSV_URL, debugLog, fetchImpl) {
7
11
  this.baseUrl = baseUrl;
8
12
  this.debugLog = debugLog;
9
13
  this.fetchImpl = fetchImpl ?? fetch;
10
14
  }
11
15
  async queryBatch(packages, meta) {
12
16
  const requestUrl = `${this.baseUrl}/v1/querybatch`;
13
- const payload = {
14
- queries: packages.map(p => ({
15
- package: {
16
- ecosystem: p.ecosystem,
17
- name: p.name,
18
- },
19
- version: p.version,
20
- })),
21
- };
22
17
  const startedAt = this.debugLog ? Date.now() : 0;
23
18
  try {
24
- if (this.debugLog) {
25
- this.debugLog("OSV request", {
26
- batchId: meta?.batchId ?? null,
27
- method: "POST",
28
- url: requestUrl,
29
- headers: {
30
- "Content-Type": "application/json",
31
- },
32
- queryCount: packages.length,
33
- sample: packages.slice(0, 3).map(p => ({
34
- ecosystem: p.ecosystem,
35
- name: p.name,
36
- version: p.version,
37
- })),
38
- });
39
- }
40
- const response = await this.fetchImpl(requestUrl, {
41
- method: "POST",
42
- headers: {
43
- "Content-Type": "application/json",
44
- },
45
- body: JSON.stringify(payload),
46
- redirect: "error",
47
- });
48
- if (this.debugLog) {
49
- this.debugLog("OSV response", {
50
- batchId: meta?.batchId ?? null,
51
- method: "POST",
52
- url: requestUrl,
53
- status: response.status,
54
- statusText: response.statusText,
55
- durationMs: Date.now() - startedAt,
56
- });
57
- }
58
- if (!response.ok) {
59
- throw new Error(`OSV batch query failed: ${response.status} ${response.statusText}`);
60
- }
61
- const data = await response.json();
62
- if (!Array.isArray(data?.results)) {
63
- throw new Error('OSV batch query returned a 200 response with no "results" array');
64
- }
65
- if (data.results.length !== packages.length) {
66
- throw new Error(`OSV batch query returned ${data.results.length} result(s) for ${packages.length} quer(ies) - response length mismatch`);
67
- }
68
- return data.results.map((r, i) => ({
69
- package: packages[i].name,
70
- version: packages[i].version,
71
- vulnerabilities: r.vulns || [],
19
+ const matches = packages.map(() => new Map());
20
+ const seenTokens = packages.map(() => new Set());
21
+ let pending = packages.map((_, index) => ({ index }));
22
+ let pageCount = 0;
23
+ do {
24
+ if (pageCount >= MAX_OSV_QUERY_PAGES) {
25
+ throw new Error(`OSV batch query exceeded the ${MAX_OSV_QUERY_PAGES}-page limit`);
26
+ }
27
+ pageCount++;
28
+ const rows = await this.queryPage(pending.map(query => packages[query.index]), meta, pending.map(query => query.token));
29
+ const next = [];
30
+ for (let i = 0; i < rows.length; i++) {
31
+ const { index } = pending[i];
32
+ for (const vuln of rows[i].vulns ?? [])
33
+ matches[index].set(vuln.id, vuln);
34
+ const token = rows[i].next_page_token;
35
+ if (token === undefined)
36
+ continue;
37
+ if (typeof token !== "string" || token.length === 0 || seenTokens[index].has(token)) {
38
+ throw new Error("OSV batch query returned an invalid or repeated pagination token");
39
+ }
40
+ seenTokens[index].add(token);
41
+ next.push({ index, token });
42
+ }
43
+ pending = next;
44
+ } while (pending.length > 0);
45
+ return packages.map((pkg, index) => ({
46
+ package: pkg.name, version: pkg.version, vulnerabilities: [...matches[index].values()],
72
47
  }));
73
48
  }
74
49
  catch (error) {
@@ -84,8 +59,82 @@ export class OsvAdvisorySource {
84
59
  });
85
60
  }
86
61
  const message = extractErrorMessage(error);
87
- throw new Error(`OSV batch query failed for ${this.baseUrl}: ${message}`);
62
+ throw new Error(`OSV batch query failed for ${this.baseUrl}: ${message}. ` +
63
+ "Retry the scan. If it still fails, report the error at https://github.com/OWASP/cve-lite-cli/issues.");
64
+ }
65
+ }
66
+ async queryPage(packages, meta, tokens) {
67
+ const requestUrl = `${this.baseUrl}/v1/querybatch`;
68
+ const payload = {
69
+ queries: packages.map((p, index) => ({
70
+ package: {
71
+ ecosystem: isPep440Ecosystem(p.ecosystem) ? "PyPI" : p.ecosystem,
72
+ name: p.name,
73
+ },
74
+ version: p.version,
75
+ ...(tokens[index] ? { page_token: tokens[index] } : {}),
76
+ })),
77
+ };
78
+ const startedAt = this.debugLog ? Date.now() : 0;
79
+ if (this.debugLog) {
80
+ this.debugLog("OSV request", {
81
+ batchId: meta?.batchId ?? null,
82
+ method: "POST",
83
+ url: requestUrl,
84
+ headers: {
85
+ "Content-Type": "application/json",
86
+ },
87
+ queryCount: packages.length,
88
+ sample: packages.slice(0, 3).map(p => ({
89
+ ecosystem: isPep440Ecosystem(p.ecosystem) ? "PyPI" : p.ecosystem,
90
+ name: p.name,
91
+ version: p.version,
92
+ })),
93
+ });
94
+ }
95
+ const response = await this.fetchImpl(requestUrl, {
96
+ method: "POST",
97
+ headers: {
98
+ "Content-Type": "application/json",
99
+ },
100
+ body: JSON.stringify(payload),
101
+ redirect: "error",
102
+ });
103
+ if (this.debugLog) {
104
+ this.debugLog("OSV response", {
105
+ batchId: meta?.batchId ?? null,
106
+ method: "POST",
107
+ url: requestUrl,
108
+ status: response.status,
109
+ statusText: response.statusText,
110
+ durationMs: Date.now() - startedAt,
111
+ });
112
+ }
113
+ if (!response.ok) {
114
+ throw new Error(`OSV batch query failed: ${response.status} ${response.statusText}`);
115
+ }
116
+ const data = await response.json();
117
+ if (!Array.isArray(data?.results)) {
118
+ throw new Error('OSV batch query returned a 200 response with no "results" array');
119
+ }
120
+ if (data.results.length !== packages.length) {
121
+ throw new Error(`OSV batch query returned ${data.results.length} result(s) for ${packages.length} quer(ies) - response length mismatch`);
122
+ }
123
+ for (const row of data.results) {
124
+ if (row === null || typeof row !== "object" || Array.isArray(row)) {
125
+ throw new Error("OSV batch query returned an invalid result row");
126
+ }
127
+ if (row.vulns !== undefined && !Array.isArray(row.vulns)) {
128
+ throw new Error("OSV batch query returned an invalid vulnerability list");
129
+ }
130
+ for (const vuln of row.vulns ?? []) {
131
+ if (vuln === null || typeof vuln !== "object" || Array.isArray(vuln) ||
132
+ typeof vuln.id !== "string" || vuln.id.trim().length === 0) {
133
+ throw new Error("OSV batch query returned an invalid advisory ID");
134
+ }
135
+ }
88
136
  }
137
+ return data.results;
89
138
  }
90
139
  async getVuln(id) {
91
140
  const requestUrl = `${this.baseUrl}/v1/vulns/${encodeURIComponent(id)}`;
@@ -109,9 +158,10 @@ export class OsvAdvisorySource {
109
158
  });
110
159
  }
111
160
  if (!response.ok) {
112
- throw new Error(`OSV vuln fetch failed for ${id}: ${response.status} ${response.statusText}`);
161
+ throw Object.assign(new Error(`OSV vuln fetch failed for ${id}: ${response.status} ${response.statusText}`), { status: response.status });
113
162
  }
114
- return response.json();
163
+ // Awaited so an unreadable body is wrapped like any other failed lookup.
164
+ return (await response.json());
115
165
  }
116
166
  catch (error) {
117
167
  if (this.debugLog) {
@@ -125,7 +175,8 @@ export class OsvAdvisorySource {
125
175
  });
126
176
  }
127
177
  const message = extractErrorMessage(error);
128
- throw new Error(`OSV vuln fetch failed for ${id} via ${this.baseUrl}: ${message}`);
178
+ const status = error?.status;
179
+ throw Object.assign(new Error(`OSV vuln fetch failed for ${id} via ${this.baseUrl}: ${message}`), typeof status === "number" ? { status } : {});
129
180
  }
130
181
  }
131
182
  }
@@ -4,9 +4,9 @@ import path from "node:path";
4
4
  import { randomUUID } from "node:crypto";
5
5
  import { unzipSync } from "fflate";
6
6
  import { LocalAdvisoryDatabase } from "./local-db.js";
7
- import { ensureSentence, pluralize } from "../utils/string.js";
7
+ import { ensureSentence, isOutputStdout, pluralize } from "../utils/string.js";
8
8
  import { cleanupAbandonedStagingDirectories, recordStagingOwner, stagingDirectoryPrefix } from "./staging-directory.js";
9
- const DEFAULT_OSV_NPM_DUMP_URL = "https://storage.googleapis.com/osv-vulnerabilities/npm/all.zip";
9
+ import { DEFAULT_OSV_NPM_DUMP_URL } from "../constants.js";
10
10
  export const ADVISORY_DB_STALE_AFTER_MS = 7 * 24 * 60 * 60 * 1000;
11
11
  export class AdvisorySyncError extends Error {
12
12
  databaseState;
@@ -17,6 +17,9 @@ export class AdvisorySyncError extends Error {
17
17
  }
18
18
  }
19
19
  export function getDefaultAdvisoryDbPath(outputPath) {
20
+ if (isOutputStdout(outputPath)) {
21
+ throw new Error("Advisory database output cannot be stdout ('-'). Choose a database file path.");
22
+ }
20
23
  if (outputPath) {
21
24
  return path.resolve(outputPath);
22
25
  }
@@ -100,29 +103,28 @@ function rejectDirectoryOutput(dbPath) {
100
103
  async function ingestAdvisories(db, advisoryEntries, sourceUrl, onProgress) {
101
104
  const parsedVulns = [];
102
105
  let advisoryCount = 0;
106
+ let processedEntries = 0;
107
+ const totalAdvisories = advisoryEntries.length;
103
108
  const progressInterval = 250;
104
109
  for (const [, bytes] of advisoryEntries) {
110
+ processedEntries += 1;
105
111
  const text = Buffer.from(bytes).toString("utf8");
106
112
  const vuln = JSON.parse(text);
107
- if (!vuln.id)
108
- continue;
109
- if (!Array.isArray(vuln.affected) || vuln.affected.length === 0)
110
- continue;
111
113
  // OSV's /v1/querybatch endpoint excludes withdrawn advisories at query
112
114
  // time; mirror that here so offline scans don't surface false positives
113
115
  // from advisories that have been retracted.
114
- if (vuln.withdrawn)
115
- continue;
116
- parsedVulns.push(vuln);
117
- advisoryCount += 1;
118
- if (advisoryCount === 1 ||
119
- advisoryCount % progressInterval === 0 ||
120
- advisoryCount === advisoryEntries.length) {
116
+ if (vuln.id && Array.isArray(vuln.affected) && vuln.affected.length > 0 && !vuln.withdrawn) {
117
+ parsedVulns.push(vuln);
118
+ advisoryCount += 1;
119
+ }
120
+ if (processedEntries === 1 ||
121
+ processedEntries % progressInterval === 0 ||
122
+ processedEntries === totalAdvisories) {
121
123
  onProgress?.({
122
124
  phase: "ingest",
123
- processedAdvisories: advisoryCount,
124
- totalAdvisories: advisoryEntries.length,
125
- message: `Processing advisory records: ${advisoryCount} / ${advisoryEntries.length}`,
125
+ processedAdvisories: processedEntries,
126
+ totalAdvisories,
127
+ message: `Processing advisory records: ${processedEntries} / ${totalAdvisories}`,
126
128
  });
127
129
  await yieldToEventLoop();
128
130
  }