cve-lite-cli 1.41.5 → 1.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -19
- package/dist/advisory/detail-resolution.js +62 -0
- package/dist/advisory/local-db.js +90 -15
- package/dist/advisory/osv-advisory-source.js +113 -62
- package/dist/advisory/osv-sync.js +18 -16
- package/dist/advisory/sqlite.js +50 -0
- package/dist/audit-log/ndjson-writer.js +6 -3
- package/dist/audit-log/open-error.js +22 -0
- package/dist/cli/args.js +145 -87
- package/dist/cli/flags.js +2 -0
- package/dist/cli/help.js +6 -2
- package/dist/cli/overrides.js +5 -2
- package/dist/cli/validate.js +45 -5
- package/dist/constants.js +10 -0
- package/dist/index.js +15 -6
- package/dist/licenses/license-check.js +17 -4
- package/dist/osv/cache.js +7 -13
- package/dist/output/cyclonedx.js +57 -18
- package/dist/output/finding-display.js +8 -1
- package/dist/output/formatters.js +52 -11
- package/dist/output/html-reporter.js +40 -8
- package/dist/output/override-findings-terminal.js +21 -13
- package/dist/output/printers.js +185 -76
- package/dist/output/sarif.js +182 -16
- package/dist/output/spdx.js +15 -1
- package/dist/output/write-outputs.js +29 -20
- package/dist/overrides/context-builder.js +1 -7
- package/dist/overrides/detectors/oa005-nested-ineffective.js +6 -1
- package/dist/overrides/detectors/oa008-materialized.js +33 -10
- package/dist/overrides/detectors/oa009-stale-floor.js +2 -2
- package/dist/overrides/detectors/oa010-vulnerable-floor.js +19 -13
- package/dist/overrides/detectors/phantom-utils.js +2 -10
- package/dist/overrides/fixer.js +13 -2
- package/dist/overrides/index.js +1 -0
- package/dist/overrides/types.js +15 -1
- package/dist/parsers/bun-lock.js +6 -0
- package/dist/parsers/index.js +177 -40
- package/dist/parsers/multi-package.js +4 -0
- package/dist/parsers/package-lock.js +6 -2
- package/dist/parsers/pnpm-lock.js +4 -7
- package/dist/parsers/requirements-txt.js +185 -0
- package/dist/parsers/utils.js +11 -6
- package/dist/parsers/yarn-lock.js +1 -5
- package/dist/remediation/fix-commands.js +60 -36
- package/dist/remediation/parent-update-child-note.js +3 -2
- package/dist/scan/completeness.js +57 -4
- package/dist/scan/multi-folder-scan.js +27 -13
- package/dist/scan/single-scan.js +118 -37
- package/dist/scanner.js +59 -36
- package/dist/types.js +1 -1
- package/dist/utils/array.js +5 -0
- package/dist/utils/baseline.js +8 -2
- package/dist/utils/chalk.js +1 -3
- package/dist/utils/create-pr.js +2 -2
- package/dist/utils/debug-bundle.js +37 -22
- package/dist/utils/debug-project.js +6 -5
- package/dist/utils/debug-redaction.js +1 -1
- package/dist/utils/effective-options.js +5 -1
- package/dist/utils/finding.js +4 -0
- package/dist/utils/fix-runner.js +110 -36
- package/dist/utils/import-type-queries.js +5 -1
- package/dist/utils/install-command.js +6 -0
- package/dist/utils/malicious.js +11 -0
- package/dist/utils/package-identity.js +12 -2
- package/dist/utils/path-coverage.js +1 -5
- package/dist/utils/sanitize.js +59 -0
- package/dist/utils/sbom-format.js +17 -8
- package/dist/utils/severity.js +61 -7
- package/dist/utils/url.js +175 -0
- package/dist/utils/version.js +44 -0
- package/dist/utils/vuln.js +11 -14
- package/package.json +2 -4
package/README.md
CHANGED
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
<tr>
|
|
35
35
|
<td align="center" width="33%" valign="top"><strong>OWASP Lab Project</strong><br/><sub>Peer-reviewed by the org behind the OWASP Top 10:<br/>the security standard followed by millions of developers</sub></td>
|
|
36
36
|
<td align="center" width="33%" valign="top"><strong>Remediation-first</strong><br/><sub>Validated fix commands + parent-aware<br/>transitive guidance: not just CVE IDs</sub></td>
|
|
37
|
-
<td align="center" width="33%" valign="top"><strong>Runs locally</strong><br/><sub>
|
|
37
|
+
<td align="center" width="33%" valign="top"><strong>Runs locally</strong><br/><sub>Package names and versions are checked online, or fully offline with <code>--offline</code>.</sub></td>
|
|
38
38
|
</tr>
|
|
39
39
|
</table>
|
|
40
40
|
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
<br/>
|
|
69
69
|
|
|
70
70
|
<p>
|
|
71
|
-
⚡ <strong>One command. No account, no API key,
|
|
71
|
+
⚡ <strong>One command. No account, no API key, no source code uploaded.</strong>
|
|
72
72
|
<code>npx cve-lite-cli .</code>
|
|
73
73
|
<a href="#quick-start"><strong>→ Quick Start</strong></a>
|
|
74
74
|
</p>
|
|
@@ -121,7 +121,9 @@ npx cve-lite-cli /path/to/project
|
|
|
121
121
|
|
|
122
122
|
No account. No configuration. No source code leaves your machine.
|
|
123
123
|
|
|
124
|
-
|
|
124
|
+
CVE Lite CLI never uploads source code or lockfiles. In online mode, it sends package names and versions to advisory services, including OSV and the public npm registry advisory API. OSV and npm advisory APIs are not the only destinations receiving data during online scans: fix validation looks up package names (not versions) on the configured package registry, which is public npm by default, and EPSS enrichment sends the CVE IDs of findings to FIRST's API. Configuring an internal OSV endpoint only redirects OSV requests; npm advisory requests still use the public npm registry. Use `--offline` to scan against a local advisory database when dependency metadata must not leave your machine.
|
|
125
|
+
|
|
126
|
+
> Requires Node.js 22.13.0 or later on the 22.x line, or 23.4.0 and later. Installation works with npm scripts disabled.
|
|
125
127
|
|
|
126
128
|
## Usage
|
|
127
129
|
|
|
@@ -138,6 +140,7 @@ cve-lite /path/to/project --report # interactive HTML dashboard
|
|
|
138
140
|
cve-lite /path/to/project --check-overrides # audit override hygiene alongside the CVE scan
|
|
139
141
|
cve-lite /path/to/project --check-maintenance # flag CVE-blocking version drag and npm-deprecated direct deps
|
|
140
142
|
cve-lite /path/to/project --check-licenses # detect copyleft and unknown licenses alongside the CVE scan
|
|
143
|
+
cve-lite /path/to/project --python # scan pinned Python requirements.txt (experimental)
|
|
141
144
|
cve-lite advisories sync # sync advisory DB for offline use
|
|
142
145
|
cve-lite advisories init # create an empty advisory DB to populate yourself
|
|
143
146
|
cve-lite /path/to/project --offline # scan with no runtime API calls
|
|
@@ -322,6 +325,19 @@ jobs:
|
|
|
322
325
|
|
|
323
326
|
[Full Action documentation ->](https://owasp.org/cve-lite-cli/docs/github-action) - for scan-only CI gating see the [CI and Workflow Integration guide](https://owasp.org/cve-lite-cli/docs/workflow-integration).
|
|
324
327
|
|
|
328
|
+
## GitLab CI/CD
|
|
329
|
+
|
|
330
|
+
Add the scan job to `.gitlab-ci.yml`. The URL references the `v1.42.0` release tag of the template.
|
|
331
|
+
|
|
332
|
+
```yaml
|
|
333
|
+
include:
|
|
334
|
+
- remote: 'https://raw.githubusercontent.com/OWASP/cve-lite-cli/v1.42.0/templates/cve-lite.yml'
|
|
335
|
+
inputs:
|
|
336
|
+
fail-on: high
|
|
337
|
+
```
|
|
338
|
+
|
|
339
|
+
Set `path` to select one project when the repository has multiple lockfiles. Omit `inputs` for an informational scan of the current directory. Both modes publish the JSON report and CycloneDX SBOM as job artifacts. [GitLab CI inputs and offline setup](https://owasp.org/cve-lite-cli/docs/gitlab-ci).
|
|
340
|
+
|
|
325
341
|
## Real-world validation
|
|
326
342
|
|
|
327
343
|
CVE Lite CLI has been evaluated against real open-source projects to verify that it surfaces meaningful issues - including non-obvious transitive vulnerabilities and complex upgrade paths - not just low-signal advisory matches.
|
|
@@ -399,26 +415,13 @@ CVE Lite Classroom is a companion project maintained by the author of CVE Lite C
|
|
|
399
415
|
|
|
400
416
|
## Dependency footprint
|
|
401
417
|
|
|
402
|
-
**Runtime:** `yaml` · `yarn-lockfile` · `
|
|
418
|
+
**Runtime:** `yaml` · `yarn-lockfile` · `fflate` · `semver`
|
|
403
419
|
|
|
404
420
|
**Dev only:** `@types/node` · `tsx` · `typescript`
|
|
405
421
|
|
|
406
422
|
This is intentional. Because CVE Lite CLI is a security-oriented tool, runtime dependencies are kept minimal and reviewable.
|
|
407
423
|
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
`better-sqlite3` is a native Node.js addon that provides the SQLite binding for CVE Lite CLI's local advisory database. This database powers offline scanning (`--offline`, `--offline-db`), advisory sync (`cve-lite advisories sync`), and air-gapped workflows (`cve-lite advisories init`).
|
|
411
|
-
|
|
412
|
-
Because it includes compiled C/C++ code, npm must run an install script when installing `better-sqlite3`. During installation:
|
|
413
|
-
|
|
414
|
-
- **Prebuilt binary available:** `prebuild-install` downloads a prebuilt binary for your platform: fast, though npm may still print a deprecation warning for `prebuild-install` itself ([#519](https://github.com/OWASP/cve-lite-cli/issues/519)).
|
|
415
|
-
- **No prebuilt available:** the package falls back to compiling from source using `node-gyp`, which requires a C/C++ toolchain and produces more console output.
|
|
416
|
-
|
|
417
|
-
This install step belongs to `better-sqlite3`, not to CVE Lite CLI: the project itself does not define any `preinstall`, `install`, or `postinstall` hooks.
|
|
418
|
-
|
|
419
|
-
**npm install-script permissions:** Depending on your npm version and configuration, npm may prompt you to approve the install script before it runs. This is an npm security feature. Online scanning works even when the install script is blocked: the native binding is loaded lazily and only used by `--offline`, `--offline-db`, `cve-lite advisories sync`, and `cve-lite advisories init`. If you use any of those features, the `better-sqlite3` install script must be allowed to complete.
|
|
420
|
-
|
|
421
|
-
**npx behavior:** When using `npx cve-lite-cli`, a quiet run does not necessarily mean the install script was skipped. If npm's cache already contains the required prebuilt binary, installation is silent. On a cold cache, the same native install step runs.
|
|
424
|
+
The [local advisory database](https://owasp.org/cve-lite-cli/docs/offline-advisory-db) uses the SQLite driver bundled with Node.js. Existing database files continue to work.
|
|
422
425
|
|
|
423
426
|
## More documentation
|
|
424
427
|
|
|
@@ -446,7 +449,7 @@ For governance details - roles, decision-making, and dispute resolution - see [C
|
|
|
446
449
|
|
|
447
450
|
Feedback on output clarity, remediation guidance, ecosystem coverage, and CI usage is especially valuable.
|
|
448
451
|
|
|
449
|
-
See [CONTRIBUTING.md](https://github.com/OWASP/cve-lite-cli/blob/main/src/docs/CONTRIBUTING.md) to get started.
|
|
452
|
+
See [CONTRIBUTING.md](https://github.com/OWASP/cve-lite-cli/blob/main/src/docs/CONTRIBUTING.md) to get started. Every pull request must close an issue assigned to you, or one you opened that nobody has claimed.
|
|
450
453
|
|
|
451
454
|
## Add a badge to your project
|
|
452
455
|
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
// Pauses between attempts at one advisory detail lookup, so a lookup is tried
|
|
2
|
+
// once plus once per entry here. Short on purpose: lookups run ten at a time,
|
|
3
|
+
// and the worst case adds about a second per failing advisory (#1443).
|
|
4
|
+
export const OSV_DETAIL_RETRY_DELAYS_MS = [250, 750];
|
|
5
|
+
// The HTTP status of a failed lookup, as the OSV source attaches it to the
|
|
6
|
+
// error. The message is never read: a network error or a proxy's reason text
|
|
7
|
+
// can name a status the server did not send, and a 404 read from text would
|
|
8
|
+
// replace a cached record with a negative one.
|
|
9
|
+
function httpStatusOf(error) {
|
|
10
|
+
const attached = error.status;
|
|
11
|
+
return typeof attached === "number" ? attached : null;
|
|
12
|
+
}
|
|
13
|
+
// A 404 or 410 means OSV has no such record, which no retry will change.
|
|
14
|
+
// Everything else, network errors, rate limits and server errors included,
|
|
15
|
+
// may succeed on another attempt.
|
|
16
|
+
export function isTransientDetailFailure(error) {
|
|
17
|
+
if (!(error instanceof Error))
|
|
18
|
+
return true;
|
|
19
|
+
const status = httpStatusOf(error);
|
|
20
|
+
return status !== 404 && status !== 410;
|
|
21
|
+
}
|
|
22
|
+
// A client error other than a timeout or rate limit, such as a 400 or a 403
|
|
23
|
+
// from a proxy, gets the same answer however often it is asked. It is not
|
|
24
|
+
// retried, but it is not evidence the record is missing either, so it stays
|
|
25
|
+
// a failed lookup.
|
|
26
|
+
export function isRetryableDetailFailure(error) {
|
|
27
|
+
if (!(error instanceof Error))
|
|
28
|
+
return true;
|
|
29
|
+
const status = httpStatusOf(error);
|
|
30
|
+
if (status === null || status < 400 || status >= 500)
|
|
31
|
+
return true;
|
|
32
|
+
return status === 408 || status === 429;
|
|
33
|
+
}
|
|
34
|
+
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
35
|
+
// Fetches one advisory's details, retrying transient failures. When every
|
|
36
|
+
// attempt fails, a cached copy older than the TTL is used if one exists,
|
|
37
|
+
// because it carries the advisory's real id and aliases and keeps the finding
|
|
38
|
+
// intact; the caller discloses that it is cached data. A negative cache entry
|
|
39
|
+
// is never used as a fallback: it would turn a failed lookup into a
|
|
40
|
+
// confirmed-missing one on stale evidence.
|
|
41
|
+
export async function resolveAdvisoryDetail(params) {
|
|
42
|
+
const sleep = params.sleep ?? defaultSleep;
|
|
43
|
+
let lastError;
|
|
44
|
+
for (let attempt = 0; attempt <= OSV_DETAIL_RETRY_DELAYS_MS.length; attempt++) {
|
|
45
|
+
if (attempt > 0)
|
|
46
|
+
await sleep(OSV_DETAIL_RETRY_DELAYS_MS[attempt - 1]);
|
|
47
|
+
try {
|
|
48
|
+
return { kind: "fresh", vuln: await params.fetchDetail(params.id) };
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
if (!isTransientDetailFailure(error))
|
|
52
|
+
return { kind: "confirmed-missing" };
|
|
53
|
+
lastError = error;
|
|
54
|
+
if (!isRetryableDetailFailure(error))
|
|
55
|
+
break;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
if (params.staleEntry?.vuln) {
|
|
59
|
+
return { kind: "stale", vuln: params.staleEntry.vuln, cachedAt: params.staleEntry.cachedAt };
|
|
60
|
+
}
|
|
61
|
+
return { kind: "transient", error: lastError };
|
|
62
|
+
}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import
|
|
3
|
+
import { openDatabase, inTransaction } from "./sqlite.js";
|
|
4
4
|
import { compareVersions, isPep440Ecosystem, isPreReleaseVersion, looksLikeVersion, normalizeAdvisoryVersion } from "../utils/version.js";
|
|
5
5
|
import { readPep440Range, versionInPep440Window } from "../utils/versions/osv-range.js";
|
|
6
6
|
import { pep440Scheme } from "../utils/versions/pep440.js";
|
|
7
|
+
import { quoteShellPath } from "../utils/shell-quote.js";
|
|
7
8
|
const SCHEMA_SQL = `
|
|
8
9
|
CREATE TABLE IF NOT EXISTS advisories (
|
|
9
10
|
id TEXT PRIMARY KEY,
|
|
@@ -37,6 +38,7 @@ const SCHEMA_SQL = `
|
|
|
37
38
|
`;
|
|
38
39
|
export class LocalAdvisoryDatabase {
|
|
39
40
|
dbPath;
|
|
41
|
+
static warnedTemporaryIndexes = new Set();
|
|
40
42
|
db;
|
|
41
43
|
upsertAdvisoryStatement;
|
|
42
44
|
deleteRangesStatement;
|
|
@@ -45,14 +47,18 @@ export class LocalAdvisoryDatabase {
|
|
|
45
47
|
getMetadataStatement;
|
|
46
48
|
getVulnerabilityStatement;
|
|
47
49
|
findMatchingIdsStatement;
|
|
50
|
+
deleteVersionsStatement;
|
|
51
|
+
insertVersionStatement;
|
|
52
|
+
findExactIdsStatement;
|
|
48
53
|
constructor(dbPath, options) {
|
|
49
54
|
this.dbPath = dbPath;
|
|
50
55
|
const readonly = !!options?.readonly;
|
|
51
56
|
if (!readonly) {
|
|
52
57
|
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
53
58
|
}
|
|
54
|
-
this.db =
|
|
55
|
-
|
|
59
|
+
this.db = openDatabase(dbPath, readonly);
|
|
60
|
+
// Refuse app-defined functions and unsafe virtual tables inside the schema of a database we did not build (--offline-db).
|
|
61
|
+
this.db.exec("PRAGMA trusted_schema = OFF");
|
|
56
62
|
this.db.exec(SCHEMA_SQL);
|
|
57
63
|
this.upsertAdvisoryStatement = this.db.prepare(`
|
|
58
64
|
INSERT INTO advisories (id, modified_at, aliases_json, summary, osv_json)
|
|
@@ -100,6 +106,16 @@ export class LocalAdvisoryDatabase {
|
|
|
100
106
|
SELECT osv_json
|
|
101
107
|
FROM advisories
|
|
102
108
|
WHERE id = ?
|
|
109
|
+
`);
|
|
110
|
+
this.ensureVersionsIndex(readonly);
|
|
111
|
+
this.deleteVersionsStatement = this.db.prepare("DELETE FROM advisory_versions WHERE advisory_id = ?");
|
|
112
|
+
this.insertVersionStatement = this.db.prepare(`
|
|
113
|
+
INSERT INTO advisory_versions (advisory_id, ecosystem, package_name, versions_json)
|
|
114
|
+
VALUES (@advisory_id, @ecosystem, @package_name, @versions_json)
|
|
115
|
+
`);
|
|
116
|
+
this.findExactIdsStatement = this.db.prepare(`
|
|
117
|
+
SELECT v.advisory_id FROM advisory_versions v, json_each(v.versions_json) version
|
|
118
|
+
WHERE v.ecosystem = ? AND v.package_name = ? AND version.type = 'text' AND version.value = ?
|
|
103
119
|
`);
|
|
104
120
|
this.findMatchingIdsStatement = this.db.prepare(`
|
|
105
121
|
SELECT advisory_id, introduced, fixed, last_affected
|
|
@@ -107,6 +123,50 @@ export class LocalAdvisoryDatabase {
|
|
|
107
123
|
WHERE ecosystem = ? AND package_name = ?
|
|
108
124
|
`);
|
|
109
125
|
}
|
|
126
|
+
ensureVersionsIndex(readonly) {
|
|
127
|
+
// Older databases retain the full OSV document but have no exact-version
|
|
128
|
+
// index. Backfill it once; read-only scans use a temporary index so the
|
|
129
|
+
// supplied file remains untouched and does not need to be synced again.
|
|
130
|
+
// Index each version list by package rather than materializing a separate
|
|
131
|
+
// row for every release in a large advisory dump.
|
|
132
|
+
const hasVersionsIndex = !!this.db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'advisory_versions'").get();
|
|
133
|
+
if (!hasVersionsIndex) {
|
|
134
|
+
inTransaction(this.db, () => {
|
|
135
|
+
this.db.exec(`CREATE ${readonly ? "TEMP " : ""}TABLE advisory_versions (
|
|
136
|
+
advisory_id TEXT NOT NULL,
|
|
137
|
+
ecosystem TEXT NOT NULL,
|
|
138
|
+
package_name TEXT NOT NULL,
|
|
139
|
+
versions_json TEXT NOT NULL
|
|
140
|
+
${readonly ? "" : ", FOREIGN KEY (advisory_id) REFERENCES advisories(id) ON DELETE CASCADE"}
|
|
141
|
+
)`);
|
|
142
|
+
this.db.exec("CREATE INDEX idx_advisory_versions_lookup ON advisory_versions (ecosystem, package_name)");
|
|
143
|
+
this.db.exec("CREATE INDEX idx_advisory_versions_advisory ON advisory_versions (advisory_id)");
|
|
144
|
+
this.db.exec(`
|
|
145
|
+
INSERT INTO advisory_versions (advisory_id, ecosystem, package_name, versions_json)
|
|
146
|
+
WITH affected_entries AS (
|
|
147
|
+
SELECT a.id, CASE WHEN affected.type = 'object' THEN affected.value ELSE '{}' END AS entry
|
|
148
|
+
FROM advisories a,
|
|
149
|
+
json_each(CASE WHEN json_valid(a.osv_json) THEN a.osv_json ELSE '{}' END, '$.affected') affected
|
|
150
|
+
)
|
|
151
|
+
SELECT id, json_extract(entry, '$.package.ecosystem'),
|
|
152
|
+
json_extract(entry, '$.package.name'), json_extract(entry, '$.versions')
|
|
153
|
+
FROM affected_entries
|
|
154
|
+
WHERE json_type(entry, '$.versions') = 'array'
|
|
155
|
+
AND json_array_length(entry, '$.versions') > 0
|
|
156
|
+
AND json_extract(entry, '$.package.ecosystem') IS NOT NULL
|
|
157
|
+
AND json_extract(entry, '$.package.name') IS NOT NULL
|
|
158
|
+
`);
|
|
159
|
+
});
|
|
160
|
+
// Source metadata and package scanning can open the same file twice.
|
|
161
|
+
// Keep the resync remedy to one line per database in this process.
|
|
162
|
+
const warningKey = path.resolve(this.dbPath);
|
|
163
|
+
if (readonly && !LocalAdvisoryDatabase.warnedTemporaryIndexes.has(warningKey)) {
|
|
164
|
+
LocalAdvisoryDatabase.warnedTemporaryIndexes.add(warningKey);
|
|
165
|
+
console.warn("Warning: Built a temporary affected-version index for this older advisory database; " +
|
|
166
|
+
`run \`cve-lite advisories sync --output ${quoteShellPath(warningKey)}\` to persist the index and avoid rebuilding it on every offline scan.`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
110
170
|
close() {
|
|
111
171
|
this.db.close();
|
|
112
172
|
}
|
|
@@ -124,18 +184,14 @@ export class LocalAdvisoryDatabase {
|
|
|
124
184
|
};
|
|
125
185
|
}
|
|
126
186
|
upsertVulnerability(vuln) {
|
|
127
|
-
|
|
128
|
-
this.upsertVulnerabilityInternal(item);
|
|
129
|
-
});
|
|
130
|
-
transaction(vuln);
|
|
187
|
+
inTransaction(this.db, () => this.upsertVulnerabilityInternal(vuln));
|
|
131
188
|
}
|
|
132
189
|
bulkUpsertVulnerabilities(vulns) {
|
|
133
|
-
const
|
|
134
|
-
|
|
190
|
+
const items = [...vulns];
|
|
191
|
+
inTransaction(this.db, () => {
|
|
192
|
+
for (const item of items)
|
|
135
193
|
this.upsertVulnerabilityInternal(item);
|
|
136
|
-
}
|
|
137
194
|
});
|
|
138
|
-
transaction([...vulns]);
|
|
139
195
|
}
|
|
140
196
|
upsertVulnerabilityInternal(vuln) {
|
|
141
197
|
const advisoryRows = deriveAdvisoryPackageRows(vuln);
|
|
@@ -148,6 +204,9 @@ export class LocalAdvisoryDatabase {
|
|
|
148
204
|
summary: vuln.summary ?? null,
|
|
149
205
|
osv_json: advisoryJson,
|
|
150
206
|
});
|
|
207
|
+
this.deleteVersionsStatement.run(vuln.id);
|
|
208
|
+
for (const row of deriveAdvisoryVersionRows(vuln))
|
|
209
|
+
this.insertVersionStatement.run(row);
|
|
151
210
|
this.deleteRangesStatement.run(vuln.id);
|
|
152
211
|
for (const row of advisoryRows) {
|
|
153
212
|
this.insertRangeStatement.run(row);
|
|
@@ -174,13 +233,15 @@ export class LocalAdvisoryDatabase {
|
|
|
174
233
|
* abort the lookup and is not reported as a clean miss.
|
|
175
234
|
*/
|
|
176
235
|
matchPackage(pkg) {
|
|
236
|
+
// Explicit membership does not require an orderable version string.
|
|
237
|
+
const exactRows = this.findExactIdsStatement.all(pkg.ecosystem, pkg.name, pkg.version);
|
|
238
|
+
const ids = new Set(exactRows.map(row => row.advisory_id));
|
|
177
239
|
if (!looksLikeVersion(pkg.version, pkg.ecosystem)) {
|
|
178
|
-
//
|
|
179
|
-
//
|
|
180
|
-
return { ids: [], unreadable: isPep440Ecosystem(pkg.ecosystem) };
|
|
240
|
+
// Skip range comparisons for an unreadable version, preserving any exact
|
|
241
|
+
// matches. An unreadable PyPI version remains inconclusive.
|
|
242
|
+
return { ids: [...ids], unreadable: isPep440Ecosystem(pkg.ecosystem) };
|
|
181
243
|
}
|
|
182
244
|
const rows = this.findMatchingIdsStatement.all(pkg.ecosystem, pkg.name);
|
|
183
|
-
const ids = new Set();
|
|
184
245
|
let unreadable = false;
|
|
185
246
|
for (const row of rows) {
|
|
186
247
|
if (isPep440Ecosystem(pkg.ecosystem)) {
|
|
@@ -369,3 +430,17 @@ function pep440PackageRows(advisoryId, ecosystem, packageName, events) {
|
|
|
369
430
|
}
|
|
370
431
|
return rows;
|
|
371
432
|
}
|
|
433
|
+
function deriveAdvisoryVersionRows(vuln) {
|
|
434
|
+
const affectedEntries = Array.isArray(vuln.affected) ? vuln.affected : [];
|
|
435
|
+
return affectedEntries.flatMap(affected => {
|
|
436
|
+
const ecosystem = affected.package?.ecosystem;
|
|
437
|
+
const packageName = affected.package?.name;
|
|
438
|
+
if (!ecosystem || !packageName)
|
|
439
|
+
return [];
|
|
440
|
+
if (!Array.isArray(affected.versions) || affected.versions.length === 0)
|
|
441
|
+
return [];
|
|
442
|
+
return [{
|
|
443
|
+
advisory_id: vuln.id, ecosystem, package_name: packageName, versions_json: JSON.stringify(affected.versions),
|
|
444
|
+
}];
|
|
445
|
+
});
|
|
446
|
+
}
|
|
@@ -1,74 +1,49 @@
|
|
|
1
1
|
import { extractErrorMessage } from "../utils/network.js";
|
|
2
|
+
import { DEFAULT_OSV_URL } from "../constants.js";
|
|
3
|
+
import { isPep440Ecosystem } from "../utils/version.js";
|
|
4
|
+
// Bound even a faulty server that never repeats a continuation token.
|
|
5
|
+
const MAX_OSV_QUERY_PAGES = 100;
|
|
2
6
|
export class OsvAdvisorySource {
|
|
3
7
|
baseUrl;
|
|
4
8
|
debugLog;
|
|
5
9
|
fetchImpl;
|
|
6
|
-
constructor(baseUrl =
|
|
10
|
+
constructor(baseUrl = DEFAULT_OSV_URL, debugLog, fetchImpl) {
|
|
7
11
|
this.baseUrl = baseUrl;
|
|
8
12
|
this.debugLog = debugLog;
|
|
9
13
|
this.fetchImpl = fetchImpl ?? fetch;
|
|
10
14
|
}
|
|
11
15
|
async queryBatch(packages, meta) {
|
|
12
16
|
const requestUrl = `${this.baseUrl}/v1/querybatch`;
|
|
13
|
-
const payload = {
|
|
14
|
-
queries: packages.map(p => ({
|
|
15
|
-
package: {
|
|
16
|
-
ecosystem: p.ecosystem,
|
|
17
|
-
name: p.name,
|
|
18
|
-
},
|
|
19
|
-
version: p.version,
|
|
20
|
-
})),
|
|
21
|
-
};
|
|
22
17
|
const startedAt = this.debugLog ? Date.now() : 0;
|
|
23
18
|
try {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
url: requestUrl,
|
|
53
|
-
status: response.status,
|
|
54
|
-
statusText: response.statusText,
|
|
55
|
-
durationMs: Date.now() - startedAt,
|
|
56
|
-
});
|
|
57
|
-
}
|
|
58
|
-
if (!response.ok) {
|
|
59
|
-
throw new Error(`OSV batch query failed: ${response.status} ${response.statusText}`);
|
|
60
|
-
}
|
|
61
|
-
const data = await response.json();
|
|
62
|
-
if (!Array.isArray(data?.results)) {
|
|
63
|
-
throw new Error('OSV batch query returned a 200 response with no "results" array');
|
|
64
|
-
}
|
|
65
|
-
if (data.results.length !== packages.length) {
|
|
66
|
-
throw new Error(`OSV batch query returned ${data.results.length} result(s) for ${packages.length} quer(ies) - response length mismatch`);
|
|
67
|
-
}
|
|
68
|
-
return data.results.map((r, i) => ({
|
|
69
|
-
package: packages[i].name,
|
|
70
|
-
version: packages[i].version,
|
|
71
|
-
vulnerabilities: r.vulns || [],
|
|
19
|
+
const matches = packages.map(() => new Map());
|
|
20
|
+
const seenTokens = packages.map(() => new Set());
|
|
21
|
+
let pending = packages.map((_, index) => ({ index }));
|
|
22
|
+
let pageCount = 0;
|
|
23
|
+
do {
|
|
24
|
+
if (pageCount >= MAX_OSV_QUERY_PAGES) {
|
|
25
|
+
throw new Error(`OSV batch query exceeded the ${MAX_OSV_QUERY_PAGES}-page limit`);
|
|
26
|
+
}
|
|
27
|
+
pageCount++;
|
|
28
|
+
const rows = await this.queryPage(pending.map(query => packages[query.index]), meta, pending.map(query => query.token));
|
|
29
|
+
const next = [];
|
|
30
|
+
for (let i = 0; i < rows.length; i++) {
|
|
31
|
+
const { index } = pending[i];
|
|
32
|
+
for (const vuln of rows[i].vulns ?? [])
|
|
33
|
+
matches[index].set(vuln.id, vuln);
|
|
34
|
+
const token = rows[i].next_page_token;
|
|
35
|
+
if (token === undefined)
|
|
36
|
+
continue;
|
|
37
|
+
if (typeof token !== "string" || token.length === 0 || seenTokens[index].has(token)) {
|
|
38
|
+
throw new Error("OSV batch query returned an invalid or repeated pagination token");
|
|
39
|
+
}
|
|
40
|
+
seenTokens[index].add(token);
|
|
41
|
+
next.push({ index, token });
|
|
42
|
+
}
|
|
43
|
+
pending = next;
|
|
44
|
+
} while (pending.length > 0);
|
|
45
|
+
return packages.map((pkg, index) => ({
|
|
46
|
+
package: pkg.name, version: pkg.version, vulnerabilities: [...matches[index].values()],
|
|
72
47
|
}));
|
|
73
48
|
}
|
|
74
49
|
catch (error) {
|
|
@@ -84,8 +59,82 @@ export class OsvAdvisorySource {
|
|
|
84
59
|
});
|
|
85
60
|
}
|
|
86
61
|
const message = extractErrorMessage(error);
|
|
87
|
-
throw new Error(`OSV batch query failed for ${this.baseUrl}: ${message}`
|
|
62
|
+
throw new Error(`OSV batch query failed for ${this.baseUrl}: ${message}. ` +
|
|
63
|
+
"Retry the scan. If it still fails, report the error at https://github.com/OWASP/cve-lite-cli/issues.");
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
async queryPage(packages, meta, tokens) {
|
|
67
|
+
const requestUrl = `${this.baseUrl}/v1/querybatch`;
|
|
68
|
+
const payload = {
|
|
69
|
+
queries: packages.map((p, index) => ({
|
|
70
|
+
package: {
|
|
71
|
+
ecosystem: isPep440Ecosystem(p.ecosystem) ? "PyPI" : p.ecosystem,
|
|
72
|
+
name: p.name,
|
|
73
|
+
},
|
|
74
|
+
version: p.version,
|
|
75
|
+
...(tokens[index] ? { page_token: tokens[index] } : {}),
|
|
76
|
+
})),
|
|
77
|
+
};
|
|
78
|
+
const startedAt = this.debugLog ? Date.now() : 0;
|
|
79
|
+
if (this.debugLog) {
|
|
80
|
+
this.debugLog("OSV request", {
|
|
81
|
+
batchId: meta?.batchId ?? null,
|
|
82
|
+
method: "POST",
|
|
83
|
+
url: requestUrl,
|
|
84
|
+
headers: {
|
|
85
|
+
"Content-Type": "application/json",
|
|
86
|
+
},
|
|
87
|
+
queryCount: packages.length,
|
|
88
|
+
sample: packages.slice(0, 3).map(p => ({
|
|
89
|
+
ecosystem: isPep440Ecosystem(p.ecosystem) ? "PyPI" : p.ecosystem,
|
|
90
|
+
name: p.name,
|
|
91
|
+
version: p.version,
|
|
92
|
+
})),
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
const response = await this.fetchImpl(requestUrl, {
|
|
96
|
+
method: "POST",
|
|
97
|
+
headers: {
|
|
98
|
+
"Content-Type": "application/json",
|
|
99
|
+
},
|
|
100
|
+
body: JSON.stringify(payload),
|
|
101
|
+
redirect: "error",
|
|
102
|
+
});
|
|
103
|
+
if (this.debugLog) {
|
|
104
|
+
this.debugLog("OSV response", {
|
|
105
|
+
batchId: meta?.batchId ?? null,
|
|
106
|
+
method: "POST",
|
|
107
|
+
url: requestUrl,
|
|
108
|
+
status: response.status,
|
|
109
|
+
statusText: response.statusText,
|
|
110
|
+
durationMs: Date.now() - startedAt,
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
if (!response.ok) {
|
|
114
|
+
throw new Error(`OSV batch query failed: ${response.status} ${response.statusText}`);
|
|
115
|
+
}
|
|
116
|
+
const data = await response.json();
|
|
117
|
+
if (!Array.isArray(data?.results)) {
|
|
118
|
+
throw new Error('OSV batch query returned a 200 response with no "results" array');
|
|
119
|
+
}
|
|
120
|
+
if (data.results.length !== packages.length) {
|
|
121
|
+
throw new Error(`OSV batch query returned ${data.results.length} result(s) for ${packages.length} quer(ies) - response length mismatch`);
|
|
122
|
+
}
|
|
123
|
+
for (const row of data.results) {
|
|
124
|
+
if (row === null || typeof row !== "object" || Array.isArray(row)) {
|
|
125
|
+
throw new Error("OSV batch query returned an invalid result row");
|
|
126
|
+
}
|
|
127
|
+
if (row.vulns !== undefined && !Array.isArray(row.vulns)) {
|
|
128
|
+
throw new Error("OSV batch query returned an invalid vulnerability list");
|
|
129
|
+
}
|
|
130
|
+
for (const vuln of row.vulns ?? []) {
|
|
131
|
+
if (vuln === null || typeof vuln !== "object" || Array.isArray(vuln) ||
|
|
132
|
+
typeof vuln.id !== "string" || vuln.id.trim().length === 0) {
|
|
133
|
+
throw new Error("OSV batch query returned an invalid advisory ID");
|
|
134
|
+
}
|
|
135
|
+
}
|
|
88
136
|
}
|
|
137
|
+
return data.results;
|
|
89
138
|
}
|
|
90
139
|
async getVuln(id) {
|
|
91
140
|
const requestUrl = `${this.baseUrl}/v1/vulns/${encodeURIComponent(id)}`;
|
|
@@ -109,9 +158,10 @@ export class OsvAdvisorySource {
|
|
|
109
158
|
});
|
|
110
159
|
}
|
|
111
160
|
if (!response.ok) {
|
|
112
|
-
throw new Error(`OSV vuln fetch failed for ${id}: ${response.status} ${response.statusText}`);
|
|
161
|
+
throw Object.assign(new Error(`OSV vuln fetch failed for ${id}: ${response.status} ${response.statusText}`), { status: response.status });
|
|
113
162
|
}
|
|
114
|
-
|
|
163
|
+
// Awaited so an unreadable body is wrapped like any other failed lookup.
|
|
164
|
+
return (await response.json());
|
|
115
165
|
}
|
|
116
166
|
catch (error) {
|
|
117
167
|
if (this.debugLog) {
|
|
@@ -125,7 +175,8 @@ export class OsvAdvisorySource {
|
|
|
125
175
|
});
|
|
126
176
|
}
|
|
127
177
|
const message = extractErrorMessage(error);
|
|
128
|
-
|
|
178
|
+
const status = error?.status;
|
|
179
|
+
throw Object.assign(new Error(`OSV vuln fetch failed for ${id} via ${this.baseUrl}: ${message}`), typeof status === "number" ? { status } : {});
|
|
129
180
|
}
|
|
130
181
|
}
|
|
131
182
|
}
|
|
@@ -4,9 +4,9 @@ import path from "node:path";
|
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
5
5
|
import { unzipSync } from "fflate";
|
|
6
6
|
import { LocalAdvisoryDatabase } from "./local-db.js";
|
|
7
|
-
import { ensureSentence, pluralize } from "../utils/string.js";
|
|
7
|
+
import { ensureSentence, isOutputStdout, pluralize } from "../utils/string.js";
|
|
8
8
|
import { cleanupAbandonedStagingDirectories, recordStagingOwner, stagingDirectoryPrefix } from "./staging-directory.js";
|
|
9
|
-
|
|
9
|
+
import { DEFAULT_OSV_NPM_DUMP_URL } from "../constants.js";
|
|
10
10
|
export const ADVISORY_DB_STALE_AFTER_MS = 7 * 24 * 60 * 60 * 1000;
|
|
11
11
|
export class AdvisorySyncError extends Error {
|
|
12
12
|
databaseState;
|
|
@@ -17,6 +17,9 @@ export class AdvisorySyncError extends Error {
|
|
|
17
17
|
}
|
|
18
18
|
}
|
|
19
19
|
export function getDefaultAdvisoryDbPath(outputPath) {
|
|
20
|
+
if (isOutputStdout(outputPath)) {
|
|
21
|
+
throw new Error("Advisory database output cannot be stdout ('-'). Choose a database file path.");
|
|
22
|
+
}
|
|
20
23
|
if (outputPath) {
|
|
21
24
|
return path.resolve(outputPath);
|
|
22
25
|
}
|
|
@@ -100,29 +103,28 @@ function rejectDirectoryOutput(dbPath) {
|
|
|
100
103
|
async function ingestAdvisories(db, advisoryEntries, sourceUrl, onProgress) {
|
|
101
104
|
const parsedVulns = [];
|
|
102
105
|
let advisoryCount = 0;
|
|
106
|
+
let processedEntries = 0;
|
|
107
|
+
const totalAdvisories = advisoryEntries.length;
|
|
103
108
|
const progressInterval = 250;
|
|
104
109
|
for (const [, bytes] of advisoryEntries) {
|
|
110
|
+
processedEntries += 1;
|
|
105
111
|
const text = Buffer.from(bytes).toString("utf8");
|
|
106
112
|
const vuln = JSON.parse(text);
|
|
107
|
-
if (!vuln.id)
|
|
108
|
-
continue;
|
|
109
|
-
if (!Array.isArray(vuln.affected) || vuln.affected.length === 0)
|
|
110
|
-
continue;
|
|
111
113
|
// OSV's /v1/querybatch endpoint excludes withdrawn advisories at query
|
|
112
114
|
// time; mirror that here so offline scans don't surface false positives
|
|
113
115
|
// from advisories that have been retracted.
|
|
114
|
-
if (vuln.withdrawn)
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
if (
|
|
119
|
-
|
|
120
|
-
|
|
116
|
+
if (vuln.id && Array.isArray(vuln.affected) && vuln.affected.length > 0 && !vuln.withdrawn) {
|
|
117
|
+
parsedVulns.push(vuln);
|
|
118
|
+
advisoryCount += 1;
|
|
119
|
+
}
|
|
120
|
+
if (processedEntries === 1 ||
|
|
121
|
+
processedEntries % progressInterval === 0 ||
|
|
122
|
+
processedEntries === totalAdvisories) {
|
|
121
123
|
onProgress?.({
|
|
122
124
|
phase: "ingest",
|
|
123
|
-
processedAdvisories:
|
|
124
|
-
totalAdvisories
|
|
125
|
-
message: `Processing advisory records: ${
|
|
125
|
+
processedAdvisories: processedEntries,
|
|
126
|
+
totalAdvisories,
|
|
127
|
+
message: `Processing advisory records: ${processedEntries} / ${totalAdvisories}`,
|
|
126
128
|
});
|
|
127
129
|
await yieldToEventLoop();
|
|
128
130
|
}
|