@blamejs/exceptd-skills 0.19.33 → 0.19.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/bin/exceptd.js +896 -2824
- package/data/_indexes/_meta.json +2 -2
- package/lib/auto-discovery.js +56 -286
- package/lib/canonical-eq.js +7 -40
- package/lib/citation-resolve.js +22 -70
- package/lib/collectors/ai-api.js +20 -54
- package/lib/collectors/cicd-pipeline-compromise.js +40 -108
- package/lib/collectors/citation-hygiene.js +72 -210
- package/lib/collectors/containers.js +41 -130
- package/lib/collectors/cred-stores.js +31 -115
- package/lib/collectors/crypto-codebase.js +55 -138
- package/lib/collectors/crypto.js +24 -54
- package/lib/collectors/hardening.js +20 -78
- package/lib/collectors/kernel.js +16 -46
- package/lib/collectors/library-author.js +57 -206
- package/lib/collectors/mcp.js +24 -70
- package/lib/collectors/runtime.js +24 -86
- package/lib/collectors/sbom.js +34 -106
- package/lib/collectors/scan-excludes.js +31 -138
- package/lib/collectors/secrets.js +62 -178
- package/lib/cross-ref-api.js +39 -123
- package/lib/currency-severity.js +8 -27
- package/lib/cve-batch.js +13 -21
- package/lib/cve-cli.js +13 -20
- package/lib/cve-curation.js +72 -239
- package/lib/cve-regression-watcher.js +29 -152
- package/lib/cvss.js +13 -54
- package/lib/doctor-bucketing.js +3 -19
- package/lib/exit-codes.js +10 -42
- package/lib/flag-suggest.js +7 -25
- package/lib/framework-gap.js +35 -114
- package/lib/gap-detectors.js +37 -159
- package/lib/id-validation.js +9 -30
- package/lib/job-queue.js +13 -36
- package/lib/lint-skills.js +64 -232
- package/lib/playbook-runner.js +693 -2095
- package/lib/prefetch.js +100 -376
- package/lib/refresh-external.js +199 -627
- package/lib/refresh-network.js +75 -307
- package/lib/rfc-cli.js +23 -68
- package/lib/scoring.js +77 -145
- package/lib/sign.js +43 -229
- package/lib/source-advisories.js +43 -194
- package/lib/source-ghsa.js +37 -120
- package/lib/source-osv.js +94 -266
- package/lib/ttp-mapper.js +14 -24
- package/lib/upstream-check-cli.js +10 -28
- package/lib/upstream-check.js +19 -44
- package/lib/validate-catalog-meta.js +17 -61
- package/lib/validate-cve-catalog.js +43 -119
- package/lib/validate-indexes.js +25 -76
- package/lib/validate-package.js +16 -62
- package/lib/validate-playbooks.js +69 -275
- package/lib/validate-vendor.js +16 -49
- package/lib/verify.js +56 -286
- package/lib/version-pins.js +5 -34
- package/lib/worker-pool.js +11 -30
- package/lib/xml-tokenizer.js +47 -152
- package/manifest.json +53 -53
- package/orchestrator/dispatcher.js +17 -68
- package/orchestrator/event-bus.js +11 -74
- package/orchestrator/index.js +138 -412
- package/orchestrator/pipeline.js +28 -85
- package/orchestrator/scanner.js +34 -138
- package/orchestrator/scheduler.js +20 -84
- package/package.json +1 -1
- package/sbom.cdx.json +241 -241
- package/scripts/audit-catalog-gaps.js +9 -62
- package/scripts/audit-cross-skill.js +5 -31
- package/scripts/audit-perf.js +6 -16
- package/scripts/backfill-theater-test.js +7 -64
- package/scripts/bootstrap.js +12 -44
- package/scripts/build-indexes.js +40 -154
- package/scripts/builders/activity-feed.js +4 -14
- package/scripts/builders/catalog-summaries.js +3 -10
- package/scripts/builders/currency.js +7 -20
- package/scripts/builders/cwe-chains.js +7 -30
- package/scripts/builders/did-ladders.js +6 -13
- package/scripts/builders/frequency.js +5 -19
- package/scripts/builders/jurisdiction-clocks.js +6 -25
- package/scripts/builders/recipes.js +6 -14
- package/scripts/builders/section-offsets.js +13 -51
- package/scripts/builders/stale-content.js +7 -28
- package/scripts/builders/summary-cards.js +8 -29
- package/scripts/builders/theater-fingerprints.js +12 -27
- package/scripts/builders/token-budget.js +4 -31
- package/scripts/check-agents-md-collectors.js +11 -54
- package/scripts/check-catalog-gap-budget.js +15 -32
- package/scripts/check-changelog-extract.js +18 -48
- package/scripts/check-codebase-patterns-currency.js +6 -22
- package/scripts/check-codebase-patterns.js +50 -143
- package/scripts/check-epss-consistency.js +9 -64
- package/scripts/check-framework-gap-coverage.js +13 -31
- package/scripts/check-manifest-snapshot.js +13 -73
- package/scripts/check-sbom-currency.js +44 -142
- package/scripts/check-test-count.js +15 -52
- package/scripts/check-test-coverage.js +66 -197
- package/scripts/check-test-subjects.js +21 -62
- package/scripts/check-ttp-references.js +14 -38
- package/scripts/check-ttp-upstream.js +8 -40
- package/scripts/check-version-bump.js +9 -61
- package/scripts/check-version-tags.js +20 -121
- package/scripts/predeploy.js +38 -184
- package/scripts/refresh-manifest-snapshot.js +16 -38
- package/scripts/refresh-mitre-atlas.js +3 -8
- package/scripts/refresh-mitre-attack.js +1 -8
- package/scripts/refresh-mitre-d3fend.js +3 -9
- package/scripts/refresh-mitre-ics-attack.js +3 -8
- package/scripts/refresh-reverse-refs.js +27 -94
- package/scripts/refresh-rfc-index.js +2 -10
- package/scripts/refresh-sbom.js +31 -161
- package/scripts/refresh-upstream-catalogs.js +40 -137
- package/scripts/release.js +69 -232
- package/scripts/run-e2e-scenarios.js +24 -71
- package/scripts/sync-manifest-metadata.js +10 -34
- package/scripts/sync-package-description.js +8 -17
- package/scripts/validate-vendor-online.js +13 -44
- package/scripts/verify-shipped-tarball.js +35 -140
package/lib/refresh-external.js
CHANGED
|
@@ -1,38 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* External-data refresh orchestrator. Pulls the latest from the canonical
|
|
6
|
-
* upstream sources and either reports drift (dry-run, default) or applies
|
|
7
|
-
* the drift as an upsert into the local catalog (--apply).
|
|
8
|
-
*
|
|
9
|
-
* Sources (each is independently pluggable):
|
|
10
|
-
*
|
|
11
|
-
* KEV — CISA Known Exploited Vulnerabilities (per-CVE upsert of
|
|
12
|
-
* cisa_kev / cisa_kev_date)
|
|
13
|
-
* EPSS — FIRST EPSS (per-CVE upsert of epss_score / epss_percentile /
|
|
14
|
-
* epss_date)
|
|
15
|
-
* NVD — NIST NVD 2.0 (per-CVE upsert of cvss_score / cvss_vector)
|
|
16
|
-
* RFC — IETF Datatracker (per-RFC upsert of status)
|
|
17
|
-
* PINS — MITRE ATLAS / ATT&CK / D3FEND / CWE upstream releases
|
|
18
|
-
* (REPORT-ONLY — version bumps need audit per AGENTS.md
|
|
19
|
-
* Hard Rule #12, so they surface as findings, not auto-applied)
|
|
20
|
-
*
|
|
21
|
-
* Usage:
|
|
22
|
-
* node lib/refresh-external.js # dry-run all sources
|
|
23
|
-
* node lib/refresh-external.js --apply # apply all sources
|
|
24
|
-
* node lib/refresh-external.js --source kev # one source, dry-run
|
|
25
|
-
* node lib/refresh-external.js --apply --source kev,epss
|
|
26
|
-
* node lib/refresh-external.js --from-fixture <path> # use frozen fixture
|
|
27
|
-
* payloads (offline)
|
|
28
|
-
*
|
|
29
|
-
* Exit codes:
|
|
30
|
-
* 0 — dry-run completed (regardless of whether drift was found)
|
|
31
|
-
* 1 — apply mode AND a downstream gate (validate-indexes, lint) failed
|
|
32
|
-
* 2 — unrecoverable runner error
|
|
33
|
-
*
|
|
34
|
-
* The refresh-report.json artifact lives at the repo root and is
|
|
35
|
-
* gitignored. CI uploads it as a workflow artifact.
|
|
3
|
+
* External-data refresh orchestrator: reports upstream drift (dry-run, the
|
|
4
|
+
* default) or upserts it into the local catalog (--apply).
|
|
36
5
|
*/
|
|
37
6
|
|
|
38
7
|
const fs = require("fs");
|
|
@@ -45,15 +14,8 @@ const ROOT = path.join(__dirname, "..");
|
|
|
45
14
|
const ABS = (p) => path.join(ROOT, p);
|
|
46
15
|
const TODAY = new Date().toISOString().slice(0, 10);
|
|
47
16
|
|
|
48
|
-
//
|
|
49
|
-
//
|
|
50
|
-
// data/cve-catalog.json. Resolution order:
|
|
51
|
-
// 1. opts.catalog (--catalog CLI arg)
|
|
52
|
-
// 2. process.env.EXCEPTD_CVE_CATALOG (env var)
|
|
53
|
-
// 3. ROOT/data/cve-catalog.json (default)
|
|
54
|
-
// All four write-sites in this file route through resolveCatalogPath() so
|
|
55
|
-
// that the redirect is consistent across the advisory-import, GHSA-import,
|
|
56
|
-
// and per-source merge code paths.
|
|
17
|
+
// Every write site in this file resolves the catalog path through here, so a
|
|
18
|
+
// --catalog / EXCEPTD_CVE_CATALOG redirect holds across all of them.
|
|
57
19
|
function resolveCatalogPath(opts) {
|
|
58
20
|
if (opts && opts.catalog) return path.resolve(opts.catalog);
|
|
59
21
|
if (process.env.EXCEPTD_CVE_CATALOG) return path.resolve(process.env.EXCEPTD_CVE_CATALOG);
|
|
@@ -67,7 +29,7 @@ function parseArgs(argv) {
|
|
|
67
29
|
fromFixture: null, // path to fixture dir
|
|
68
30
|
fromCache: null, // path to .cache/upstream dir (or default if --from-cache passed bare)
|
|
69
31
|
swarm: false, // fan-out sources across worker threads
|
|
70
|
-
advisory: null, //
|
|
32
|
+
advisory: null, // single-advisory seed (CVE-* or GHSA-*)
|
|
71
33
|
help: false,
|
|
72
34
|
quiet: false,
|
|
73
35
|
json: false,
|
|
@@ -81,16 +43,10 @@ function parseArgs(argv) {
|
|
|
81
43
|
else if (a === "--help" || a === "-h") out.help = true;
|
|
82
44
|
else if (a === "--advisory") { out.advisory = argv[++i]; }
|
|
83
45
|
else if (a.startsWith("--advisory=")) { out.advisory = a.slice("--advisory=".length); }
|
|
84
|
-
// --check-advisories polls the primary-source advisory feeds (Qualys TRU,
|
|
85
|
-
// RHSA, USN, ZDI, kernel.org, oss-security, vendor research blogs) and
|
|
86
|
-
// reports newly-seen CVE IDs ahead of NVD enrichment. Report-only: it
|
|
87
|
-
// selects the `advisories` source and never applies — operators triage the
|
|
88
|
-
// diffs[] and seed promising IDs via `refresh --advisory <id> --apply`.
|
|
89
46
|
else if (a === "--check-advisories") { out.source = "advisories"; out.apply = false; out.checkAdvisories = true; }
|
|
90
47
|
else if (a === "--catalog") { out.catalog = argv[++i]; }
|
|
91
48
|
else if (a.startsWith("--catalog=")) { out.catalog = a.slice("--catalog=".length); }
|
|
92
49
|
else if (a === "--from-cache") {
|
|
93
|
-
// accept either --from-cache <path> or --from-cache (default path)
|
|
94
50
|
const next = argv[i + 1];
|
|
95
51
|
if (next && !next.startsWith("--")) { out.fromCache = next; i++; }
|
|
96
52
|
else out.fromCache = ".cache/upstream";
|
|
@@ -102,37 +58,19 @@ function parseArgs(argv) {
|
|
|
102
58
|
else if (a.startsWith("--from-fixture=")) out.fromFixture = a.slice("--from-fixture=".length);
|
|
103
59
|
else if (a === "--report-out") out.reportOut = argv[++i];
|
|
104
60
|
else if (a.startsWith("--report-out=")) out.reportOut = a.slice("--report-out=".length);
|
|
105
|
-
// Honour `--air-gap` here so it reaches the GHSA/OSV source modules.
|
|
106
|
-
// EXCEPTD_AIR_GAP=1 still works as an env-var fallback so existing
|
|
107
|
-
// automation isn't broken.
|
|
108
61
|
else if (a === "--air-gap") out.airGap = true;
|
|
109
|
-
//
|
|
110
|
-
// Required when the operator intentionally wants to consume a cache
|
|
111
|
-
// older than 7d or one that was prefetched without a signing keypair.
|
|
112
|
-
// EXCEPTD_FORCE_STALE=1 mirrors for non-interactive automation.
|
|
62
|
+
// Bypasses the cache freshness and signature refusals.
|
|
113
63
|
else if (a === "--force-stale") out.forceStale = true;
|
|
114
|
-
// --drift-only reconciles the entries the catalog already holds and skips
|
|
115
|
-
// auto-discovery of new ones. Discovery imports drafts that still need
|
|
116
|
-
// curation, so an operator correcting stale fields on shipped entries would
|
|
117
|
-
// otherwise have to take that work in the same commit or leave the fields
|
|
118
|
-
// stale.
|
|
119
64
|
else if (a === "--drift-only") out.driftOnly = true;
|
|
120
|
-
//
|
|
121
|
-
// main() can delegate to lib/prefetch.js (the same routing bin/exceptd.js
|
|
122
|
-
// performs) when this script is invoked directly — otherwise the help
|
|
123
|
-
// text's "report-only, no cache write" promise for --no-network is a lie
|
|
124
|
-
// on the direct path, which would fall through to the live refresh loop.
|
|
65
|
+
// Cache operations; main() delegates them to lib/prefetch.js.
|
|
125
66
|
else if (a === "--no-network") { out.noNetwork = true; }
|
|
126
67
|
else if (a === "--prefetch") { out.prefetch = true; }
|
|
127
|
-
//
|
|
128
|
-
// so the unknown-flag guard below doesn't false-reject them.
|
|
68
|
+
// bin-translated aliases, accepted so the unknown-flag guard below spares them.
|
|
129
69
|
else if (
|
|
130
70
|
a === "--indexes-only" ||
|
|
131
71
|
a === "--network" || a === "--curate" || a === "--force-stale-acked"
|
|
132
72
|
) { /* accepted, no-op at this layer */ }
|
|
133
|
-
// Any remaining --flag is
|
|
134
|
-
// the loop rather than silently dropping it into a default full-refresh
|
|
135
|
-
// (which previously hit the live network on every source).
|
|
73
|
+
// Any remaining --flag is a typo: recorded here, refused in main().
|
|
136
74
|
else if (typeof a === "string" && a.startsWith("--")) {
|
|
137
75
|
const base = a.indexOf("=") === -1 ? a : a.slice(0, a.indexOf("="));
|
|
138
76
|
(out._unknownFlags || (out._unknownFlags = [])).push(base);
|
|
@@ -253,17 +191,12 @@ AGENTS.md Hard Rule #12 and are surfaced as report-only findings.
|
|
|
253
191
|
`);
|
|
254
192
|
}
|
|
255
193
|
|
|
256
|
-
// --- Source modules ----------------------------------------------------
|
|
257
|
-
|
|
258
194
|
/**
|
|
259
|
-
*
|
|
260
|
-
*
|
|
261
|
-
*
|
|
262
|
-
*
|
|
263
|
-
*
|
|
264
|
-
* summary: brief string
|
|
265
|
-
* applyDiff(ctx, diffs) -> Promise<{ updated, errors }>
|
|
266
|
-
* mutates local catalog and writes it
|
|
195
|
+
* Every source module exposes:
|
|
196
|
+
* fetchDiff(ctx, opts) -> { status: "ok" | "unreachable" | "partial",
|
|
197
|
+
* diffs: [{ id, field, before, after, severity? }],
|
|
198
|
+
* errors, summary }
|
|
199
|
+
* applyDiff(ctx, diffs) -> { updated, errors }, having written the catalog
|
|
267
200
|
*/
|
|
268
201
|
|
|
269
202
|
const { discoverNewKev, discoverNewRfcs } = require("./auto-discovery");
|
|
@@ -279,14 +212,9 @@ const KEV_SOURCE = {
|
|
|
279
212
|
const report = await validateAllCves(ctx.cveCatalog, { concurrency: 4 });
|
|
280
213
|
const diffs = [];
|
|
281
214
|
let errors = 0;
|
|
282
|
-
//
|
|
283
|
-
//
|
|
284
|
-
//
|
|
285
|
-
// is far below a real CISA snapshot (a partial CDN response, a momentarily
|
|
286
|
-
// near-empty feed) must not be trusted to de-list curated entries. When the
|
|
287
|
-
// size is known and below the floor we hold ALL de-listings for review,
|
|
288
|
-
// exactly as kevDiffFromCache does; when the size is unknown (no reachable
|
|
289
|
-
// KEV result carried it) we fall back to the per-entry curated-signal guard.
|
|
215
|
+
// A feed that parses but sits far below a real CISA snapshot must not be
|
|
216
|
+
// trusted to de-list curated entries. With no size available, the per-entry
|
|
217
|
+
// curated-signal guard is the fallback.
|
|
290
218
|
let liveFeedSize = null;
|
|
291
219
|
for (const r of report.results) {
|
|
292
220
|
const n = r && r.fetched && r.fetched.sources && r.fetched.sources.kev
|
|
@@ -297,26 +225,16 @@ const KEV_SOURCE = {
|
|
|
297
225
|
for (const r of report.results) {
|
|
298
226
|
if (r.status === "unreachable") errors++;
|
|
299
227
|
for (const d of r.discrepancies || []) {
|
|
300
|
-
// Every KEV field the validator compares, not just the flag and its
|
|
301
|
-
// date: a filter naming two of the four leaves the deadline and the
|
|
302
|
-
// ransomware designation reconciling on --from-cache runs alone, which
|
|
303
|
-
// is not the path most operators take.
|
|
304
228
|
if (KEV_RECONCILED_FIELDS.has(d.field)) {
|
|
305
229
|
const diff = { id: r.cve_id, field: d.field, before: d.local, after: d.fetched, severity: d.severity };
|
|
306
|
-
// A designation
|
|
307
|
-
//
|
|
308
|
-
// gone — held for review exactly as a de-listing is.
|
|
230
|
+
// A designation REMOVED against an implausibly small feed is the feed
|
|
231
|
+
// failing to say something, not upstream saying it is gone.
|
|
309
232
|
if (d.field === "known_ransomware_use" && d.local === true && d.fetched === false && !feedComplete) {
|
|
310
233
|
diff.review_only = true;
|
|
311
234
|
diff.note = `Ransomware designation removal held for review: live feed returned only ${liveFeedSize} entries (< ${KEV_FEED_MIN_PLAUSIBLE}), likely incomplete.`;
|
|
312
235
|
}
|
|
313
|
-
//
|
|
314
|
-
//
|
|
315
|
-
// instead of auto-downgrading the entry when EITHER the entry carries
|
|
316
|
-
// strong human-curated exploitation signal OR the live feed is
|
|
317
|
-
// implausibly small. Without this the live path silently de-listed
|
|
318
|
-
// confirmed-exploitation CVEs the cache path would have held back, and
|
|
319
|
-
// a truncated-but-valid feed could de-list every non-curated entry.
|
|
236
|
+
// A de-listing is review-only — applyDiff skips those — when the entry
|
|
237
|
+
// carries curated exploitation signal or the feed is implausibly small.
|
|
320
238
|
if (d.field === "cisa_kev" && d.local === true && d.fetched === false &&
|
|
321
239
|
(!feedComplete || hasCuratedExploitSignal(ctx.cveCatalog && ctx.cveCatalog[r.cve_id]))) {
|
|
322
240
|
diff.review_only = true;
|
|
@@ -344,8 +262,6 @@ const KEV_SOURCE = {
|
|
|
344
262
|
await withCatalogLock(catalogPath, (catalog) => {
|
|
345
263
|
for (const d of diffs) {
|
|
346
264
|
if (d.op === "add") {
|
|
347
|
-
// Auto-discovered new entry. Refuse to overwrite if the entry
|
|
348
|
-
// somehow exists (race condition / stale fixture); skip silently.
|
|
349
265
|
if (catalog[d.id]) continue;
|
|
350
266
|
catalog[d.id] = d.entry;
|
|
351
267
|
added++;
|
|
@@ -355,43 +271,31 @@ const KEV_SOURCE = {
|
|
|
355
271
|
errors.push(`KEV: no local entry for ${d.id}`);
|
|
356
272
|
continue;
|
|
357
273
|
}
|
|
358
|
-
//
|
|
359
|
-
//
|
|
360
|
-
// in the report but NOT auto-applied: the curated flag, factor, score
|
|
361
|
-
// and dates are left intact so a maintainer can confirm a genuine
|
|
362
|
-
// CISA removal vs a transient / incomplete feed before the entry is
|
|
363
|
-
// downgraded. Skip the write here.
|
|
274
|
+
// review_only keeps the flag, factor, score and dates intact until a
|
|
275
|
+
// maintainer confirms a genuine CISA removal.
|
|
364
276
|
if (d.review_only) continue;
|
|
365
277
|
catalog[d.id][d.field] = d.after;
|
|
366
|
-
//
|
|
367
|
-
//
|
|
368
|
-
// rwep_score to equal the factor sum. Writing the flag without the
|
|
369
|
-
// factor + score left entries failing scoring.validate() (stored 45
|
|
370
|
-
// vs computed 70 on the first real KEV listing the refresh applied).
|
|
278
|
+
// The catalog invariant is rwep_score === Σ rwep_factors, so a cisa_kev
|
|
279
|
+
// flip must rewrite the factor and the score with it.
|
|
371
280
|
if (d.field === "cisa_kev") {
|
|
372
281
|
const entry = catalog[d.id];
|
|
373
282
|
if (entry.rwep_factors && typeof entry.rwep_factors === "object") {
|
|
374
283
|
const scoring = require("./scoring");
|
|
375
|
-
// Match the stored factor shape:
|
|
376
|
-
//
|
|
284
|
+
// Match the stored factor shape: a boolean, or the post-weight
|
|
285
|
+
// contribution the catalog norm holds.
|
|
377
286
|
entry.rwep_factors.cisa_kev =
|
|
378
287
|
typeof entry.rwep_factors.cisa_kev === "boolean"
|
|
379
288
|
? !!d.after
|
|
380
289
|
: (d.after ? scoring.RWEP_WEIGHTS.cisa_kev : 0);
|
|
381
290
|
entry.rwep_score = scoring.deriveRwepFromFactors(entry.rwep_factors);
|
|
382
291
|
}
|
|
383
|
-
//
|
|
384
|
-
//
|
|
385
|
-
// The upstream diff producer only emits a cisa_kev_date diff when
|
|
386
|
-
// upstream has a date, which a de-listed CVE no longer does — so
|
|
387
|
-
// nothing else clears it. Drop the now-meaningless date fields here.
|
|
292
|
+
// Nothing else clears the dates: the diff producer emits a
|
|
293
|
+
// cisa_kev_date diff only when upstream HAS a date.
|
|
388
294
|
if (d.after === false) {
|
|
389
295
|
if ("cisa_kev_date" in entry) entry.cisa_kev_date = null;
|
|
390
296
|
if ("cisa_kev_due_date" in entry) entry.cisa_kev_due_date = null;
|
|
391
|
-
//
|
|
392
|
-
//
|
|
393
|
-
// CISA records no ransomware use, which is a claim about a record
|
|
394
|
-
// that is gone, not a fact upstream supplied.
|
|
297
|
+
// Null, not false: false would assert a claim about a record that
|
|
298
|
+
// no longer exists.
|
|
395
299
|
if ("known_ransomware_use" in entry) entry.known_ransomware_use = null;
|
|
396
300
|
}
|
|
397
301
|
}
|
|
@@ -400,8 +304,7 @@ const KEV_SOURCE = {
|
|
|
400
304
|
}
|
|
401
305
|
catalog._meta = catalog._meta || {};
|
|
402
306
|
catalog._meta.last_updated = TODAY;
|
|
403
|
-
// Refresh the in-memory view so later sources in the
|
|
404
|
-
// (sequential or --swarm) see the post-write state.
|
|
307
|
+
// Refresh the in-memory view so later sources in this process see the write.
|
|
405
308
|
ctx.cveCatalog = catalog;
|
|
406
309
|
return catalog;
|
|
407
310
|
});
|
|
@@ -410,9 +313,7 @@ const KEV_SOURCE = {
|
|
|
410
313
|
};
|
|
411
314
|
|
|
412
315
|
/**
|
|
413
|
-
* Cache-mode KEV
|
|
414
|
-
* for existing entries plus discoverNewKev() for entries upstream that
|
|
415
|
-
* aren't in the local catalog. Spill count is surfaced in the summary.
|
|
316
|
+
* Cache-mode KEV drift plus discoverNewKev() for entries the catalog lacks.
|
|
416
317
|
*/
|
|
417
318
|
function kevDiffWithDiscoveryFromCache(ctx) {
|
|
418
319
|
const drift = kevDiffFromCache(ctx);
|
|
@@ -435,21 +336,9 @@ function kevDiffWithDiscoveryFromCache(ctx) {
|
|
|
435
336
|
}
|
|
436
337
|
|
|
437
338
|
/**
|
|
438
|
-
* EPSS publishes score, percentile and date as ONE row,
|
|
439
|
-
* the
|
|
440
|
-
*
|
|
441
|
-
* both shaped like EPSS values, and nothing downstream able to tell them apart
|
|
442
|
-
* from a coherent pair. That is the corruption scripts/check-epss-consistency.js
|
|
443
|
-
* detects, and a detector cannot repair it — as that gate's own header says, the
|
|
444
|
-
* provenance guarantee has to come from the write side.
|
|
445
|
-
*
|
|
446
|
-
* The bug this replaces applied the drift threshold to score and percentile
|
|
447
|
-
* INDEPENDENTLY: a score that moved past the threshold was written while its
|
|
448
|
-
* percentile, having moved less, was left at yesterday's value. So the rule is
|
|
449
|
-
* now that the threshold decides WHETHER an entry is refreshed, never WHICH of
|
|
450
|
-
* its fields is. Once either number has moved, every field comes from that same
|
|
451
|
-
* row — including the one whose own delta is under the threshold, and including
|
|
452
|
-
* a half of the pair the entry was missing entirely.
|
|
339
|
+
* EPSS publishes score, percentile and date as ONE row, so a field written
|
|
340
|
+
* without the others leaves the entry describing two different days. The drift
|
|
341
|
+
* threshold decides WHETHER an entry is refreshed, never WHICH of its fields.
|
|
453
342
|
*
|
|
454
343
|
* `local` is the catalog entry; `fetched` is {score, percentile, date} read from
|
|
455
344
|
* one EPSS row.
|
|
@@ -458,44 +347,22 @@ const EPSS_DRIFT = 0.05;
|
|
|
458
347
|
|
|
459
348
|
function epssTripleDiffs(id, local, fetched, drift) {
|
|
460
349
|
const { score, percentile, date } = fetched;
|
|
461
|
-
// Only a COMPLETE row
|
|
462
|
-
//
|
|
463
|
-
//
|
|
464
|
-
// threshold on that number, and writing it beside the other half from the
|
|
465
|
-
// previous publication is precisely the mixed-publication state this function
|
|
466
|
-
// exists to prevent. At the top, the guard covers a complete local entry and
|
|
467
|
-
// a half-populated one alike.
|
|
468
|
-
//
|
|
469
|
-
// The date is the third member, not decoration. A row with both numbers and
|
|
470
|
-
// no date would write the new numbers while the entry kept its old
|
|
471
|
-
// epss_date — and the regenerated note would then state those numbers "as of"
|
|
472
|
-
// a publication they did not come from, which is the same defect wearing
|
|
473
|
-
// different clothes.
|
|
474
|
-
// Finite, not merely present: the callers build these with Number(), which
|
|
475
|
-
// turns a malformed cached value into NaN rather than null. A NaN would pass
|
|
476
|
-
// a nullish check, ride the repair path, and land in the catalog serialised
|
|
477
|
-
// as null — reintroducing the half-populated entry the repair exists to fix.
|
|
350
|
+
// Only a COMPLETE row produces a coherent triple, the date included — without
|
|
351
|
+
// it the numbers land under the old epss_date. Finite rather than present: the
|
|
352
|
+
// callers use Number(), and a NaN passes a nullish check and serialises null.
|
|
478
353
|
if (!Number.isFinite(score) || !Number.isFinite(percentile) || !date) return [];
|
|
479
354
|
|
|
480
355
|
const scoreMoved = local.epss_score != null && Math.abs(score - local.epss_score) > drift;
|
|
481
356
|
const pctMoved = local.epss_percentile != null && Math.abs(percentile - local.epss_percentile) > drift;
|
|
482
357
|
|
|
483
|
-
//
|
|
484
|
-
//
|
|
485
|
-
//
|
|
486
|
-
// side happens to be stable the entry stays half-populated forever while the
|
|
487
|
-
// consistency gate keeps failing on it. A complete row repairs it regardless
|
|
488
|
-
// of how far anything moved. An entry carrying NO EPSS at all is left alone —
|
|
489
|
-
// that is an absent field, not an incoherent pair.
|
|
358
|
+
// The drift threshold cannot reach a half-populated entry: the absent side has
|
|
359
|
+
// nothing to compare against, so a stable present side leaves it broken. An
|
|
360
|
+
// entry carrying NO EPSS is left alone — an absent field is not an incoherent pair.
|
|
490
361
|
const localIncomplete = (local.epss_score == null) !== (local.epss_percentile == null);
|
|
491
362
|
if (!scoreMoved && !pctMoved && !localIncomplete) return [];
|
|
492
363
|
|
|
493
|
-
// `coherent` records PROVENANCE:
|
|
494
|
-
// complete
|
|
495
|
-
// entry is left describing one publication. The consumer cannot infer that
|
|
496
|
-
// from the fields alone — a complete row whose percentile rounds to the same
|
|
497
|
-
// value legitimately emits the score by itself, which is indistinguishable
|
|
498
|
-
// from a hand-built partial diff set unless the emitter says so.
|
|
364
|
+
// `coherent` records PROVENANCE: whatever subset differs, these diffs came
|
|
365
|
+
// from one complete row. A consumer cannot infer that from the fields alone.
|
|
499
366
|
const out = [];
|
|
500
367
|
const emit = (field, before, after, severity) =>
|
|
501
368
|
out.push({ id, field, before, after, severity, coherent: true });
|
|
@@ -520,10 +387,8 @@ const EPSS_SOURCE = {
|
|
|
520
387
|
let errors = 0;
|
|
521
388
|
for (const r of report.results) {
|
|
522
389
|
if (r.status === "unreachable") errors++;
|
|
523
|
-
// Diff off the FETCHED ROW
|
|
524
|
-
//
|
|
525
|
-
// which is precisely the input that used to let score and percentile be
|
|
526
|
-
// written from different publications — see epssTripleDiffs.
|
|
390
|
+
// Diff off the FETCHED ROW, not r.discrepancies: that list names only the
|
|
391
|
+
// fields differing enough to report, which mixes publications.
|
|
527
392
|
if (!r.fetched?.epss || !r.local) continue;
|
|
528
393
|
diffs.push(...epssTripleDiffs(r.cve_id, r.local, r.fetched.epss, EPSS_DRIFT));
|
|
529
394
|
}
|
|
@@ -548,32 +413,20 @@ const EPSS_SOURCE = {
|
|
|
548
413
|
catalog[d.id][d.field] = d.after;
|
|
549
414
|
catalog[d.id].last_verified = TODAY;
|
|
550
415
|
// An id is coherent only if EVERY diff applied to it declares that
|
|
551
|
-
// provenance
|
|
552
|
-
// note rewrite.
|
|
416
|
+
// provenance.
|
|
553
417
|
touchedFields.set(d.id, (touchedFields.get(d.id) !== false) && d.coherent === true);
|
|
554
418
|
updated++;
|
|
555
419
|
}
|
|
556
|
-
// `epss_note` restates the three fields in prose
|
|
557
|
-
//
|
|
558
|
-
//
|
|
559
|
-
// consistency gate reports as a failure on data that was otherwise
|
|
560
|
-
// refreshed correctly. Rebuild it from the same renderer the gate checks
|
|
561
|
-
// against, and only for entries that already carry one: adding the note
|
|
562
|
-
// to an entry that never had it is a content change, not a refresh.
|
|
420
|
+
// `epss_note` restates the three fields in prose, so moving the numbers
|
|
421
|
+
// without it leaves the entry stating two scores as of two dates. Rebuilt
|
|
422
|
+
// only where a note exists — adding one is a content change.
|
|
563
423
|
for (const [id, coherent] of touchedFields) {
|
|
564
424
|
const e = catalog[id];
|
|
565
425
|
if (typeof e.epss_note !== "string") continue;
|
|
566
426
|
if (typeof e.epss_score !== "number" || typeof e.epss_percentile !== "number") continue;
|
|
567
|
-
//
|
|
568
|
-
//
|
|
569
|
-
//
|
|
570
|
-
// set can move half the pair on its own; rewriting the note from the
|
|
571
|
-
// surviving stale percentile would make a mixed-publication entry read
|
|
572
|
-
// as current and suppress the only signal the consistency gate has for
|
|
573
|
-
// it. Deciding this from WHICH fields changed does not work — a
|
|
574
|
-
// complete row whose percentile rounds to the same value emits the
|
|
575
|
-
// score alone, and skipping it there would strand the note stale
|
|
576
|
-
// forever, because the next refresh sees no diff left to repair it.
|
|
427
|
+
// Only diffs whose emitter vouched for one complete publication: this is
|
|
428
|
+
// exported, and a fixture moving half the pair would rewrite the note
|
|
429
|
+
// from a stale percentile and make a mixed entry read as current.
|
|
577
430
|
if (!coherent) continue;
|
|
578
431
|
e.epss_note = renderEpssNote(e);
|
|
579
432
|
}
|
|
@@ -618,9 +471,7 @@ const NVD_SOURCE = {
|
|
|
618
471
|
const catalogPath = ctx.cvePath || ABS("data/cve-catalog.json");
|
|
619
472
|
await withCatalogLock(catalogPath, (catalog) => {
|
|
620
473
|
for (const d of diffs) {
|
|
621
|
-
// A curator-owned CVSS re-score is
|
|
622
|
-
// applied — the curated value is preserved until a maintainer accepts
|
|
623
|
-
// the upstream delta (symmetric with the KEV review_only path).
|
|
474
|
+
// A curator-owned CVSS re-score is reported, not applied.
|
|
624
475
|
if (d.review_only) continue;
|
|
625
476
|
if (!catalog[d.id]) {
|
|
626
477
|
errors.push(`NVD: no local entry for ${d.id}`);
|
|
@@ -657,8 +508,7 @@ const RFC_SOURCE = {
|
|
|
657
508
|
}
|
|
658
509
|
if (r.status === "drift" && r.discrepancies) {
|
|
659
510
|
for (const msg of r.discrepancies) {
|
|
660
|
-
// The
|
|
661
|
-
// attempt to parse a status drift; fall back to the raw message.
|
|
511
|
+
// The validator returns discrepancies as strings.
|
|
662
512
|
const m = msg.match(/local "([^"]+)" vs Datatracker "([^"]+)"/);
|
|
663
513
|
if (m) {
|
|
664
514
|
diffs.push({ id: r.id, field: "status", before: m[1], after: m[2], severity: "medium" });
|
|
@@ -708,11 +558,8 @@ const RFC_SOURCE = {
|
|
|
708
558
|
};
|
|
709
559
|
|
|
710
560
|
/**
|
|
711
|
-
* Cache-mode RFC
|
|
712
|
-
*
|
|
713
|
-
* Datatracker live for new RFCs in project-relevant working groups.
|
|
714
|
-
* Discovery makes ~30 HTTP calls (one per project WG) per refresh —
|
|
715
|
-
* Datatracker's read budget is generous so this is well within limit.
|
|
561
|
+
* Cache-mode RFC drift plus discoverNewRfcs(), which hits Datatracker live —
|
|
562
|
+
* one HTTP call per working group per refresh.
|
|
716
563
|
*/
|
|
717
564
|
async function rfcDiffWithDiscoveryFromCache(ctx) {
|
|
718
565
|
const drift = rfcDiffFromCache(ctx);
|
|
@@ -776,23 +623,14 @@ const PINS_SOURCE = {
|
|
|
776
623
|
};
|
|
777
624
|
},
|
|
778
625
|
async applyDiff() {
|
|
779
|
-
// Version pins are intentionally not auto-applied.
|
|
780
626
|
return { updated: 0, errors: ["pin bumps are report-only — see Hard Rule #12"] };
|
|
781
627
|
},
|
|
782
628
|
};
|
|
783
629
|
|
|
784
630
|
/**
|
|
785
|
-
*
|
|
786
|
-
*
|
|
787
|
-
*
|
|
788
|
-
* (variable, often days) or NVD (~10 days).
|
|
789
|
-
*
|
|
790
|
-
* Apply path: new CVE IDs from GHSA land in data/cve-catalog.json as
|
|
791
|
-
* DRAFTS (`_auto_imported: true` + `_draft: true`). The strict catalog
|
|
792
|
-
* validator treats drafts as warnings, not errors, so the nightly
|
|
793
|
-
* auto-PR pipeline can ship them without blocking on editorial review.
|
|
794
|
-
* Framework gaps + IoCs + ATLAS/ATT&CK refs require human or AI-assisted
|
|
795
|
-
* synthesis via `exceptd run cve-curation --advisory <id>`.
|
|
631
|
+
* GitHub Advisory Database. New CVE IDs land as drafts (`_auto_imported` +
|
|
632
|
+
* `_draft`), which the strict validator treats as warnings; framework gaps, IoCs
|
|
633
|
+
* and ATLAS/ATT&CK refs need `exceptd run cve-curation --advisory <id>`.
|
|
796
634
|
*/
|
|
797
635
|
const GHSA_SOURCE = {
|
|
798
636
|
name: "ghsa",
|
|
@@ -801,11 +639,6 @@ const GHSA_SOURCE = {
|
|
|
801
639
|
async fetchDiff(ctx) {
|
|
802
640
|
if (ctx.fixtures?.ghsa) return synthesizeFromFixture(ctx, "ghsa");
|
|
803
641
|
if (ctx.cacheDir) {
|
|
804
|
-
// --from-cache is the offline ingest path: every source reads only
|
|
805
|
-
// local cache files, never the network. GHSA has no cache layer, so
|
|
806
|
-
// there is nothing to read offline. Skip it with a structured status
|
|
807
|
-
// instead of falling through to the live api.github.com fetch, which
|
|
808
|
-
// would silently egress on a host the operator believes is isolated.
|
|
809
642
|
return {
|
|
810
643
|
status: "unreachable",
|
|
811
644
|
diffs: [],
|
|
@@ -817,13 +650,8 @@ const GHSA_SOURCE = {
|
|
|
817
650
|
return ghsa.buildDiff(ctx);
|
|
818
651
|
},
|
|
819
652
|
async applyDiff(ctx, diffs) {
|
|
820
|
-
//
|
|
821
|
-
//
|
|
822
|
-
// reported "applied: N updates" while the catalog file gained zero
|
|
823
|
-
// entries. Worse under `--swarm`: KEV's withCatalogLock would re-read
|
|
824
|
-
// catalog from disk INSIDE the lock and overwrite the unflushed
|
|
825
|
-
// in-memory mutations. Route through the same withCatalogLock helper
|
|
826
|
-
// that KEV/EPSS/NVD/RFC use (v0.12.12 concurrency fix).
|
|
653
|
+
// A mutation of ctx.cveCatalog alone never reaches disk, and under --swarm
|
|
654
|
+
// another source's lock re-reads from disk and overwrites it.
|
|
827
655
|
const catalogPath = ctx.cvePath || ABS("data/cve-catalog.json");
|
|
828
656
|
let updated = 0;
|
|
829
657
|
const errors = [];
|
|
@@ -847,18 +675,9 @@ const GHSA_SOURCE = {
|
|
|
847
675
|
};
|
|
848
676
|
|
|
849
677
|
/**
|
|
850
|
-
*
|
|
851
|
-
*
|
|
852
|
-
*
|
|
853
|
-
* package-compromise class that doesn't have a CVE yet — the MAL-*
|
|
854
|
-
* namespace is the canonical key for those (e.g. MAL-2026-3083, the
|
|
855
|
-
* elementary-data PyPI worm).
|
|
856
|
-
*
|
|
857
|
-
* Apply path mirrors GHSA: new entries land in data/cve-catalog.json as
|
|
858
|
-
* drafts (`_auto_imported: true` + `_draft: true`). Catalog key is either
|
|
859
|
-
* the CVE alias (when present) or the OSV id verbatim — preserving the
|
|
860
|
-
* existing CVE-keyed convention while accepting OSV's broader identifier
|
|
861
|
-
* shapes.
|
|
678
|
+
* OSV.dev — MAL-*, Snyk, GHSA, RustSec, Mageia, Go Vuln DB and Ubuntu USN behind
|
|
679
|
+
* one unauthenticated API. Entries land as drafts like GHSA's, keyed by the CVE
|
|
680
|
+
* alias when there is one and by the OSV id verbatim otherwise.
|
|
862
681
|
*/
|
|
863
682
|
const OSV_SOURCE = {
|
|
864
683
|
name: "osv",
|
|
@@ -867,10 +686,6 @@ const OSV_SOURCE = {
|
|
|
867
686
|
async fetchDiff(ctx) {
|
|
868
687
|
if (ctx.fixtures?.osv) return synthesizeFromFixture(ctx, "osv");
|
|
869
688
|
if (ctx.cacheDir) {
|
|
870
|
-
// --from-cache is the offline ingest path. OSV resolves advisories by
|
|
871
|
-
// live id lookup (ctx.osv_ids) and has no cache layer, so skip it with
|
|
872
|
-
// a structured status rather than risk a live osv.dev fetch on a host
|
|
873
|
-
// the operator believes is isolated.
|
|
874
689
|
return {
|
|
875
690
|
status: "unreachable",
|
|
876
691
|
diffs: [],
|
|
@@ -882,9 +697,7 @@ const OSV_SOURCE = {
|
|
|
882
697
|
return osv.buildDiff(ctx);
|
|
883
698
|
},
|
|
884
699
|
async applyDiff(ctx, diffs) {
|
|
885
|
-
//
|
|
886
|
-
// through withCatalogLock so writes actually land on disk and so
|
|
887
|
-
// concurrent --source osv --apply doesn't lose updates.
|
|
700
|
+
// Lock-gated like GHSA: a ctx-only mutation never reaches disk.
|
|
888
701
|
const catalogPath = ctx.cvePath || ABS("data/cve-catalog.json");
|
|
889
702
|
let updated = 0;
|
|
890
703
|
const errors = [];
|
|
@@ -907,22 +720,12 @@ const OSV_SOURCE = {
|
|
|
907
720
|
},
|
|
908
721
|
};
|
|
909
722
|
|
|
910
|
-
//
|
|
911
|
-
// feeds and surfaces CVE IDs not yet in the catalog. Report-only — no
|
|
912
|
-
// auto-catalog mutation. Closes the post-mortem gap on CVE-2026-46333
|
|
913
|
-
// (ssh-keysign-pwn) where the existing NVD-based pollers lagged by 3+ days.
|
|
723
|
+
// Primary advisory feeds (Qualys TRU, RHSA, USN, ZDI), report-only.
|
|
914
724
|
const { ADVISORIES_SOURCE } = require('./source-advisories');
|
|
915
725
|
|
|
916
|
-
//
|
|
917
|
-
//
|
|
918
|
-
//
|
|
919
|
-
// drop that re-broke CVE-2020-17103 without any new ID being assigned).
|
|
920
|
-
// Report-only; consumes the prior advisories run's output. The main()
|
|
921
|
-
// source loop threads the advisories fetchDiff() result onto
|
|
922
|
-
// ctx.advisoriesObservations (preferred) + ctx.advisoriesDiffs (fallback)
|
|
923
|
-
// after the advisories source resolves and before the watcher runs, so the
|
|
924
|
-
// two must be invoked in that order (advisories first). Under --swarm the
|
|
925
|
-
// watcher runs in a second pass after the parallel batch resolves.
|
|
726
|
+
// NEW-CTRL-074: flags poller-diff historical-CVE references as candidate silent
|
|
727
|
+
// regressions. Report-only, and it consumes the advisories run's output through
|
|
728
|
+
// ctx.advisoriesObservations (or ctx.advisoriesDiffs), so advisories runs first.
|
|
926
729
|
const { REGRESSION_WATCHER_SOURCE } = require('./cve-regression-watcher');
|
|
927
730
|
|
|
928
731
|
const ALL_SOURCES = {
|
|
@@ -937,29 +740,11 @@ const ALL_SOURCES = {
|
|
|
937
740
|
'cve-regression-watcher': REGRESSION_WATCHER_SOURCE,
|
|
938
741
|
};
|
|
939
742
|
|
|
940
|
-
//
|
|
941
|
-
//
|
|
942
|
-
//
|
|
943
|
-
// is
|
|
944
|
-
//
|
|
945
|
-
// <cacheDir>/nvd/<cve>.json
|
|
946
|
-
// <cacheDir>/epss/<cve>.json
|
|
947
|
-
// <cacheDir>/rfc/<doc-name>.json
|
|
948
|
-
// <cacheDir>/pins/<owner__repo__releases>.json
|
|
949
|
-
//
|
|
950
|
-
// readCachedJson returns null on miss; callers report it as "unreachable"
|
|
951
|
-
// for that entry rather than failing the whole source.
|
|
952
|
-
|
|
953
|
-
// sha256 of on-disk cache entries is recorded in _index.json at fetch time
|
|
954
|
-
// but was never verified on consume. A coordinated tamper that rewrote
|
|
955
|
-
// e.g. `.cache/upstream/kev/known_exploited_vulnerabilities.json` between
|
|
956
|
-
// prefetch and refresh would silently feed false intelligence into the
|
|
957
|
-
// applied catalog. We now recompute the sha256 inside readCachedJson and
|
|
958
|
-
// refuse on mismatch.
|
|
959
|
-
//
|
|
960
|
-
// The sha256 stored at prefetch time is computed over JSON.stringify(payload)
|
|
961
|
-
// — unindented. The on-disk bytes use JSON.stringify(payload, null, 2)+"\n",
|
|
962
|
-
// so we round-trip parse and re-canonicalize to compute the comparable hash.
|
|
743
|
+
// Reads the prefetch cache lib/prefetch.js writes at <cacheDir>/<source>/<id>.json;
|
|
744
|
+
// a miss returns null, so one entry is unreachable rather than the whole source.
|
|
745
|
+
// Each payload's sha256 is recorded in _index.json at fetch time and verified here.
|
|
746
|
+
// The stored hash is over unindented JSON.stringify(payload) while the on-disk
|
|
747
|
+
// bytes are indented, so the parse round-trip re-canonicalizes before comparing.
|
|
963
748
|
function readCachedJson(cacheDir, source, id, opts) {
|
|
964
749
|
const forceStale = !!(opts && opts.forceStale);
|
|
965
750
|
const safe = id.replace(/[^A-Za-z0-9._-]/g, "_");
|
|
@@ -968,13 +753,9 @@ function readCachedJson(cacheDir, source, id, opts) {
|
|
|
968
753
|
let parsed;
|
|
969
754
|
try { parsed = JSON.parse(fs.readFileSync(p, "utf8")); }
|
|
970
755
|
catch { return null; }
|
|
971
|
-
//
|
|
972
|
-
//
|
|
973
|
-
//
|
|
974
|
-
// signed entry in _index.json. `--force-stale` is the operator escape
|
|
975
|
-
// hatch for pre-v0.12.24 caches that lack the per-entry sha256 records;
|
|
976
|
-
// we proceed without integrity checking but emit a warning so the gap
|
|
977
|
-
// is visible in logs.
|
|
756
|
+
// Every payload on disk has a signed entry in _index.json, so an absent index
|
|
757
|
+
// or a missing entry refuses rather than fails open. `--force-stale` proceeds
|
|
758
|
+
// unverified but warns.
|
|
978
759
|
const indexPath = path.join(cacheDir, "_index.json");
|
|
979
760
|
if (!fs.existsSync(indexPath)) {
|
|
980
761
|
if (forceStale) {
|
|
@@ -1025,9 +806,7 @@ function readCachedJson(cacheDir, source, id, opts) {
|
|
|
1025
806
|
const cryptoMod = require("crypto");
|
|
1026
807
|
const actual = cryptoMod.createHash("sha256").update(JSON.stringify(parsed)).digest("hex");
|
|
1027
808
|
if (expected !== actual) {
|
|
1028
|
-
//
|
|
1029
|
-
// bypass it. An operator who knows the cache is stale can re-prefetch;
|
|
1030
|
-
// an operator whose cache has been tampered should not proceed.
|
|
809
|
+
// A hard tamper signal: `--force-stale` does NOT bypass it.
|
|
1031
810
|
const err = new Error(`cache-integrity: sha256 mismatch for ${source}/${id} (expected ${expected.slice(0, 16)}..., got ${actual.slice(0, 16)}...)`);
|
|
1032
811
|
err._exceptd_cache_integrity = true;
|
|
1033
812
|
err._exceptd_hint = true;
|
|
@@ -1037,11 +816,8 @@ function readCachedJson(cacheDir, source, id, opts) {
|
|
|
1037
816
|
return parsed;
|
|
1038
817
|
}
|
|
1039
818
|
|
|
1040
|
-
//
|
|
1041
|
-
//
|
|
1042
|
-
// description / verification sources. A de-listing of such an entry is far
|
|
1043
|
-
// more likely a transient or incomplete upstream feed than a genuine CISA
|
|
1044
|
-
// removal, so it is surfaced for review rather than auto-applied.
|
|
819
|
+
// Strong human-curated exploitation signal. A de-listing of such an entry is far
|
|
820
|
+
// likelier a transient feed than a genuine CISA removal.
|
|
1045
821
|
function hasCuratedExploitSignal(entry) {
|
|
1046
822
|
if (!entry || typeof entry !== "object") return false;
|
|
1047
823
|
const ae = typeof entry.active_exploitation === "string" ? entry.active_exploitation.toLowerCase() : "";
|
|
@@ -1051,22 +827,15 @@ function hasCuratedExploitSignal(entry) {
|
|
|
1051
827
|
return false;
|
|
1052
828
|
}
|
|
1053
829
|
|
|
1054
|
-
//
|
|
1055
|
-
//
|
|
1056
|
-
// a
|
|
1057
|
-
// same principle as the curated-KEV de-listing guard above. The version-
|
|
1058
|
-
// downgrade guards already suppress a v3.x→v2 regression; this additionally
|
|
1059
|
-
// keeps a *same-version* NVD re-score (e.g. a curated 10.0 the maintainer pinned
|
|
1060
|
-
// dropping to NVD's 9.8) from silently overwriting the curated value. Raw
|
|
1061
|
-
// auto-imported drafts (`_auto_imported: true`) are not yet curated, so NVD is
|
|
1062
|
-
// their source of truth and their CVSS applies normally.
|
|
830
|
+
// An entry is curator-owned — its CVSS hand-verified — unless it carries
|
|
831
|
+
// `_auto_imported: true`. This keeps a same-version NVD re-score from overwriting
|
|
832
|
+
// a curated value, where the version-downgrade guards cannot.
|
|
1063
833
|
function isCuratorOwnedCvss(entry) {
|
|
1064
834
|
return !!entry && typeof entry === "object" && entry._auto_imported !== true;
|
|
1065
835
|
}
|
|
1066
836
|
|
|
1067
|
-
//
|
|
1068
|
-
//
|
|
1069
|
-
// surfaces the upstream delta for a maintainer to accept deliberately.
|
|
837
|
+
// Marks the diff review-only when the local entry is curator-owned: applyDiff
|
|
838
|
+
// preserves the curated value while the report still surfaces the delta.
|
|
1070
839
|
function cvssDiff(id, field, before, after, severity, local) {
|
|
1071
840
|
const d = { id, field, before, after, severity };
|
|
1072
841
|
if (isCuratorOwnedCvss(local)) {
|
|
@@ -1077,18 +846,13 @@ function cvssDiff(id, field, before, after, severity, local) {
|
|
|
1077
846
|
return d;
|
|
1078
847
|
}
|
|
1079
848
|
|
|
1080
|
-
// Below this many entries the
|
|
1081
|
-
//
|
|
1082
|
-
//
|
|
1083
|
-
// a partial download or a tampered cache, and trusting it to de-list curated
|
|
1084
|
-
// entries would silently erase confirmed-exploitation intel. De-listings are
|
|
1085
|
-
// refused wholesale when the feed is implausibly small.
|
|
849
|
+
// Below this many entries the KEV feed is a partial download or a tampered
|
|
850
|
+
// cache, not a CISA snapshot — the real feed carries thousands. De-listings are
|
|
851
|
+
// refused wholesale there.
|
|
1086
852
|
const KEV_FEED_MIN_PLAUSIBLE = 500;
|
|
1087
853
|
|
|
1088
|
-
// The KEV fields both refresh paths reconcile
|
|
1089
|
-
// the cache path cannot cover different subsets
|
|
1090
|
-
// deadline and the ransomware designation went unreconciled while the flag and
|
|
1091
|
-
// its date tracked upstream.
|
|
854
|
+
// The KEV fields both refresh paths reconcile, named once so the live path and
|
|
855
|
+
// the cache path cannot cover different subsets.
|
|
1092
856
|
const KEV_RECONCILED_FIELDS = new Set([
|
|
1093
857
|
"cisa_kev", "cisa_kev_date", "cisa_kev_due_date", "known_ransomware_use",
|
|
1094
858
|
]);
|
|
@@ -1107,20 +871,16 @@ function kevDiffFromCache(ctx) {
|
|
|
1107
871
|
kevSet.add(v.cveID);
|
|
1108
872
|
if (v.dateAdded) kevDates.set(v.cveID, v.dateAdded);
|
|
1109
873
|
if (v.dueDate) kevDue.set(v.cveID, v.dueDate);
|
|
1110
|
-
// Recorded ONLY for a value this code understands
|
|
1111
|
-
//
|
|
1112
|
-
//
|
|
1113
|
-
// propose downgrading a curated true to false on the strength of a field
|
|
1114
|
-
// the feed never carried. Absence is no answer, not a negative answer.
|
|
874
|
+
// Recorded ONLY for a value this code understands: absence is no answer,
|
|
875
|
+
// not a negative one, and a coerced miss would propose downgrading a
|
|
876
|
+
// curated true on a field the feed never carried.
|
|
1115
877
|
const r = typeof v.knownRansomwareCampaignUse === "string"
|
|
1116
878
|
? v.knownRansomwareCampaignUse.trim().toLowerCase() : null;
|
|
1117
879
|
if (r === "known") kevRansom.set(v.cveID, true);
|
|
1118
880
|
else if (r === "unknown") kevRansom.set(v.cveID, false);
|
|
1119
881
|
}
|
|
1120
882
|
}
|
|
1121
|
-
//
|
|
1122
|
-
// First-listings (false→true) still flow — a small feed never invents new
|
|
1123
|
-
// exploitation; only the de-list direction is suppressed.
|
|
883
|
+
// Only the de-list direction is suppressed; a small feed never invents a listing.
|
|
1124
884
|
const feedComplete = kevSet.size >= KEV_FEED_MIN_PLAUSIBLE;
|
|
1125
885
|
const diffs = [];
|
|
1126
886
|
for (const [id, entry] of Object.entries(ctx.cveCatalog)) {
|
|
@@ -1128,13 +888,8 @@ function kevDiffFromCache(ctx) {
|
|
|
1128
888
|
const upstream = kevSet.has(id);
|
|
1129
889
|
if (typeof entry.cisa_kev === "boolean" && entry.cisa_kev !== upstream) {
|
|
1130
890
|
const isDelist = entry.cisa_kev === true && upstream === false;
|
|
1131
|
-
//
|
|
1132
|
-
//
|
|
1133
|
-
// A de-listing of a curated entry with strong exploitation signal, OR
|
|
1134
|
-
// any de-listing when the feed is implausibly small, is re-tagged as a
|
|
1135
|
-
// review-only diff — surfaced in the report so a maintainer confirms a
|
|
1136
|
-
// genuine CISA removal, but NOT auto-applied (applyDiff skips
|
|
1137
|
-
// review_only diffs, leaving cisa_kev / rwep / dates intact).
|
|
891
|
+
// A de-listing of an entry with strong exploitation signal, or any against
|
|
892
|
+
// an implausibly small feed, is review-only: applyDiff skips those.
|
|
1138
893
|
if (isDelist && (!feedComplete || hasCuratedExploitSignal(entry))) {
|
|
1139
894
|
diffs.push({
|
|
1140
895
|
id,
|
|
@@ -1153,42 +908,29 @@ function kevDiffFromCache(ctx) {
|
|
|
1153
908
|
}
|
|
1154
909
|
}
|
|
1155
910
|
const upDate = kevDates.get(id) || null;
|
|
1156
|
-
//
|
|
1157
|
-
//
|
|
1158
|
-
// so the flag flip and its listing date apply together (the strict
|
|
1159
|
-
// catalog validator requires KEV-listed entries to carry the date).
|
|
911
|
+
// The flag flip and its listing date apply together — the strict validator
|
|
912
|
+
// requires the date on a KEV-listed entry.
|
|
1160
913
|
if (upDate && (entry.cisa_kev_date || null) !== upDate) {
|
|
1161
914
|
diffs.push({ id, field: "cisa_kev_date", before: entry.cisa_kev_date ?? null, after: upDate, severity: "low" });
|
|
1162
915
|
}
|
|
1163
916
|
|
|
1164
|
-
// CISA edits a listing after publishing it
|
|
1165
|
-
//
|
|
1166
|
-
//
|
|
1167
|
-
// fields held then, indefinitely, while every other KEV field tracked the
|
|
1168
|
-
// feed. Only entries upstream still lists are considered; a de-listing
|
|
1169
|
-
// clears both fields through the cisa_kev branch above.
|
|
917
|
+
// CISA edits a listing after publishing it, moving the deadline and the
|
|
918
|
+
// ransomware designation. Only entries upstream still lists are considered —
|
|
919
|
+
// a de-listing clears both fields through the cisa_kev branch above.
|
|
1170
920
|
if (upstream) {
|
|
1171
921
|
const upDue = kevDue.get(id) || null;
|
|
1172
|
-
// The deadline is rendered into remediation output and regulator drafts
|
|
1173
|
-
// so a stale one states a date CISA does not require. It is not decoration
|
|
1174
|
-
// and its severity says so.
|
|
922
|
+
// The deadline is rendered into remediation output and regulator drafts.
|
|
1175
923
|
if (upDue && (entry.cisa_kev_due_date || null) !== upDue) {
|
|
1176
924
|
diffs.push({ id, field: "cisa_kev_due_date", before: entry.cisa_kev_due_date ?? null, after: upDue, severity: "medium" });
|
|
1177
925
|
}
|
|
1178
926
|
const upRansom = kevRansom.has(id) ? kevRansom.get(id) : null;
|
|
1179
|
-
//
|
|
1180
|
-
//
|
|
1181
|
-
//
|
|
1182
|
-
// is the wrong answer rather than no answer. For a KEV-listed CVE the
|
|
1183
|
-
// value is a restatement of the feed and carries no curator judgment, so
|
|
1184
|
-
// it is filled from upstream in both directions and the field becomes
|
|
1185
|
-
// total across KEV-listed entries instead of sometimes-absent.
|
|
927
|
+
// Consumers read a missing boolean as false, so an omission reads as "no
|
|
928
|
+
// ransomware association" — the wrong answer rather than no answer. On a
|
|
929
|
+
// KEV-listed CVE the value restates the feed, so it fills both directions.
|
|
1186
930
|
const localRansom = typeof entry.known_ransomware_use === "boolean" ? entry.known_ransomware_use : null;
|
|
1187
931
|
if (upRansom !== null && localRansom !== upRansom) {
|
|
1188
|
-
//
|
|
1189
|
-
//
|
|
1190
|
-
// implausibly small feed is the feed failing to tell us anything. Hold
|
|
1191
|
-
// the removal for review rather than erasing a curated designation.
|
|
932
|
+
// An ADDED designation is upstream saying something new; one REMOVED
|
|
933
|
+
// against an implausibly small feed is the feed saying nothing.
|
|
1192
934
|
const removing = localRansom === true && upRansom === false;
|
|
1193
935
|
const d = { id, field: "known_ransomware_use", before: localRansom, after: upRansom, severity: "medium" };
|
|
1194
936
|
if (removing && !feedComplete) {
|
|
@@ -1210,11 +952,9 @@ function epssDiffFromCache(ctx) {
|
|
|
1210
952
|
for (const id of cves) {
|
|
1211
953
|
const payload = readCachedJson(ctx.cacheDir, "epss", id, { forceStale: ctx.forceStale });
|
|
1212
954
|
if (!payload) { errors++; continue; }
|
|
1213
|
-
// Match the
|
|
1214
|
-
//
|
|
1215
|
-
//
|
|
1216
|
-
// single-row fallback ONLY when that row carries no cve key (a keyless
|
|
1217
|
-
// legacy payload), never a row naming a different CVE.
|
|
955
|
+
// Match the row by id: a blanket `|| data[0]` fallback attributes another
|
|
956
|
+
// CVE's score to this id. The single-row fallback holds only for a keyless
|
|
957
|
+
// payload, never for a row naming a different CVE.
|
|
1218
958
|
let row = (payload.data || []).find((r) => r?.cve === id);
|
|
1219
959
|
if (!row && (payload.data || []).length === 1 && (payload.data || [])[0]?.cve == null) row = (payload.data || [])[0];
|
|
1220
960
|
if (!row) continue;
|
|
@@ -1236,27 +976,20 @@ function nvdDiffFromCache(ctx) {
|
|
|
1236
976
|
for (const id of cves) {
|
|
1237
977
|
const payload = readCachedJson(ctx.cacheDir, "nvd", id, { forceStale: ctx.forceStale });
|
|
1238
978
|
if (!payload) { errors++; continue; }
|
|
1239
|
-
// Resolve the NVD vuln by id, not by position
|
|
1240
|
-
//
|
|
1241
|
-
// the cache entry keyed under `id` held another CVE's response.
|
|
979
|
+
// Resolve the NVD vuln by id, not by position: vulnerabilities[0] attributes
|
|
980
|
+
// another CVE's CVSS to this id.
|
|
1242
981
|
const vulnCves = (payload.vulnerabilities || []).map((v) => v?.cve).filter(Boolean);
|
|
1243
982
|
let vuln = vulnCves.find((c) => c.id && String(c.id).toUpperCase() === id.toUpperCase());
|
|
1244
|
-
//
|
|
1245
|
-
//
|
|
1246
|
-
// use it. A record whose id names a DIFFERENT cve is still rejected.
|
|
983
|
+
// A single record whose cve carries no id cannot be a mismatch — the
|
|
984
|
+
// id-keyed cache file is the binding. One naming a DIFFERENT cve is not.
|
|
1247
985
|
if (!vuln && vulnCves.length === 1 && vulnCves[0].id == null) vuln = vulnCves[0];
|
|
1248
986
|
if (!vuln) continue;
|
|
1249
|
-
// Prefer the newest CVSS version NVD publishes (Primary within that
|
|
1250
|
-
// version), and normalize a bare v2 vector to its canonical prefix.
|
|
1251
987
|
const up = selectNvdCvss(vuln.metrics);
|
|
1252
988
|
if (!up) continue;
|
|
1253
989
|
const local = ctx.cveCatalog[id];
|
|
1254
|
-
// Never regress a curated higher-version CVSS to an older upstream metric
|
|
1255
|
-
// NVD
|
|
1256
|
-
//
|
|
1257
|
-
// selected upstream metric is an older CVSS version than the curated one,
|
|
1258
|
-
// suppress both the score and the vector diff. A same-version drift (a
|
|
1259
|
-
// genuine NVD re-score) still flows through.
|
|
990
|
+
// Never regress a curated higher-version CVSS to an older upstream metric:
|
|
991
|
+
// NVD tags v2 "Primary" over a v3.1 "Secondary" on older CVEs. A same-version
|
|
992
|
+
// drift still flows through.
|
|
1260
993
|
const localVersion = cvssVersionOf(local.cvss_vector);
|
|
1261
994
|
const isDowngrade =
|
|
1262
995
|
up.version != null && localVersion != null && up.version < localVersion;
|
|
@@ -1302,14 +1035,9 @@ function rfcDiffFromCache(ctx) {
|
|
|
1302
1035
|
}
|
|
1303
1036
|
|
|
1304
1037
|
function pinsDiffFromCache(ctx) {
|
|
1305
|
-
// Cache layout under pins/: <owner>__<repo>__releases.json arrays
|
|
1306
|
-
//
|
|
1307
|
-
//
|
|
1308
|
-
// SOURCES.pins (neither project tags releases on GitHub; D3FEND ships
|
|
1309
|
-
// the ontology from d3fend/d3fend-ontology without tagged releases,
|
|
1310
|
-
// and CWE distributes XML from cwe.mitre.org). Pin currency for those
|
|
1311
|
-
// two frameworks is monitored via lib/upstream-check.js against their
|
|
1312
|
-
// canonical mitre.org endpoints, not through the prefetch cache.
|
|
1038
|
+
// Cache layout under pins/: <owner>__<repo>__releases.json arrays, so only
|
|
1039
|
+
// repos publishing via GitHub Releases appear. D3FEND and CWE tag none;
|
|
1040
|
+
// lib/upstream-check.js monitors their pin currency against mitre.org instead.
|
|
1313
1041
|
const PIN_REPOS = {
|
|
1314
1042
|
atlas_version: "mitre-atlas__atlas-data__releases",
|
|
1315
1043
|
attack_version: "mitre-attack__attack-stix-data__releases",
|
|
@@ -1346,12 +1074,8 @@ function pinsDiffFromCache(ctx) {
|
|
|
1346
1074
|
return { status, diffs, errors, summary: `${diffs.length} pin drifts (from cache); ${errors} missing entries` };
|
|
1347
1075
|
}
|
|
1348
1076
|
|
|
1349
|
-
// --- Fixture-mode helper ----------------------------------------------
|
|
1350
|
-
|
|
1351
1077
|
function synthesizeFromFixture(ctx, sourceName) {
|
|
1352
|
-
//
|
|
1353
|
-
// { diffs: [...], errors: 0, summary: "..." }
|
|
1354
|
-
// tests/fixtures/refresh/<sourceName>.json drives this path.
|
|
1078
|
+
// tests/fixtures/refresh/<sourceName>.json holds { diffs, errors, summary }.
|
|
1355
1079
|
const fp = path.join(ctx.fixtures.dir, `${sourceName}.json`);
|
|
1356
1080
|
if (!fs.existsSync(fp)) {
|
|
1357
1081
|
return { status: "ok", diffs: [], errors: 0, summary: `${sourceName}: no fixture` };
|
|
@@ -1365,8 +1089,6 @@ function synthesizeFromFixture(ctx, sourceName) {
|
|
|
1365
1089
|
};
|
|
1366
1090
|
}
|
|
1367
1091
|
|
|
1368
|
-
// --- IO helpers --------------------------------------------------------
|
|
1369
|
-
|
|
1370
1092
|
function loadCtx(opts) {
|
|
1371
1093
|
const cvePath = resolveCatalogPath(opts);
|
|
1372
1094
|
const ctx = {
|
|
@@ -1378,28 +1100,17 @@ function loadCtx(opts) {
|
|
|
1378
1100
|
d3fendCatalog: JSON.parse(fs.readFileSync(ABS("data/d3fend-catalog.json"), "utf8")),
|
|
1379
1101
|
fixtures: null,
|
|
1380
1102
|
cacheDir: null,
|
|
1381
|
-
//
|
|
1382
|
-
// GHSA + OSV source modules (lib/source-ghsa.js, lib/source-osv.js)
|
|
1383
|
-
// branch on it and refuse network egress.
|
|
1103
|
+
// Reaches lib/source-ghsa.js and lib/source-osv.js, which refuse egress on it.
|
|
1384
1104
|
airGap: !!(opts && opts.airGap) || process.env.EXCEPTD_AIR_GAP === "1",
|
|
1385
|
-
//
|
|
1386
|
-
// integrity refusals to warnings when an operator explicitly opts out.
|
|
1105
|
+
// Lets readCachedJson downgrade cache-integrity refusals to warnings.
|
|
1387
1106
|
forceStale: !!(opts && opts.forceStale),
|
|
1388
|
-
//
|
|
1389
|
-
//
|
|
1390
|
-
// we lack" against "what have we got that upstream has since changed" — and
|
|
1391
|
-
// an operator correcting the second does not necessarily want the first,
|
|
1392
|
-
// because a discovered entry arrives as a draft that has to be curated
|
|
1393
|
-
// before it can ship. Bundling them forces a choice between leaving known
|
|
1394
|
-
// fields stale and importing work that is not ready.
|
|
1107
|
+
// Discovery yields drafts that need curation before they can ship, so
|
|
1108
|
+
// drift-only reconciles shipped entries without pulling that work in.
|
|
1395
1109
|
driftOnly: !!(opts && opts.driftOnly),
|
|
1396
1110
|
};
|
|
1397
1111
|
if (opts.fromFixture) {
|
|
1398
|
-
// `--from-fixture` injects
|
|
1399
|
-
//
|
|
1400
|
-
// caller forge KEV / NVD / EPSS / pin diffs into the applied catalog.
|
|
1401
|
-
// Gate the flag behind EXCEPTD_TEST_HARNESS=1 so it only activates in
|
|
1402
|
-
// explicit test contexts.
|
|
1112
|
+
// `--from-fixture` injects payloads as if they were live upstream responses,
|
|
1113
|
+
// so outside the harness it would forge diffs into the applied catalog.
|
|
1403
1114
|
if (process.env.EXCEPTD_TEST_HARNESS !== "1") {
|
|
1404
1115
|
const err = new Error(
|
|
1405
1116
|
`refresh: --from-fixture is disabled outside the test harness.\n` +
|
|
@@ -1412,14 +1123,7 @@ function loadCtx(opts) {
|
|
|
1412
1123
|
}
|
|
1413
1124
|
const fixtureDir = path.resolve(opts.fromFixture);
|
|
1414
1125
|
ctx.fixtures = { dir: fixtureDir, kev: true, epss: true, nvd: true, rfc: true, pins: true, ghsa: true, osv: true };
|
|
1415
|
-
//
|
|
1416
|
-
// ctx.fixtures.advisories so the advisory poller (Qualys / RHSA / USN /
|
|
1417
|
-
// ZDI / kernel.org / oss-security / JFrog / CISA) uses frozen content
|
|
1418
|
-
// instead of falling through to live RSS. Prior to this load, two
|
|
1419
|
-
// back-to-back fixture-mode runs (e.g. sequential vs `--swarm`) hit
|
|
1420
|
-
// the real-world feeds at different moments and diverged on Pwn2Own /
|
|
1421
|
-
// Trend Micro / ZDI advisories rotated within the window — surfaced as
|
|
1422
|
-
// a CI flake on macOS runners where the test took longer to complete.
|
|
1126
|
+
// Frozen advisories keep the poller off live RSS, which rotates between runs.
|
|
1423
1127
|
const advFixPath = path.join(fixtureDir, "advisories.json");
|
|
1424
1128
|
if (fs.existsSync(advFixPath)) {
|
|
1425
1129
|
try {
|
|
@@ -1437,12 +1141,6 @@ function loadCtx(opts) {
|
|
|
1437
1141
|
const abs = path.resolve(opts.fromCache);
|
|
1438
1142
|
ctx.cacheDir = abs;
|
|
1439
1143
|
if (!fs.existsSync(abs)) {
|
|
1440
|
-
// Operators following the air-gap workflow hit this with an unhelpful
|
|
1441
|
-
// "path does not exist" stack trace. The cache is populated by
|
|
1442
|
-
// `exceptd refresh --prefetch` (which routes to prefetch) — NOT by
|
|
1443
|
-
// `--no-network`, which is the report-only dry run that writes nothing.
|
|
1444
|
-
// Tell them exactly that, and emit a structured JSON error to stderr
|
|
1445
|
-
// instead of a fatal stack trace.
|
|
1446
1144
|
const err = new Error(
|
|
1447
1145
|
`refresh: --from-cache path does not exist: ${abs}\n` +
|
|
1448
1146
|
`Hint: the cache is populated by running \`exceptd refresh --prefetch\` ` +
|
|
@@ -1452,12 +1150,8 @@ function loadCtx(opts) {
|
|
|
1452
1150
|
err._exceptd_hint = true;
|
|
1453
1151
|
throw err;
|
|
1454
1152
|
}
|
|
1455
|
-
//
|
|
1456
|
-
//
|
|
1457
|
-
// cache whose sidecar signature does not verify against keys/public.pem,
|
|
1458
|
-
// unless the operator explicitly accepts the risk via --force-stale.
|
|
1459
|
-
// A missing sidecar (cache prefetched on a host without the signing
|
|
1460
|
-
// keypair) is treated identically: same refusal, same override.
|
|
1153
|
+
// The cache is Ed25519-signed at prefetch time; a sidecar that does not
|
|
1154
|
+
// verify against keys/public.pem, or is missing, is refused alike.
|
|
1461
1155
|
try {
|
|
1462
1156
|
const { verifyIndexSignature } = require("./prefetch.js");
|
|
1463
1157
|
const sigResult = verifyIndexSignature(abs);
|
|
@@ -1474,9 +1168,7 @@ function loadCtx(opts) {
|
|
|
1474
1168
|
}
|
|
1475
1169
|
} catch (e) {
|
|
1476
1170
|
if (e && e._exceptd_hint) throw e;
|
|
1477
|
-
//
|
|
1478
|
-
// refusal rather than fail-open. Operators on --force-stale still
|
|
1479
|
-
// pass through.
|
|
1171
|
+
// A loader error is a hard refusal, not a fail-open.
|
|
1480
1172
|
if (!opts.forceStale) {
|
|
1481
1173
|
const err = new Error(
|
|
1482
1174
|
`refresh: --from-cache signature verifier unavailable: ${e && e.message}.\n` +
|
|
@@ -1488,10 +1180,8 @@ function loadCtx(opts) {
|
|
|
1488
1180
|
throw err;
|
|
1489
1181
|
}
|
|
1490
1182
|
}
|
|
1491
|
-
//
|
|
1492
|
-
//
|
|
1493
|
-
// to be misleading than helpful (KEV gains entries weekly; EPSS shifts
|
|
1494
|
-
// daily). --force-stale overrides for genuine air-gap workflows.
|
|
1183
|
+
// A cache whose freshest fetched_at is older than 7 days is refused: KEV
|
|
1184
|
+
// gains entries weekly and EPSS shifts daily.
|
|
1495
1185
|
try {
|
|
1496
1186
|
const idxPath = path.join(abs, "_index.json");
|
|
1497
1187
|
if (fs.existsSync(idxPath)) {
|
|
@@ -1521,7 +1211,6 @@ function loadCtx(opts) {
|
|
|
1521
1211
|
}
|
|
1522
1212
|
} catch (e) {
|
|
1523
1213
|
if (e && e._exceptd_hint) throw e;
|
|
1524
|
-
// Index parse error — bubble up as a hint
|
|
1525
1214
|
const err = new Error(`refresh: --from-cache _index.json unreadable: ${e && e.message}`);
|
|
1526
1215
|
err._exceptd_hint = true;
|
|
1527
1216
|
err._exceptd_exit_code = 4;
|
|
@@ -1531,18 +1220,14 @@ function loadCtx(opts) {
|
|
|
1531
1220
|
return ctx;
|
|
1532
1221
|
}
|
|
1533
1222
|
|
|
1534
|
-
//
|
|
1535
|
-
// tmp
|
|
1536
|
-
//
|
|
1537
|
-
//
|
|
1538
|
-
// in full or the new content in full — never a half-written buffer. The
|
|
1539
|
-
// tmp name carries pid + random so two writers in the same process (e.g.
|
|
1540
|
-
// worker threads) never collide on the same scratch path.
|
|
1223
|
+
// Every persisted JSON write goes through here. fs.renameSync is atomic for a
|
|
1224
|
+
// `.tmp.<pid>.<rand>` beside the target, so a concurrent reader sees the old
|
|
1225
|
+
// file or the new one, never a half-written buffer; the pid + random keeps two
|
|
1226
|
+
// writers in one process off the same scratch path.
|
|
1541
1227
|
function writeJsonAtomic(p, obj) {
|
|
1542
1228
|
const tmpPath = `${p}.tmp.${process.pid}.${Math.random().toString(36).slice(2, 10)}`;
|
|
1543
|
-
//
|
|
1544
|
-
//
|
|
1545
|
-
// matching helper in lib/cve-curation.js for the rationale.
|
|
1229
|
+
// fsync before the rename so a power loss between the two leaves the durable
|
|
1230
|
+
// destination intact. Matching helper in lib/cve-curation.js.
|
|
1546
1231
|
const fd = fs.openSync(tmpPath, 'w');
|
|
1547
1232
|
try {
|
|
1548
1233
|
fs.writeSync(fd, JSON.stringify(obj, null, 2) + "\n", 0, "utf8");
|
|
@@ -1558,39 +1243,18 @@ function writeJsonAtomic(p, obj) {
|
|
|
1558
1243
|
}
|
|
1559
1244
|
}
|
|
1560
1245
|
|
|
1561
|
-
// Back-compat alias — exported callers and historical sites still reference
|
|
1562
|
-
// writeJson. Atomic by default; never the unsafe direct-write form.
|
|
1563
1246
|
function writeJson(p, obj) {
|
|
1564
1247
|
writeJsonAtomic(p, obj);
|
|
1565
1248
|
}
|
|
1566
1249
|
|
|
1567
1250
|
/**
|
|
1568
|
-
*
|
|
1251
|
+
* Lockfile-gated read-modify-write for a JSON catalog: a sidecar lockfile
|
|
1252
|
+
* (O_EXCL via `flag: 'wx'`) serializes the triple, so two concurrent
|
|
1253
|
+
* `--advisory <id> --apply` processes cannot drop one another's CVE.
|
|
1569
1254
|
*
|
|
1570
|
-
*
|
|
1571
|
-
*
|
|
1572
|
-
*
|
|
1573
|
-
* the second write overwrote the first, silently dropping one CVE. The fix
|
|
1574
|
-
* is a sidecar lockfile (created with O_EXCL via `flag: 'wx'`) that
|
|
1575
|
-
* serializes the read-mutate-write triple. The mutator receives the
|
|
1576
|
-
* current-on-disk catalog (re-read inside the lock, NOT a stale in-memory
|
|
1577
|
-
* copy from before lock acquisition) and returns it after mutation; the
|
|
1578
|
-
* helper then writes atomically via writeJsonAtomic.
|
|
1579
|
-
*
|
|
1580
|
-
* Stale-lock recovery: if a holder crashes without unlinking, the lockfile
|
|
1581
|
-
* persists. After backoff, if the lockfile's mtime is older than 30s we
|
|
1582
|
-
* treat it as orphaned and unlink it before retrying. 30s is well past any
|
|
1583
|
-
* legitimate single-CVE apply (sub-second on modern disks).
|
|
1584
|
-
*
|
|
1585
|
-
* On acquisition failure after N retries, we throw — better than silently
|
|
1586
|
-
* proceeding without the lock.
|
|
1587
|
-
*
|
|
1588
|
-
* @param {string} catalogPath path to the JSON catalog to lock
|
|
1589
|
-
* @param {(catalog: object) => object | Promise<object>} mutator
|
|
1590
|
-
* receives current-on-disk catalog, returns mutated catalog. May be
|
|
1591
|
-
* async. The return value is what gets written; if it returns
|
|
1592
|
-
* undefined, the in-place mutation of the passed-in catalog is used.
|
|
1593
|
-
* @returns {Promise<{ wrote: boolean, result: any }>}
|
|
1255
|
+
* The mutator, which may be async, receives the catalog as re-read INSIDE the
|
|
1256
|
+
* lock — never a copy from before acquisition — and returns the object to
|
|
1257
|
+
* write; returning undefined writes its in-place mutation instead.
|
|
1594
1258
|
*/
|
|
1595
1259
|
async function withCatalogLock(catalogPath, mutator) {
|
|
1596
1260
|
const lockPath = `${catalogPath}.lock`;
|
|
@@ -1603,20 +1267,12 @@ async function withCatalogLock(catalogPath, mutator) {
|
|
|
1603
1267
|
acquired = true;
|
|
1604
1268
|
break;
|
|
1605
1269
|
} catch (e) {
|
|
1606
|
-
// EEXIST is the POSIX signal another process holds the lock
|
|
1607
|
-
//
|
|
1608
|
-
// when the holder is mid-unlink). Treat both as "lock held, back off."
|
|
1270
|
+
// EEXIST is the POSIX signal that another process holds the lock; Windows
|
|
1271
|
+
// raises EPERM when the holder is mid-unlink. Both mean held, so back off.
|
|
1609
1272
|
if (e.code !== "EEXIST" && e.code !== "EPERM") throw e;
|
|
1610
|
-
// PID
|
|
1611
|
-
//
|
|
1612
|
-
//
|
|
1613
|
-
// immediately rather than waiting STALE_LOCK_MS for the mtime gate
|
|
1614
|
-
// to expire. EPERM (holder alive, different user) is treated as
|
|
1615
|
-
// "alive, keep waiting." The mtime gate remains as a belt-and-
|
|
1616
|
-
// suspenders for cases where the lockfile content is missing /
|
|
1617
|
-
// malformed / belongs to a recycled PID. Matches the PID pattern in
|
|
1618
|
-
// orchestrator/index.js _acquireWatchLock and
|
|
1619
|
-
// lib/playbook-runner.js pidAlive().
|
|
1273
|
+
// Probe the holder PID before falling back to mtime: ESRCH means dead, so
|
|
1274
|
+
// reclaim now instead of waiting out STALE_LOCK_MS; EPERM means alive under
|
|
1275
|
+
// another user. Same pattern as orchestrator/index.js _acquireWatchLock.
|
|
1620
1276
|
let reclaimedByPid = false;
|
|
1621
1277
|
try {
|
|
1622
1278
|
const raw = fs.readFileSync(lockPath, "utf8").trim();
|
|
@@ -1634,8 +1290,7 @@ async function withCatalogLock(catalogPath, mutator) {
|
|
|
1634
1290
|
}
|
|
1635
1291
|
} catch {} // unreadable lockfile — proceed to mtime fallback
|
|
1636
1292
|
if (reclaimedByPid) continue;
|
|
1637
|
-
//
|
|
1638
|
-
// us waiting MAX_RETRIES * backoff before we recover.
|
|
1293
|
+
// Check for a dead holder before sleeping out MAX_RETRIES * backoff.
|
|
1639
1294
|
try {
|
|
1640
1295
|
const stat = fs.statSync(lockPath);
|
|
1641
1296
|
if (Date.now() - stat.mtimeMs > STALE_LOCK_MS) {
|
|
@@ -1661,11 +1316,8 @@ async function withCatalogLock(catalogPath, mutator) {
|
|
|
1661
1316
|
}
|
|
1662
1317
|
|
|
1663
1318
|
function chosenSources(opts) {
|
|
1664
|
-
// Flag-absent
|
|
1665
|
-
//
|
|
1666
|
-
// trims to nothing like `--source ","`) is an operator error, not a
|
|
1667
|
-
// silent run-everything: refuse and list the valid names so the typo is
|
|
1668
|
-
// visible rather than masquerading as a full refresh.
|
|
1319
|
+
// Flag-absent means "all sources"; flag-present-but-empty (`--source ""`,
|
|
1320
|
+
// `--source ","`) is an operator error, not a silent run-everything.
|
|
1669
1321
|
if (opts.source == null) return Object.values(ALL_SOURCES);
|
|
1670
1322
|
const names = opts.source.split(",").map((s) => s.trim()).filter(Boolean);
|
|
1671
1323
|
if (names.length === 0) {
|
|
@@ -1676,10 +1328,8 @@ function chosenSources(opts) {
|
|
|
1676
1328
|
const out = [];
|
|
1677
1329
|
for (const n of names) {
|
|
1678
1330
|
if (!ALL_SOURCES[n]) {
|
|
1679
|
-
//
|
|
1680
|
-
//
|
|
1681
|
-
// main().catch() surface the error through the standard channel and
|
|
1682
|
-
// exit code via process.exitCode + natural event-loop drain.
|
|
1331
|
+
// Throw rather than process.exit(): an exit can truncate a buffered stdout
|
|
1332
|
+
// write, so main().catch() sets process.exitCode and the loop drains.
|
|
1683
1333
|
const err = new Error(`refresh-external: unknown source "${n}". Valid: ${Object.keys(ALL_SOURCES).join(", ")}`);
|
|
1684
1334
|
err._exceptd_unknown_source = true;
|
|
1685
1335
|
throw err;
|
|
@@ -1690,21 +1340,12 @@ function chosenSources(opts) {
|
|
|
1690
1340
|
}
|
|
1691
1341
|
|
|
1692
1342
|
/**
|
|
1693
|
-
*
|
|
1694
|
-
*
|
|
1695
|
-
* or
|
|
1696
|
-
* exceptd refresh --advisory GHSA-xxxx-xxxx-xxxx --apply
|
|
1697
|
-
*
|
|
1698
|
-
* Tool fetches from GHSA (covers npm, PyPI, etc.), normalizes to the
|
|
1699
|
-
* exceptd catalog draft shape, and either prints the seed (default) or
|
|
1700
|
-
* writes it to data/cve-catalog.json (--apply). Always exits non-zero
|
|
1701
|
-
* when a draft is produced, signaling that editorial review is needed.
|
|
1343
|
+
* Seeds one catalog entry from an advisory id via GHSA or OSV, printing the
|
|
1344
|
+
* draft or writing it (--apply). A produced draft exits 3 — review pending.
|
|
1702
1345
|
*/
|
|
1703
1346
|
async function seedSingleAdvisory(opts) {
|
|
1704
1347
|
const id = opts.advisory;
|
|
1705
|
-
//
|
|
1706
|
-
// through source-osv. CVE-* and GHSA-* keep routing through GHSA because
|
|
1707
|
-
// GHSA carries richer field coverage for those identifier shapes.
|
|
1348
|
+
// OSV-native ids route through source-osv; CVE-* and GHSA-* go to GHSA.
|
|
1708
1349
|
const osvMod = require("./source-osv");
|
|
1709
1350
|
const useOsv = osvMod.isOsvId(id) && !/^GHSA-/i.test(id);
|
|
1710
1351
|
const ghsa = require("./source-ghsa");
|
|
@@ -1712,18 +1353,12 @@ async function seedSingleAdvisory(opts) {
|
|
|
1712
1353
|
const sourceName = useOsv ? "osv" : "ghsa";
|
|
1713
1354
|
const fixtureEnv = useOsv ? "EXCEPTD_OSV_FIXTURE" : "EXCEPTD_GHSA_FIXTURE";
|
|
1714
1355
|
|
|
1715
|
-
//
|
|
1716
|
-
// into the fetch. Previously this passed {} and dropped --air-gap, so
|
|
1717
|
-
// `refresh --advisory <id> --air-gap` egressed to the network — an air-gap
|
|
1718
|
-
// violation. Both source modules refuse (no fixture) when airGap is set.
|
|
1356
|
+
// The air-gap disposition must reach the fetch; dropping it here egresses.
|
|
1719
1357
|
const airGap = !!opts.airGap || process.env.EXCEPTD_AIR_GAP === "1";
|
|
1720
1358
|
|
|
1721
1359
|
let result = await sourceMod.fetchAdvisoryById(id, { airGap });
|
|
1722
|
-
//
|
|
1723
|
-
//
|
|
1724
|
-
// When GHSA returns 404 specifically, retry through OSV's /v1/vulns/{id}
|
|
1725
|
-
// — OSV indexes CVE ids as primary keys. If both 404, surface a combined
|
|
1726
|
-
// error message so operators know both sources were tried before failing.
|
|
1360
|
+
// A CVE-* id can have an OSV record before GHSA publishes one, so a GHSA 404
|
|
1361
|
+
// retries through OSV's /v1/vulns/{id}. When both 404 the error names both.
|
|
1727
1362
|
let fallbackSourceUsed = null;
|
|
1728
1363
|
if (!result.ok && !useOsv && /^CVE-/i.test(id) && /HTTP 404/.test(result.error || "")) {
|
|
1729
1364
|
const fallback = await osvMod.fetchAdvisoryById(id, { airGap });
|
|
@@ -1731,7 +1366,6 @@ async function seedSingleAdvisory(opts) {
|
|
|
1731
1366
|
result = fallback;
|
|
1732
1367
|
fallbackSourceUsed = "osv";
|
|
1733
1368
|
} else if (/HTTP 404/.test(fallback.error || "") || /not in fixture/.test(fallback.error || "")) {
|
|
1734
|
-
// Both sources tried, both 404 — combine the error message.
|
|
1735
1369
|
const combined = { ok: false, verb: "refresh", error: `--advisory ${id}: not found in GHSA or OSV (GHSA: ${result.error}; OSV: ${fallback.error})`, source: "offline", routed_to: "ghsa+osv", hint: `Both GHSA and OSV.dev returned 404 for ${id}. Verify the CVE id (CVE-YYYY-NNNN) and that an advisory record exists upstream.` };
|
|
1736
1370
|
if (opts.json) process.stdout.write(JSON.stringify(combined) + "\n");
|
|
1737
1371
|
else process.stderr.write(`[refresh --advisory] ${combined.error}\n hint: ${combined.hint}\n`);
|
|
@@ -1746,8 +1380,7 @@ async function seedSingleAdvisory(opts) {
|
|
|
1746
1380
|
process.exitCode = 2;
|
|
1747
1381
|
return;
|
|
1748
1382
|
}
|
|
1749
|
-
//
|
|
1750
|
-
// here on — the advisory shape is OSV's, not GHSA's.
|
|
1383
|
+
// After an OSV fallback the advisory shape is OSV's, so normalize through it.
|
|
1751
1384
|
const effectiveMod = fallbackSourceUsed === "osv" ? osvMod : sourceMod;
|
|
1752
1385
|
const effectiveName = fallbackSourceUsed === "osv" ? "osv" : sourceName;
|
|
1753
1386
|
const advisory = result.advisories[0];
|
|
@@ -1769,8 +1402,6 @@ async function seedSingleAdvisory(opts) {
|
|
|
1769
1402
|
const cveId = Object.keys(normalized)[0];
|
|
1770
1403
|
|
|
1771
1404
|
if (!opts.apply) {
|
|
1772
|
-
// Print the draft to stdout — operator pipes to jq / inspects /
|
|
1773
|
-
// commits manually. Exit 3 = "draft produced, not applied."
|
|
1774
1405
|
const output = {
|
|
1775
1406
|
ok: true,
|
|
1776
1407
|
verb: "refresh",
|
|
@@ -1790,18 +1421,12 @@ async function seedSingleAdvisory(opts) {
|
|
|
1790
1421
|
return;
|
|
1791
1422
|
}
|
|
1792
1423
|
|
|
1793
|
-
// Apply: write to cve-catalog.json with the _auto_imported flag.
|
|
1794
|
-
// v0.12.8: honor --catalog / EXCEPTD_CVE_CATALOG so tests can redirect.
|
|
1795
|
-
// v0.12.12 C1: lock-gated RMW. Without this, two concurrent
|
|
1796
|
-
// `refresh --advisory CVE-A --apply` + `--advisory CVE-B --apply`
|
|
1797
|
-
// processes against the same catalog silently dropped one CVE 1-in-20
|
|
1798
|
-
// trials (read-old → mutate → write-overwrites-sibling-mutation).
|
|
1799
1424
|
const catalogPath = resolveCatalogPath(opts);
|
|
1800
1425
|
let humanCurated = null;
|
|
1801
1426
|
await withCatalogLock(catalogPath, (catalog) => {
|
|
1802
1427
|
if (catalog[cveId] && !catalog[cveId]._auto_imported && !catalog[cveId]._draft) {
|
|
1803
|
-
// Refuse to overwrite a human-curated entry
|
|
1804
|
-
//
|
|
1428
|
+
// Refuse to overwrite a human-curated entry; signalled through the closure
|
|
1429
|
+
// so the structured error emits after the lock releases.
|
|
1805
1430
|
humanCurated = { last_updated: catalog[cveId].last_updated };
|
|
1806
1431
|
return catalog; // unchanged write — idempotent, releases lock
|
|
1807
1432
|
}
|
|
@@ -1831,8 +1456,6 @@ async function seedSingleAdvisory(opts) {
|
|
|
1831
1456
|
process.exitCode = 3;
|
|
1832
1457
|
}
|
|
1833
1458
|
|
|
1834
|
-
// Known --flag base names refresh accepts (operator-facing surface + the
|
|
1835
|
-
// bin-translated aliases). Drives the unknown-flag error message's known list.
|
|
1836
1459
|
const REFRESH_KNOWN_FLAGS = Object.freeze([
|
|
1837
1460
|
"--apply", "--quiet", "--swarm", "--json", "--help", "-h", "--advisory",
|
|
1838
1461
|
"--check-advisories", "--catalog", "--from-cache", "--source", "--from-fixture",
|
|
@@ -1844,14 +1467,13 @@ async function main() {
|
|
|
1844
1467
|
const opts = parseArgs(process.argv);
|
|
1845
1468
|
if (opts.help) {
|
|
1846
1469
|
printHelp();
|
|
1847
|
-
//
|
|
1470
|
+
// exitCode + return, not process.exit() — buffered stdout must flush.
|
|
1848
1471
|
process.exitCode = 0;
|
|
1849
1472
|
return;
|
|
1850
1473
|
}
|
|
1851
1474
|
|
|
1852
|
-
// Reject unknown flags BEFORE any network
|
|
1853
|
-
// (
|
|
1854
|
-
// refresh. Exit 2 matches refresh's own scheme (2 = error / unknown source).
|
|
1475
|
+
// Reject unknown flags BEFORE any network or catalog work: a swallowed typo
|
|
1476
|
+
// (`--aply`) otherwise falls through to a default all-sources live refresh.
|
|
1855
1477
|
if (Array.isArray(opts._unknownFlags) && opts._unknownFlags.length > 0) {
|
|
1856
1478
|
const uniq = [...new Set(opts._unknownFlags)];
|
|
1857
1479
|
process.stderr.write(JSON.stringify({
|
|
@@ -1865,23 +1487,12 @@ async function main() {
|
|
|
1865
1487
|
return;
|
|
1866
1488
|
}
|
|
1867
1489
|
|
|
1868
|
-
//
|
|
1869
|
-
// path
|
|
1870
|
-
// invoked directly, delegate the SAME way so behavior matches the help text:
|
|
1871
|
-
// --prefetch populates the cache, --no-network is a report-only dry run that
|
|
1872
|
-
// writes nothing. Without this, the direct path fell through to the live
|
|
1873
|
-
// refresh loop and could egress + write refresh-report.json despite
|
|
1874
|
-
// --no-network.
|
|
1490
|
+
// Cache operations delegate to lib/prefetch.js exactly as bin/exceptd.js does,
|
|
1491
|
+
// so the direct path cannot egress and write a report despite --no-network.
|
|
1875
1492
|
if (opts.prefetch || opts.noNetwork) {
|
|
1876
|
-
//
|
|
1877
|
-
//
|
|
1878
|
-
// the refresh
|
|
1879
|
-
// resolve advisories by live id lookup and have no cache layer. Without
|
|
1880
|
-
// this guard, `refresh --prefetch --source osv` reached prefetch.js and
|
|
1881
|
-
// died with `prefetch: fatal: unknown source "osv"` — leaking the internal
|
|
1882
|
-
// verb name (the operator typed `refresh`) and calling a source "unknown"
|
|
1883
|
-
// that the refresh help just listed as valid. Emit a refresh-prefixed,
|
|
1884
|
-
// actionable message instead and forward only the cacheable subset.
|
|
1493
|
+
// prefetch.js knows only kev/nvd/epss/rfc/pins; the refresh-only sources
|
|
1494
|
+
// resolve by live id lookup and would die there as "unknown source", calling
|
|
1495
|
+
// a source unknown that the refresh help lists as valid.
|
|
1885
1496
|
const PREFETCHABLE = new Set(["kev", "nvd", "epss", "rfc", "pins"]);
|
|
1886
1497
|
let forwardSource = opts.source;
|
|
1887
1498
|
if (opts.source) {
|
|
@@ -1927,13 +1538,8 @@ async function main() {
|
|
|
1927
1538
|
return;
|
|
1928
1539
|
}
|
|
1929
1540
|
|
|
1930
|
-
//
|
|
1931
|
-
//
|
|
1932
|
-
// written, please review") so CI pipelines surface the needed editorial
|
|
1933
|
-
// step. Operator must run `--apply` for the write to land; without it,
|
|
1934
|
-
// the seed is printed to stdout for review.
|
|
1935
|
-
// An empty --advisory value (`--advisory ""` / `--advisory=`) must error
|
|
1936
|
-
// rather than silently falling through to a full-refresh dry-run.
|
|
1541
|
+
// `--advisory <id>` short-circuits the source loop. An empty value must error
|
|
1542
|
+
// rather than falling through to a full-refresh dry run.
|
|
1937
1543
|
if (opts.advisory != null && opts.advisory.trim() === "") {
|
|
1938
1544
|
process.stderr.write(JSON.stringify({
|
|
1939
1545
|
ok: false,
|
|
@@ -1966,17 +1572,11 @@ async function main() {
|
|
|
1966
1572
|
|
|
1967
1573
|
let hadFailure = false;
|
|
1968
1574
|
|
|
1969
|
-
//
|
|
1970
|
-
//
|
|
1971
|
-
// out via Promise.all() — each source already has its own per-source
|
|
1972
|
-
// queue with its own rate budget, so parallel sources don't compete
|
|
1973
|
-
// against each other for tokens. The two modes produce the same report
|
|
1974
|
-
// structure; only wall-clock differs.
|
|
1575
|
+
// Sequential keeps log output interleaved cleanly; --swarm fans the sources
|
|
1576
|
+
// out through Promise.all. Both modes produce the same report structure.
|
|
1975
1577
|
const runOne = async (src) => {
|
|
1976
|
-
//
|
|
1977
|
-
//
|
|
1978
|
-
// branches when neither fixtures nor cacheDir was wired up. Refuse
|
|
1979
|
-
// here so the air-gap guarantee holds uniformly across every source.
|
|
1578
|
+
// GHSA and OSV honour --air-gap at the module level, but kev/epss/nvd/rfc/
|
|
1579
|
+
// pins fall through to their live branches without fixtures or a cacheDir.
|
|
1980
1580
|
if (ctx.airGap && !ctx.fixtures?.[src.name] && !ctx.cacheDir) {
|
|
1981
1581
|
return {
|
|
1982
1582
|
src,
|
|
@@ -1998,12 +1598,9 @@ async function main() {
|
|
|
1998
1598
|
return { src, diff };
|
|
1999
1599
|
};
|
|
2000
1600
|
|
|
2001
|
-
//
|
|
2002
|
-
//
|
|
2003
|
-
//
|
|
2004
|
-
// The orchestrator otherwise runs every source independently and only
|
|
2005
|
-
// persists outputs into the report, never back onto ctx — so without this
|
|
2006
|
-
// thread the watcher always sees empty input and emits zero candidates.
|
|
1601
|
+
// cve-regression-watcher consumes the advisories source's per-feed CVE
|
|
1602
|
+
// observations — they carry the in-catalog historical IDs the annotate verdict
|
|
1603
|
+
// needs — falling back to its diffs. Nothing else writes source output onto ctx.
|
|
2007
1604
|
const threadAdvisoriesIntoCtx = (src, diff) => {
|
|
2008
1605
|
if (src && src.name === "advisories" && diff && !diff.air_gap_blocked) {
|
|
2009
1606
|
if (Array.isArray(diff.observations)) ctx.advisoriesObservations = diff.observations;
|
|
@@ -2013,8 +1610,7 @@ async function main() {
|
|
|
2013
1610
|
|
|
2014
1611
|
let outcomes;
|
|
2015
1612
|
if (!opts.swarm) {
|
|
2016
|
-
//
|
|
2017
|
-
// resolves, BEFORE the next source's fetchDiff(ctx) is invoked.
|
|
1613
|
+
// Thread onto ctx as it resolves, BEFORE the next source's fetchDiff(ctx).
|
|
2018
1614
|
outcomes = [];
|
|
2019
1615
|
for (const src of sources) {
|
|
2020
1616
|
const outcome = await runOne(src);
|
|
@@ -2022,13 +1618,9 @@ async function main() {
|
|
|
2022
1618
|
outcomes.push(outcome);
|
|
2023
1619
|
}
|
|
2024
1620
|
} else {
|
|
2025
|
-
//
|
|
2026
|
-
//
|
|
2027
|
-
//
|
|
2028
|
-
// thread the resolved advisories observations onto ctx, then run the
|
|
2029
|
-
// watcher. If advisories is NOT also selected, the watcher runs in the
|
|
2030
|
-
// first batch like any other source (its empty-input contract is the
|
|
2031
|
-
// operator's choice, not a silent race).
|
|
1621
|
+
// Chaining through shared ctx cannot work inside one Promise.all, so the
|
|
1622
|
+
// watcher runs in a second pass. With advisories unselected it joins the
|
|
1623
|
+
// first batch — empty input is then the operator's choice, not a race.
|
|
2032
1624
|
const hasAdvisories = sources.some((s) => s.name === "advisories");
|
|
2033
1625
|
const watcher = hasAdvisories
|
|
2034
1626
|
? sources.find((s) => s.name === "cve-regression-watcher")
|
|
@@ -2046,12 +1638,9 @@ async function main() {
|
|
|
2046
1638
|
}
|
|
2047
1639
|
}
|
|
2048
1640
|
|
|
2049
|
-
//
|
|
2050
|
-
//
|
|
2051
|
-
//
|
|
2052
|
-
// throw never reaches main().catch where the code is otherwise honored.
|
|
2053
|
-
// Carry the marker through here so main() can prefer exit 4 (BLOCKED /
|
|
2054
|
-
// precondition refusal) over the generic per-source-failure exit 1.
|
|
1641
|
+
// runOne catches readCachedJson's cache-integrity refusals as per-source
|
|
1642
|
+
// errors, so _exceptd_exit_code=4 never reaches main().catch. Carry the marker
|
|
1643
|
+
// so exit 4 (precondition refusal) wins over the generic per-source failure 1.
|
|
2055
1644
|
let cacheIntegrityFailure = false;
|
|
2056
1645
|
for (const { src, diff, error } of outcomes) {
|
|
2057
1646
|
if (error) {
|
|
@@ -2075,13 +1664,10 @@ async function main() {
|
|
|
2075
1664
|
diffs: diff.diffs,
|
|
2076
1665
|
applies_to: src.applies_to,
|
|
2077
1666
|
report_only: !!src.report_only,
|
|
2078
|
-
//
|
|
2079
|
-
// marker through to the persisted report so stdout-parsing consumers
|
|
2080
|
-
// and the regression test can verify the network refusal happened.
|
|
1667
|
+
// Persisted so a stdout-parsing consumer can verify the network refusal.
|
|
2081
1668
|
...(diff.air_gap_blocked ? { air_gap_blocked: true } : {}),
|
|
2082
|
-
//
|
|
2083
|
-
//
|
|
2084
|
-
// and the advisories observations[] the watcher consumes.
|
|
1669
|
+
// The watcher stamps input_field_used onto _meta; persisting it makes the
|
|
1670
|
+
// chaining observable.
|
|
2085
1671
|
...(diff._meta ? { _meta: diff._meta } : {}),
|
|
2086
1672
|
...(Array.isArray(diff.observations) ? { observations: diff.observations } : {}),
|
|
2087
1673
|
};
|
|
@@ -2094,8 +1680,7 @@ async function main() {
|
|
|
2094
1680
|
}
|
|
2095
1681
|
}
|
|
2096
1682
|
|
|
2097
|
-
//
|
|
2098
|
-
// suites don't race on a shared refresh-report.json at the repo root.
|
|
1683
|
+
// --report-out keeps concurrent runs off the shared refresh-report.json.
|
|
2099
1684
|
const reportPath = opts.reportOut ? path.resolve(opts.reportOut) : ABS("refresh-report.json");
|
|
2100
1685
|
writeJson(reportPath, report);
|
|
2101
1686
|
log(`\nWrote ${path.relative(ROOT, reportPath)}`);
|
|
@@ -2111,14 +1696,8 @@ async function main() {
|
|
|
2111
1696
|
}
|
|
2112
1697
|
}
|
|
2113
1698
|
|
|
2114
|
-
//
|
|
2115
|
-
//
|
|
2116
|
-
// truncate buffered stdout (refresh-report path log line, summary log
|
|
2117
|
-
// lines piped to a consumer). exitCode + return lets the event loop end
|
|
2118
|
-
// naturally and stdout drains in full.
|
|
2119
|
-
// Prefer the documented BLOCKED (4) code when any source refused on a
|
|
2120
|
-
// cache-integrity precondition; fall back to generic failure (1) for other
|
|
2121
|
-
// per-source errors / downstream gate failures.
|
|
1699
|
+
// exitCode + return, not process.exit(), which can truncate buffered stdout.
|
|
1700
|
+
// 4 (BLOCKED) wins over 1 when a source refused on a cache precondition.
|
|
2122
1701
|
process.exitCode = cacheIntegrityFailure ? 4 : (hadFailure ? 1 : 0);
|
|
2123
1702
|
}
|
|
2124
1703
|
|
|
@@ -2130,25 +1709,18 @@ async function sequential(items, fn) {
|
|
|
2130
1709
|
|
|
2131
1710
|
if (require.main === module) {
|
|
2132
1711
|
main().catch((err) => {
|
|
2133
|
-
//
|
|
2134
|
-
// JSON line on stderr instead of a fatal stack trace.
|
|
1712
|
+
// A hinted error prints its message plus a JSON line, never a stack trace.
|
|
2135
1713
|
if (err && err._exceptd_hint) {
|
|
2136
1714
|
console.error(err.message);
|
|
2137
1715
|
console.error(JSON.stringify({ ok: false, error: err.message.split("\n")[0], hint: err.message.split("\n").slice(1).join(" ").trim(), verb: "refresh" }));
|
|
2138
1716
|
} else if (err && err._exceptd_unknown_source) {
|
|
2139
|
-
//
|
|
2140
|
-
// stack trace; chosenSources throws this for unknown --source values.
|
|
1717
|
+
// chosenSources throws this for an unknown --source; no stack trace.
|
|
2141
1718
|
console.error(err.message);
|
|
2142
1719
|
} else {
|
|
2143
1720
|
console.error(`refresh-external: fatal: ${err && err.stack ? err.stack : err}`);
|
|
2144
1721
|
}
|
|
2145
|
-
//
|
|
2146
|
-
//
|
|
2147
|
-
// ends the process with code 2 but lets stderr drain first.
|
|
2148
|
-
// Cache-integrity / cache-stale / from-fixture-disabled refusals carry
|
|
2149
|
-
// an explicit exit code (4) via _exceptd_exit_code; honor that so
|
|
2150
|
-
// downstream automation can distinguish "blocked by precondition"
|
|
2151
|
-
// (exit 4) from "fatal/unhandled" (exit 2).
|
|
1722
|
+
// exitCode rather than process.exit(2), so stderr drains first. A refusal
|
|
1723
|
+
// carrying _exceptd_exit_code=4 reads as "blocked by precondition", not fatal.
|
|
2152
1724
|
process.exitCode = (err && Number.isInteger(err._exceptd_exit_code)) ? err._exceptd_exit_code : 2;
|
|
2153
1725
|
});
|
|
2154
1726
|
}
|