@blamejs/exceptd-skills 0.12.11 → 0.12.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +243 -0
- package/bin/exceptd.js +299 -48
- package/data/_indexes/_meta.json +49 -48
- package/data/_indexes/activity-feed.json +13 -5
- package/data/_indexes/catalog-summaries.json +51 -29
- package/data/_indexes/chains.json +3238 -3210
- package/data/_indexes/frequency.json +3 -0
- package/data/_indexes/jurisdiction-map.json +5 -3
- package/data/_indexes/section-offsets.json +712 -685
- package/data/_indexes/theater-fingerprints.json +1 -1
- package/data/_indexes/token-budget.json +355 -340
- package/data/atlas-ttps.json +144 -129
- package/data/attack-techniques.json +339 -0
- package/data/cve-catalog.json +515 -475
- package/data/cwe-catalog.json +1081 -759
- package/data/exploit-availability.json +63 -15
- package/data/framework-control-gaps.json +867 -843
- package/data/rfc-references.json +276 -276
- package/keys/EXPECTED_FINGERPRINT +1 -0
- package/lib/auto-discovery.js +21 -4
- package/lib/cross-ref-api.js +39 -6
- package/lib/cve-curation.js +505 -47
- package/lib/lint-skills.js +217 -15
- package/lib/playbook-runner.js +1224 -183
- package/lib/prefetch.js +121 -8
- package/lib/refresh-external.js +261 -95
- package/lib/refresh-network.js +208 -18
- package/lib/schemas/manifest.schema.json +16 -0
- package/lib/scoring.js +83 -7
- package/lib/sign.js +112 -3
- package/lib/source-ghsa.js +219 -37
- package/lib/source-osv.js +381 -122
- package/lib/validate-catalog-meta.js +64 -9
- package/lib/validate-cve-catalog.js +213 -7
- package/lib/validate-indexes.js +88 -37
- package/lib/validate-playbooks.js +469 -0
- package/lib/verify.js +313 -16
- package/manifest-snapshot.json +1 -1
- package/manifest-snapshot.sha256 +1 -0
- package/manifest.json +73 -73
- package/orchestrator/dispatcher.js +21 -1
- package/orchestrator/event-bus.js +52 -8
- package/orchestrator/index.js +279 -20
- package/orchestrator/pipeline.js +63 -2
- package/orchestrator/scanner.js +32 -10
- package/orchestrator/scheduler.js +196 -20
- package/package.json +3 -1
- package/sbom.cdx.json +9 -9
- package/scripts/check-manifest-snapshot.js +32 -0
- package/scripts/check-sbom-currency.js +65 -3
- package/scripts/check-test-coverage.js +142 -19
- package/scripts/predeploy.js +110 -40
- package/scripts/refresh-manifest-snapshot.js +55 -4
- package/scripts/validate-vendor-online.js +169 -0
- package/scripts/verify-shipped-tarball.js +106 -3
- package/skills/ai-attack-surface/skill.md +18 -10
- package/skills/ai-c2-detection/skill.md +7 -2
- package/skills/ai-risk-management/skill.md +5 -4
- package/skills/api-security/skill.md +3 -3
- package/skills/attack-surface-pentest/skill.md +5 -5
- package/skills/cloud-security/skill.md +1 -1
- package/skills/compliance-theater/skill.md +8 -8
- package/skills/container-runtime-security/skill.md +1 -1
- package/skills/dlp-gap-analysis/skill.md +5 -1
- package/skills/email-security-anti-phishing/skill.md +1 -1
- package/skills/exploit-scoring/skill.md +18 -18
- package/skills/framework-gap-analysis/skill.md +6 -6
- package/skills/global-grc/skill.md +3 -2
- package/skills/identity-assurance/skill.md +2 -2
- package/skills/incident-response-playbook/skill.md +4 -4
- package/skills/kernel-lpe-triage/skill.md +21 -2
- package/skills/mcp-agent-trust/skill.md +17 -10
- package/skills/mlops-security/skill.md +2 -1
- package/skills/ot-ics-security/skill.md +1 -1
- package/skills/policy-exception-gen/skill.md +3 -3
- package/skills/pqc-first/skill.md +1 -1
- package/skills/rag-pipeline-security/skill.md +7 -3
- package/skills/researcher/skill.md +20 -3
- package/skills/sector-energy/skill.md +1 -1
- package/skills/sector-federal-government/skill.md +1 -1
- package/skills/sector-financial/skill.md +3 -3
- package/skills/sector-healthcare/skill.md +2 -2
- package/skills/security-maturity-tiers/skill.md +7 -7
- package/skills/skill-update-loop/skill.md +19 -3
- package/skills/supply-chain-integrity/skill.md +1 -1
- package/skills/threat-model-currency/skill.md +11 -11
- package/skills/threat-modeling-methodology/skill.md +3 -3
- package/skills/webapp-security/skill.md +1 -1
- package/skills/zeroday-gap-learn/skill.md +51 -7
- package/vendor/blamejs/_PROVENANCE.json +4 -1
- package/vendor/blamejs/worker-pool.js +38 -0
package/lib/prefetch.js
CHANGED
|
@@ -188,9 +188,93 @@ function loadIndex(cacheDir) {
|
|
|
188
188
|
}
|
|
189
189
|
}
|
|
190
190
|
|
|
191
|
-
|
|
191
|
+
// v0.12.12 C4: atomic write helper — tmp + rename. Concurrent readers either
|
|
192
|
+
// see the prior file in full or the new file in full, never a half-written
|
|
193
|
+
// buffer. fs.renameSync is atomic on POSIX and on Windows for same-volume
|
|
194
|
+
// renames; a `.tmp.<pid>.<rand>` sibling to the destination is always
|
|
195
|
+
// same-volume.
|
|
196
|
+
function writeFileAtomic(p, body) {
|
|
197
|
+
const tmpPath = `${p}.tmp.${process.pid}.${Math.random().toString(36).slice(2, 10)}`;
|
|
198
|
+
fs.writeFileSync(tmpPath, body);
|
|
199
|
+
try {
|
|
200
|
+
fs.renameSync(tmpPath, p);
|
|
201
|
+
} catch (err) {
|
|
202
|
+
try { fs.unlinkSync(tmpPath); } catch {}
|
|
203
|
+
throw err;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// v0.12.12 C2: lockfile-gated read-modify-write for _index.json. Two
|
|
208
|
+
// concurrent prefetch runs against the same cache dir previously raced —
|
|
209
|
+
// each loaded the index at start, mutated its in-memory copy as entries
|
|
210
|
+
// fetched, then wrote at the end. The second writer overwrote the first,
|
|
211
|
+
// silently dropping any entries the first run wrote.
|
|
212
|
+
//
|
|
213
|
+
// Stale-lock recovery: if a holder crashes without unlinking, the lockfile
|
|
214
|
+
// persists. After backoff, if the lockfile's mtime is older than 30s we
|
|
215
|
+
// treat it as orphaned and unlink it before retrying.
|
|
216
|
+
async function withIndexLock(cacheDir, mutator) {
|
|
192
217
|
if (!fs.existsSync(cacheDir)) fs.mkdirSync(cacheDir, { recursive: true });
|
|
193
|
-
|
|
218
|
+
const lockPath = path.join(cacheDir, "_index.json.lock");
|
|
219
|
+
const indexPath = path.join(cacheDir, "_index.json");
|
|
220
|
+
const MAX_RETRIES = 50;
|
|
221
|
+
const STALE_LOCK_MS = 30_000;
|
|
222
|
+
let acquired = false;
|
|
223
|
+
for (let i = 0; i < MAX_RETRIES; i++) {
|
|
224
|
+
try {
|
|
225
|
+
fs.writeFileSync(lockPath, String(process.pid), { flag: "wx" });
|
|
226
|
+
acquired = true;
|
|
227
|
+
break;
|
|
228
|
+
} catch (e) {
|
|
229
|
+
// EEXIST is the POSIX signal another process holds the lock. On
|
|
230
|
+
// Windows the same race surfaces as EPERM (a sharing-violation
|
|
231
|
+
// raised when the other process is mid-unlink). Treat both as
|
|
232
|
+
// "lock held, back off" rather than a fatal error.
|
|
233
|
+
if (e.code !== "EEXIST" && e.code !== "EPERM") throw e;
|
|
234
|
+
try {
|
|
235
|
+
const stat = fs.statSync(lockPath);
|
|
236
|
+
if (Date.now() - stat.mtimeMs > STALE_LOCK_MS) {
|
|
237
|
+
try { fs.unlinkSync(lockPath); } catch {}
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
240
|
+
} catch {}
|
|
241
|
+
await new Promise((r) => setTimeout(r, 50 + Math.random() * 150));
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
if (!acquired) {
|
|
245
|
+
throw new Error(`withIndexLock: could not acquire ${lockPath} after ${MAX_RETRIES} attempts`);
|
|
246
|
+
}
|
|
247
|
+
try {
|
|
248
|
+
// Always re-read the current on-disk index inside the lock. Stale
|
|
249
|
+
// in-memory copies from before acquisition are the entire bug class.
|
|
250
|
+
let current;
|
|
251
|
+
if (fs.existsSync(indexPath)) {
|
|
252
|
+
try { current = JSON.parse(fs.readFileSync(indexPath, "utf8")); }
|
|
253
|
+
catch { current = { entries: {}, generated_at: null }; }
|
|
254
|
+
} else {
|
|
255
|
+
current = { entries: {}, generated_at: null };
|
|
256
|
+
}
|
|
257
|
+
const mutated = await mutator(current);
|
|
258
|
+
const toWrite = mutated === undefined ? current : mutated;
|
|
259
|
+
writeFileAtomic(indexPath, JSON.stringify(toWrite, null, 2) + "\n");
|
|
260
|
+
return toWrite;
|
|
261
|
+
} finally {
|
|
262
|
+
try { fs.unlinkSync(lockPath); } catch {}
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
// Back-compat: existing callers used saveIndex(cacheDir, idx). The thin
|
|
267
|
+
// wrapper merges entries under the lock so a concurrent run's writes are
|
|
268
|
+
// preserved (rather than blindly overwriting them with the caller's
|
|
269
|
+
// possibly-stale in-memory `idx`).
|
|
270
|
+
async function saveIndex(cacheDir, idx) {
|
|
271
|
+
await withIndexLock(cacheDir, (current) => {
|
|
272
|
+
const mergedEntries = { ...current.entries, ...idx.entries };
|
|
273
|
+
return {
|
|
274
|
+
entries: mergedEntries,
|
|
275
|
+
generated_at: idx.generated_at || current.generated_at,
|
|
276
|
+
};
|
|
277
|
+
});
|
|
194
278
|
}
|
|
195
279
|
|
|
196
280
|
function entryKey(source, id) {
|
|
@@ -292,17 +376,32 @@ async function prefetch(options = {}) {
|
|
|
292
376
|
run: () => timedFetch(item.url, reqHeaders),
|
|
293
377
|
meta: { id: item.id },
|
|
294
378
|
})
|
|
295
|
-
.then((res) => {
|
|
296
|
-
const
|
|
379
|
+
.then(async (res) => {
|
|
380
|
+
const targetPath = entryPath(opts.cacheDir, item.source, item.id);
|
|
381
|
+
const dir = path.dirname(targetPath);
|
|
297
382
|
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
298
|
-
|
|
299
|
-
|
|
383
|
+
// v0.12.12 C4: atomic write of the payload. A concurrent reader
|
|
384
|
+
// (refresh --from-cache running in parallel) sees the prior
|
|
385
|
+
// payload in full or the new payload in full, never a partial
|
|
386
|
+
// buffer.
|
|
387
|
+
writeFileAtomic(targetPath, JSON.stringify(res.json, null, 2) + "\n");
|
|
388
|
+
const meta = {
|
|
300
389
|
fetched_at: new Date().toISOString(),
|
|
301
390
|
etag: res.etag,
|
|
302
391
|
last_modified: res.lastModified,
|
|
303
392
|
url: item.url,
|
|
304
393
|
sha256: crypto.createHash("sha256").update(JSON.stringify(res.json)).digest("hex"),
|
|
305
394
|
};
|
|
395
|
+
idx.entries[entryKey(item.source, item.id)] = meta;
|
|
396
|
+
// v0.12.12 C2: persist this entry's metadata to _index.json under
|
|
397
|
+
// lock immediately, merging with whatever the on-disk index has
|
|
398
|
+
// (another concurrent prefetch may have written sibling entries).
|
|
399
|
+
// Without this, only the in-memory idx is updated; the final
|
|
400
|
+
// saveIndex() would overwrite a sibling run's writes.
|
|
401
|
+
await withIndexLock(opts.cacheDir, (current) => {
|
|
402
|
+
current.entries[entryKey(item.source, item.id)] = meta;
|
|
403
|
+
return current;
|
|
404
|
+
});
|
|
306
405
|
result.fetched++;
|
|
307
406
|
result.by_source[item.source].fetched++;
|
|
308
407
|
log(` [${item.source}] ${item.id} — ok`);
|
|
@@ -317,7 +416,10 @@ async function prefetch(options = {}) {
|
|
|
317
416
|
await Promise.all(jobPromises);
|
|
318
417
|
await queue.drain();
|
|
319
418
|
idx.generated_at = new Date().toISOString();
|
|
320
|
-
|
|
419
|
+
// v0.12.12 C2: saveIndex now merges under lock with whatever is on disk
|
|
420
|
+
// (another concurrent prefetch's entries). Without the merge, a sibling
|
|
421
|
+
// run's writes would be silently overwritten here at the end of our run.
|
|
422
|
+
await saveIndex(opts.cacheDir, idx);
|
|
321
423
|
|
|
322
424
|
// Final summary is unconditional — --quiet suppresses per-entry chatter
|
|
323
425
|
// (the noisy part) but the operator still needs one line confirming success.
|
|
@@ -398,4 +500,15 @@ async function main() {
|
|
|
398
500
|
|
|
399
501
|
if (require.main === module) main();
|
|
400
502
|
|
|
401
|
-
module.exports = {
|
|
503
|
+
module.exports = {
|
|
504
|
+
prefetch,
|
|
505
|
+
readCached,
|
|
506
|
+
parseArgs,
|
|
507
|
+
SOURCES,
|
|
508
|
+
DEFAULT_CACHE,
|
|
509
|
+
// v0.12.12 C2: exported for the concurrent-writer regression test.
|
|
510
|
+
// Not part of the operator-facing API — internal contract for tests
|
|
511
|
+
// that need to exercise the lockfile path without spawning the full
|
|
512
|
+
// prefetch network pipeline.
|
|
513
|
+
_internal: { withIndexLock, writeFileAtomic, loadIndex, saveIndex },
|
|
514
|
+
};
|
package/lib/refresh-external.js
CHANGED
|
@@ -141,6 +141,17 @@ Modes:
|
|
|
141
141
|
exceptd refresh --advisory GHSA-xxxx-xxxx-xxxx --apply
|
|
142
142
|
exceptd refresh --advisory MAL-2026-3083
|
|
143
143
|
exceptd refresh --advisory RUSTSEC-2025-0001
|
|
144
|
+
--curate <CVE-ID> emit editorial questions + ranked candidates
|
|
145
|
+
(ATLAS/ATT&CK/CWE/framework gaps) for a draft entry.
|
|
146
|
+
With --answers <path> the operator-supplied answers
|
|
147
|
+
are validated, applied to the catalog entry, and the
|
|
148
|
+
draft is promoted out of _auto_imported / _draft once
|
|
149
|
+
every required schema field is populated. Atomic write;
|
|
150
|
+
concurrent --apply runs against the same catalog are
|
|
151
|
+
safe. --apply is an alias for "--answers implies write".
|
|
152
|
+
Examples:
|
|
153
|
+
exceptd refresh --curate CVE-2026-45321
|
|
154
|
+
exceptd refresh --curate CVE-2026-45321 --answers a.json --apply
|
|
144
155
|
|
|
145
156
|
Sources (default = all):
|
|
146
157
|
kev CISA Known Exploited Vulnerabilities
|
|
@@ -214,26 +225,32 @@ const KEV_SOURCE = {
|
|
|
214
225
|
let updated = 0;
|
|
215
226
|
let added = 0;
|
|
216
227
|
const errors = [];
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
228
|
+
const catalogPath = ctx.cvePath || ABS("data/cve-catalog.json");
|
|
229
|
+
await withCatalogLock(catalogPath, (catalog) => {
|
|
230
|
+
for (const d of diffs) {
|
|
231
|
+
if (d.op === "add") {
|
|
232
|
+
// Auto-discovered new entry. Refuse to overwrite if the entry
|
|
233
|
+
// somehow exists (race condition / stale fixture); skip silently.
|
|
234
|
+
if (catalog[d.id]) continue;
|
|
235
|
+
catalog[d.id] = d.entry;
|
|
236
|
+
added++;
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
if (!catalog[d.id]) {
|
|
240
|
+
errors.push(`KEV: no local entry for ${d.id}`);
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
243
|
+
catalog[d.id][d.field] = d.after;
|
|
244
|
+
catalog[d.id].last_verified = TODAY;
|
|
245
|
+
updated++;
|
|
229
246
|
}
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
247
|
+
catalog._meta = catalog._meta || {};
|
|
248
|
+
catalog._meta.last_updated = TODAY;
|
|
249
|
+
// Refresh the in-memory view so later sources in the same process
|
|
250
|
+
// (sequential or --swarm) see the post-write state.
|
|
251
|
+
ctx.cveCatalog = catalog;
|
|
252
|
+
return catalog;
|
|
253
|
+
});
|
|
237
254
|
return { updated: updated + added, added, drift_updated: updated, errors };
|
|
238
255
|
},
|
|
239
256
|
};
|
|
@@ -297,18 +314,22 @@ const EPSS_SOURCE = {
|
|
|
297
314
|
async applyDiff(ctx, diffs) {
|
|
298
315
|
let updated = 0;
|
|
299
316
|
const errors = [];
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
317
|
+
const catalogPath = ctx.cvePath || ABS("data/cve-catalog.json");
|
|
318
|
+
await withCatalogLock(catalogPath, (catalog) => {
|
|
319
|
+
for (const d of diffs) {
|
|
320
|
+
if (!catalog[d.id]) {
|
|
321
|
+
errors.push(`EPSS: no local entry for ${d.id}`);
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
catalog[d.id][d.field] = d.after;
|
|
325
|
+
catalog[d.id].last_verified = TODAY;
|
|
326
|
+
updated++;
|
|
304
327
|
}
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
ctx.cveCatalog._meta.last_updated = TODAY;
|
|
311
|
-
writeJson(ctx.cvePath || ABS("data/cve-catalog.json"), ctx.cveCatalog);
|
|
328
|
+
catalog._meta = catalog._meta || {};
|
|
329
|
+
catalog._meta.last_updated = TODAY;
|
|
330
|
+
ctx.cveCatalog = catalog;
|
|
331
|
+
return catalog;
|
|
332
|
+
});
|
|
312
333
|
return { updated, errors };
|
|
313
334
|
},
|
|
314
335
|
};
|
|
@@ -342,18 +363,22 @@ const NVD_SOURCE = {
|
|
|
342
363
|
async applyDiff(ctx, diffs) {
|
|
343
364
|
let updated = 0;
|
|
344
365
|
const errors = [];
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
366
|
+
const catalogPath = ctx.cvePath || ABS("data/cve-catalog.json");
|
|
367
|
+
await withCatalogLock(catalogPath, (catalog) => {
|
|
368
|
+
for (const d of diffs) {
|
|
369
|
+
if (!catalog[d.id]) {
|
|
370
|
+
errors.push(`NVD: no local entry for ${d.id}`);
|
|
371
|
+
continue;
|
|
372
|
+
}
|
|
373
|
+
catalog[d.id][d.field] = d.after;
|
|
374
|
+
catalog[d.id].last_verified = TODAY;
|
|
375
|
+
updated++;
|
|
349
376
|
}
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
ctx.cveCatalog._meta.last_updated = TODAY;
|
|
356
|
-
writeJson(ctx.cvePath || ABS("data/cve-catalog.json"), ctx.cveCatalog);
|
|
377
|
+
catalog._meta = catalog._meta || {};
|
|
378
|
+
catalog._meta.last_updated = TODAY;
|
|
379
|
+
ctx.cveCatalog = catalog;
|
|
380
|
+
return catalog;
|
|
381
|
+
});
|
|
357
382
|
return { updated, errors };
|
|
358
383
|
},
|
|
359
384
|
};
|
|
@@ -398,26 +423,30 @@ const RFC_SOURCE = {
|
|
|
398
423
|
let updated = 0;
|
|
399
424
|
let added = 0;
|
|
400
425
|
const errors = [];
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
426
|
+
const rfcPath = ABS("data/rfc-references.json");
|
|
427
|
+
await withCatalogLock(rfcPath, (rfcCatalog) => {
|
|
428
|
+
for (const d of diffs) {
|
|
429
|
+
if (d.op === "add") {
|
|
430
|
+
if (rfcCatalog[d.id]) continue;
|
|
431
|
+
rfcCatalog[d.id] = d.entry;
|
|
432
|
+
added++;
|
|
433
|
+
continue;
|
|
434
|
+
}
|
|
435
|
+
if (d.field !== "status") continue; // notes are informational
|
|
436
|
+
const entry = rfcCatalog[d.id];
|
|
437
|
+
if (!entry) {
|
|
438
|
+
errors.push(`RFC: no local entry for ${d.id}`);
|
|
439
|
+
continue;
|
|
440
|
+
}
|
|
441
|
+
entry.status = d.after;
|
|
442
|
+
entry.last_verified = TODAY;
|
|
443
|
+
updated++;
|
|
413
444
|
}
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
ctx.rfcCatalog._meta.last_updated = TODAY;
|
|
420
|
-
writeJson(ABS("data/rfc-references.json"), ctx.rfcCatalog);
|
|
445
|
+
rfcCatalog._meta = rfcCatalog._meta || {};
|
|
446
|
+
rfcCatalog._meta.last_updated = TODAY;
|
|
447
|
+
ctx.rfcCatalog = rfcCatalog;
|
|
448
|
+
return rfcCatalog;
|
|
449
|
+
});
|
|
421
450
|
return { updated: updated + added, added, drift_updated: updated, errors };
|
|
422
451
|
},
|
|
423
452
|
};
|
|
@@ -520,20 +549,31 @@ const GHSA_SOURCE = {
|
|
|
520
549
|
return ghsa.buildDiff(ctx);
|
|
521
550
|
},
|
|
522
551
|
async applyDiff(ctx, diffs) {
|
|
523
|
-
|
|
552
|
+
// v0.12.14 (audit B-F1): the prior shape mutated ctx.cveCatalog in
|
|
553
|
+
// memory but NEVER persisted to disk. Bulk `--source ghsa --apply`
|
|
554
|
+
// reported "applied: N updates" while the catalog file gained zero
|
|
555
|
+
// entries. Worse under `--swarm`: KEV's withCatalogLock would re-read
|
|
556
|
+
// catalog from disk INSIDE the lock and overwrite the unflushed
|
|
557
|
+
// in-memory mutations. Route through the same withCatalogLock helper
|
|
558
|
+
// that KEV/EPSS/NVD/RFC use (v0.12.12 concurrency fix).
|
|
559
|
+
const catalogPath = ctx.cvePath || ABS("data/cve-catalog.json");
|
|
524
560
|
let updated = 0;
|
|
525
561
|
const errors = [];
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
562
|
+
await withCatalogLock(catalogPath, (catalog) => {
|
|
563
|
+
for (const d of diffs) {
|
|
564
|
+
if (d.field !== "_new_entry") continue;
|
|
565
|
+
if (!d.after || !d.id) continue;
|
|
566
|
+
if (catalog[d.id]) continue; // never overwrite existing entries
|
|
567
|
+
try {
|
|
568
|
+
catalog[d.id] = d.after;
|
|
569
|
+
updated++;
|
|
570
|
+
} catch (e) {
|
|
571
|
+
errors.push(`${d.id}: ${e.message}`);
|
|
572
|
+
}
|
|
535
573
|
}
|
|
536
|
-
|
|
574
|
+
ctx.cveCatalog = catalog;
|
|
575
|
+
return catalog;
|
|
576
|
+
});
|
|
537
577
|
return { updated, errors };
|
|
538
578
|
},
|
|
539
579
|
};
|
|
@@ -562,20 +602,27 @@ const OSV_SOURCE = {
|
|
|
562
602
|
return osv.buildDiff(ctx);
|
|
563
603
|
},
|
|
564
604
|
async applyDiff(ctx, diffs) {
|
|
565
|
-
//
|
|
605
|
+
// v0.12.14 (audit B-F1): same fix as GHSA — route the read-modify-write
|
|
606
|
+
// through withCatalogLock so writes actually land on disk and so
|
|
607
|
+
// concurrent --source osv --apply doesn't lose updates.
|
|
608
|
+
const catalogPath = ctx.cvePath || ABS("data/cve-catalog.json");
|
|
566
609
|
let updated = 0;
|
|
567
610
|
const errors = [];
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
611
|
+
await withCatalogLock(catalogPath, (catalog) => {
|
|
612
|
+
for (const d of diffs) {
|
|
613
|
+
if (d.field !== "_new_entry") continue;
|
|
614
|
+
if (!d.after || !d.id) continue;
|
|
615
|
+
if (catalog[d.id]) continue; // never overwrite existing entries
|
|
616
|
+
try {
|
|
617
|
+
catalog[d.id] = d.after;
|
|
618
|
+
updated++;
|
|
619
|
+
} catch (e) {
|
|
620
|
+
errors.push(`${d.id}: ${e.message}`);
|
|
621
|
+
}
|
|
577
622
|
}
|
|
578
|
-
|
|
623
|
+
ctx.cveCatalog = catalog;
|
|
624
|
+
return catalog;
|
|
625
|
+
});
|
|
579
626
|
return { updated, errors };
|
|
580
627
|
},
|
|
581
628
|
};
|
|
@@ -824,8 +871,97 @@ function loadCtx(opts) {
|
|
|
824
871
|
return ctx;
|
|
825
872
|
}
|
|
826
873
|
|
|
874
|
+
// v0.12.12 C4: every persisted JSON write goes through writeJsonAtomic — a
|
|
875
|
+
// tmp + rename pattern. fs.renameSync is atomic on POSIX and on Windows for
|
|
876
|
+
// same-volume renames (which a `.tmp.<pid>.<rand>` adjacent to the target
|
|
877
|
+
// always satisfies). A concurrent reader either sees the prior file content
|
|
878
|
+
// in full or the new content in full — never a half-written buffer. The
|
|
879
|
+
// tmp name carries pid + random so two writers in the same process (e.g.
|
|
880
|
+
// worker threads) never collide on the same scratch path.
|
|
881
|
+
function writeJsonAtomic(p, obj) {
|
|
882
|
+
const tmpPath = `${p}.tmp.${process.pid}.${Math.random().toString(36).slice(2, 10)}`;
|
|
883
|
+
fs.writeFileSync(tmpPath, JSON.stringify(obj, null, 2) + "\n", "utf8");
|
|
884
|
+
try {
|
|
885
|
+
fs.renameSync(tmpPath, p);
|
|
886
|
+
} catch (err) {
|
|
887
|
+
try { fs.unlinkSync(tmpPath); } catch {}
|
|
888
|
+
throw err;
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
// Back-compat alias — exported callers and historical sites still reference
|
|
893
|
+
// writeJson. Atomic by default; never the unsafe direct-write form.
|
|
827
894
|
function writeJson(p, obj) {
|
|
828
|
-
|
|
895
|
+
writeJsonAtomic(p, obj);
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
/**
|
|
899
|
+
* v0.12.12 C1: lockfile-gated read-modify-write helper for JSON catalogs.
|
|
900
|
+
*
|
|
901
|
+
* Two concurrent `refresh --advisory CVE-A --apply` and
|
|
902
|
+
* `refresh --advisory CVE-B --apply` processes against the same catalog used
|
|
903
|
+
* to race: each read the catalog, mutated its in-memory copy, then wrote —
|
|
904
|
+
* the second write overwrote the first, silently dropping one CVE. The fix
|
|
905
|
+
* is a sidecar lockfile (created with O_EXCL via `flag: 'wx'`) that
|
|
906
|
+
* serializes the read-mutate-write triple. The mutator receives the
|
|
907
|
+
* current-on-disk catalog (re-read inside the lock, NOT a stale in-memory
|
|
908
|
+
* copy from before lock acquisition) and returns it after mutation; the
|
|
909
|
+
* helper then writes atomically via writeJsonAtomic.
|
|
910
|
+
*
|
|
911
|
+
* Stale-lock recovery: if a holder crashes without unlinking, the lockfile
|
|
912
|
+
* persists. After backoff, if the lockfile's mtime is older than 30s we
|
|
913
|
+
* treat it as orphaned and unlink it before retrying. 30s is well past any
|
|
914
|
+
* legitimate single-CVE apply (sub-second on modern disks).
|
|
915
|
+
*
|
|
916
|
+
* On acquisition failure after N retries, we throw — better than silently
|
|
917
|
+
* proceeding without the lock.
|
|
918
|
+
*
|
|
919
|
+
* @param {string} catalogPath path to the JSON catalog to lock
|
|
920
|
+
* @param {(catalog: object) => object | Promise<object>} mutator
|
|
921
|
+
* receives current-on-disk catalog, returns mutated catalog. May be
|
|
922
|
+
* async. The return value is what gets written; if it returns
|
|
923
|
+
* undefined, the in-place mutation of the passed-in catalog is used.
|
|
924
|
+
* @returns {Promise<{ wrote: boolean, result: any }>}
|
|
925
|
+
*/
|
|
926
|
+
async function withCatalogLock(catalogPath, mutator) {
|
|
927
|
+
const lockPath = `${catalogPath}.lock`;
|
|
928
|
+
const MAX_RETRIES = 50;
|
|
929
|
+
const STALE_LOCK_MS = 30_000;
|
|
930
|
+
let acquired = false;
|
|
931
|
+
for (let i = 0; i < MAX_RETRIES; i++) {
|
|
932
|
+
try {
|
|
933
|
+
fs.writeFileSync(lockPath, String(process.pid), { flag: "wx" });
|
|
934
|
+
acquired = true;
|
|
935
|
+
break;
|
|
936
|
+
} catch (e) {
|
|
937
|
+
// EEXIST is the POSIX signal another process holds the lock. On
|
|
938
|
+
// Windows the same race surfaces as EPERM (sharing-violation raised
|
|
939
|
+
// when the holder is mid-unlink). Treat both as "lock held, back off."
|
|
940
|
+
if (e.code !== "EEXIST" && e.code !== "EPERM") throw e;
|
|
941
|
+
// Stale-lock check before sleeping — a long-dead holder shouldn't keep
|
|
942
|
+
// us waiting MAX_RETRIES * backoff before we recover.
|
|
943
|
+
try {
|
|
944
|
+
const stat = fs.statSync(lockPath);
|
|
945
|
+
if (Date.now() - stat.mtimeMs > STALE_LOCK_MS) {
|
|
946
|
+
try { fs.unlinkSync(lockPath); } catch {}
|
|
947
|
+
continue; // retry immediately without sleeping
|
|
948
|
+
}
|
|
949
|
+
} catch {} // lockfile vanished between EEXIST and stat — fine, retry
|
|
950
|
+
await new Promise((r) => setTimeout(r, 50 + Math.random() * 150));
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
if (!acquired) {
|
|
954
|
+
throw new Error(`withCatalogLock: could not acquire ${lockPath} after ${MAX_RETRIES} attempts`);
|
|
955
|
+
}
|
|
956
|
+
try {
|
|
957
|
+
const catalog = JSON.parse(fs.readFileSync(catalogPath, "utf8"));
|
|
958
|
+
const mutated = await mutator(catalog);
|
|
959
|
+
const toWrite = mutated === undefined ? catalog : mutated;
|
|
960
|
+
writeJsonAtomic(catalogPath, toWrite);
|
|
961
|
+
return { wrote: true, result: toWrite };
|
|
962
|
+
} finally {
|
|
963
|
+
try { fs.unlinkSync(lockPath); } catch {}
|
|
964
|
+
}
|
|
829
965
|
}
|
|
830
966
|
|
|
831
967
|
function chosenSources(opts) {
|
|
@@ -834,8 +970,13 @@ function chosenSources(opts) {
|
|
|
834
970
|
const out = [];
|
|
835
971
|
for (const n of names) {
|
|
836
972
|
if (!ALL_SOURCES[n]) {
|
|
837
|
-
|
|
838
|
-
|
|
973
|
+
// v0.12.12 C3: previously `process.exit(2)` after a console.error.
|
|
974
|
+
// Stdout writes elsewhere in this run could truncate; throwing lets
|
|
975
|
+
// main().catch() surface the error through the standard channel and
|
|
976
|
+
// exit code via process.exitCode + natural event-loop drain.
|
|
977
|
+
const err = new Error(`refresh-external: unknown source "${n}". Valid: ${Object.keys(ALL_SOURCES).join(", ")}`);
|
|
978
|
+
err._exceptd_unknown_source = true;
|
|
979
|
+
throw err;
|
|
839
980
|
}
|
|
840
981
|
out.push(ALL_SOURCES[n]);
|
|
841
982
|
}
|
|
@@ -939,18 +1080,29 @@ async function seedSingleAdvisory(opts) {
|
|
|
939
1080
|
|
|
940
1081
|
// Apply: write to cve-catalog.json with the _auto_imported flag.
|
|
941
1082
|
// v0.12.8: honor --catalog / EXCEPTD_CVE_CATALOG so tests can redirect.
|
|
1083
|
+
// v0.12.12 C1: lock-gated RMW. Without this, two concurrent
|
|
1084
|
+
// `refresh --advisory CVE-A --apply` + `--advisory CVE-B --apply`
|
|
1085
|
+
// processes against the same catalog silently dropped one CVE 1-in-20
|
|
1086
|
+
// trials (read-old → mutate → write-overwrites-sibling-mutation).
|
|
942
1087
|
const catalogPath = resolveCatalogPath(opts);
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
1088
|
+
let humanCurated = null;
|
|
1089
|
+
await withCatalogLock(catalogPath, (catalog) => {
|
|
1090
|
+
if (catalog[cveId] && !catalog[cveId]._auto_imported && !catalog[cveId]._draft) {
|
|
1091
|
+
// Refuse to overwrite a human-curated entry — signal via closure so
|
|
1092
|
+
// we can emit the structured error after the lock releases.
|
|
1093
|
+
humanCurated = { last_updated: catalog[cveId].last_updated };
|
|
1094
|
+
return catalog; // unchanged write — idempotent, releases lock
|
|
1095
|
+
}
|
|
1096
|
+
catalog[cveId] = normalized[cveId];
|
|
1097
|
+
return catalog;
|
|
1098
|
+
});
|
|
1099
|
+
if (humanCurated) {
|
|
1100
|
+
const err = { ok: false, verb: "refresh", error: `${cveId} already present in catalog and is human-curated (not a draft). Refusing to overwrite. Edit manually if intentional.`, existing_last_updated: humanCurated.last_updated };
|
|
947
1101
|
if (opts.json) process.stdout.write(JSON.stringify(err) + "\n");
|
|
948
1102
|
else process.stderr.write(`[refresh --advisory] ${err.error}\n`);
|
|
949
1103
|
process.exitCode = 4;
|
|
950
1104
|
return;
|
|
951
1105
|
}
|
|
952
|
-
catalog[cveId] = normalized[cveId];
|
|
953
|
-
fs.writeFileSync(catalogPath, JSON.stringify(catalog, null, 2) + "\n", "utf8");
|
|
954
1106
|
const output = {
|
|
955
1107
|
ok: true,
|
|
956
1108
|
verb: "refresh",
|
|
@@ -971,7 +1123,9 @@ async function main() {
|
|
|
971
1123
|
const opts = parseArgs(process.argv);
|
|
972
1124
|
if (opts.help) {
|
|
973
1125
|
printHelp();
|
|
974
|
-
|
|
1126
|
+
// v0.12.12 C3: exitCode + return so buffered stdout flushes naturally.
|
|
1127
|
+
process.exitCode = 0;
|
|
1128
|
+
return;
|
|
975
1129
|
}
|
|
976
1130
|
|
|
977
1131
|
// v0.12.0: `--advisory <id>` short-circuits the normal source loop and
|
|
@@ -1068,7 +1222,12 @@ async function main() {
|
|
|
1068
1222
|
}
|
|
1069
1223
|
}
|
|
1070
1224
|
|
|
1071
|
-
|
|
1225
|
+
// v0.12.12 C3: same anti-pattern v0.12.9 fixed in prefetch's main(). After
|
|
1226
|
+
// Promise.all(sources.map(runOne)) in --swarm mode, process.exit() could
|
|
1227
|
+
// truncate buffered stdout (refresh-report path log line, summary log
|
|
1228
|
+
// lines piped to a consumer). exitCode + return lets the event loop end
|
|
1229
|
+
// naturally and stdout drains in full.
|
|
1230
|
+
process.exitCode = hadFailure ? 1 : 0;
|
|
1072
1231
|
}
|
|
1073
1232
|
|
|
1074
1233
|
async function sequential(items, fn) {
|
|
@@ -1084,11 +1243,18 @@ if (require.main === module) {
|
|
|
1084
1243
|
if (err && err._exceptd_hint) {
|
|
1085
1244
|
console.error(err.message);
|
|
1086
1245
|
console.error(JSON.stringify({ ok: false, error: err.message.split("\n")[0], hint: err.message.split("\n").slice(1).join(" ").trim(), verb: "refresh" }));
|
|
1246
|
+
} else if (err && err._exceptd_unknown_source) {
|
|
1247
|
+
// v0.12.12 C3: surface the source-validation error without leaking a
|
|
1248
|
+
// stack trace; chosenSources throws this for unknown --source values.
|
|
1249
|
+
console.error(err.message);
|
|
1087
1250
|
} else {
|
|
1088
1251
|
console.error(`refresh-external: fatal: ${err && err.stack ? err.stack : err}`);
|
|
1089
1252
|
}
|
|
1090
|
-
process.exit(2)
|
|
1253
|
+
// v0.12.12 C3: exitCode + return rather than process.exit(2) — the
|
|
1254
|
+
// event loop has no further work after main()'s rejection, so this
|
|
1255
|
+
// ends the process with code 2 but lets stderr drain first.
|
|
1256
|
+
process.exitCode = 2;
|
|
1091
1257
|
});
|
|
1092
1258
|
}
|
|
1093
1259
|
|
|
1094
|
-
module.exports = { ALL_SOURCES, loadCtx, parseArgs, seedSingleAdvisory };
|
|
1260
|
+
module.exports = { ALL_SOURCES, loadCtx, parseArgs, seedSingleAdvisory, withCatalogLock, writeJsonAtomic };
|