@blamejs/exceptd-skills 0.19.32 → 0.19.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/bin/exceptd.js +896 -2824
- package/data/_indexes/_meta.json +8 -8
- package/data/_indexes/activity-feed.json +2 -2
- package/data/_indexes/catalog-summaries.json +7 -7
- package/data/_indexes/chains.json +60118 -0
- package/data/attack-techniques.json +267 -7
- package/data/cve-catalog.json +9991 -3
- package/data/cwe-catalog.json +109 -2
- package/data/framework-control-gaps.json +578 -3
- package/data/zeroday-lessons.json +8330 -1
- package/lib/auto-discovery.js +56 -286
- package/lib/canonical-eq.js +7 -40
- package/lib/citation-resolve.js +22 -70
- package/lib/collectors/ai-api.js +20 -54
- package/lib/collectors/cicd-pipeline-compromise.js +40 -108
- package/lib/collectors/citation-hygiene.js +72 -210
- package/lib/collectors/containers.js +41 -130
- package/lib/collectors/cred-stores.js +31 -115
- package/lib/collectors/crypto-codebase.js +55 -138
- package/lib/collectors/crypto.js +24 -54
- package/lib/collectors/hardening.js +20 -78
- package/lib/collectors/kernel.js +16 -46
- package/lib/collectors/library-author.js +57 -206
- package/lib/collectors/mcp.js +24 -70
- package/lib/collectors/runtime.js +24 -86
- package/lib/collectors/sbom.js +34 -106
- package/lib/collectors/scan-excludes.js +31 -138
- package/lib/collectors/secrets.js +62 -178
- package/lib/cross-ref-api.js +39 -123
- package/lib/currency-severity.js +8 -27
- package/lib/cve-batch.js +13 -21
- package/lib/cve-cli.js +13 -20
- package/lib/cve-curation.js +72 -239
- package/lib/cve-regression-watcher.js +29 -152
- package/lib/cvss.js +13 -54
- package/lib/doctor-bucketing.js +3 -19
- package/lib/exit-codes.js +10 -42
- package/lib/flag-suggest.js +7 -25
- package/lib/framework-gap.js +35 -114
- package/lib/gap-detectors.js +37 -159
- package/lib/id-validation.js +9 -30
- package/lib/job-queue.js +13 -36
- package/lib/lint-skills.js +64 -232
- package/lib/playbook-runner.js +693 -2095
- package/lib/prefetch.js +100 -376
- package/lib/refresh-external.js +199 -627
- package/lib/refresh-network.js +75 -307
- package/lib/rfc-cli.js +23 -68
- package/lib/scoring.js +77 -145
- package/lib/sign.js +43 -229
- package/lib/source-advisories.js +43 -194
- package/lib/source-ghsa.js +37 -120
- package/lib/source-osv.js +94 -266
- package/lib/ttp-mapper.js +14 -24
- package/lib/upstream-check-cli.js +10 -28
- package/lib/upstream-check.js +19 -44
- package/lib/validate-catalog-meta.js +17 -61
- package/lib/validate-cve-catalog.js +43 -119
- package/lib/validate-indexes.js +25 -76
- package/lib/validate-package.js +16 -62
- package/lib/validate-playbooks.js +69 -275
- package/lib/validate-vendor.js +16 -49
- package/lib/verify.js +56 -286
- package/lib/version-pins.js +5 -34
- package/lib/worker-pool.js +11 -30
- package/lib/xml-tokenizer.js +47 -152
- package/manifest.json +53 -53
- package/orchestrator/dispatcher.js +17 -68
- package/orchestrator/event-bus.js +11 -74
- package/orchestrator/index.js +138 -412
- package/orchestrator/pipeline.js +28 -85
- package/orchestrator/scanner.js +34 -138
- package/orchestrator/scheduler.js +20 -84
- package/package.json +2 -2
- package/sbom.cdx.json +253 -253
- package/scripts/audit-catalog-gaps.js +9 -62
- package/scripts/audit-cross-skill.js +5 -31
- package/scripts/audit-perf.js +6 -16
- package/scripts/backfill-theater-test.js +7 -64
- package/scripts/bootstrap.js +12 -44
- package/scripts/build-indexes.js +40 -154
- package/scripts/builders/activity-feed.js +4 -14
- package/scripts/builders/catalog-summaries.js +3 -10
- package/scripts/builders/currency.js +7 -20
- package/scripts/builders/cwe-chains.js +7 -30
- package/scripts/builders/did-ladders.js +6 -13
- package/scripts/builders/frequency.js +5 -19
- package/scripts/builders/jurisdiction-clocks.js +6 -25
- package/scripts/builders/recipes.js +6 -14
- package/scripts/builders/section-offsets.js +13 -51
- package/scripts/builders/stale-content.js +7 -28
- package/scripts/builders/summary-cards.js +8 -29
- package/scripts/builders/theater-fingerprints.js +12 -27
- package/scripts/builders/token-budget.js +4 -31
- package/scripts/check-agents-md-collectors.js +11 -54
- package/scripts/check-catalog-gap-budget.js +15 -32
- package/scripts/check-changelog-extract.js +18 -48
- package/scripts/check-codebase-patterns-currency.js +6 -22
- package/scripts/check-codebase-patterns.js +50 -143
- package/scripts/check-epss-consistency.js +9 -64
- package/scripts/check-framework-gap-coverage.js +13 -31
- package/scripts/check-manifest-snapshot.js +13 -73
- package/scripts/check-sbom-currency.js +44 -142
- package/scripts/check-test-count.js +15 -52
- package/scripts/check-test-coverage.js +66 -197
- package/scripts/check-test-subjects.js +21 -62
- package/scripts/check-ttp-references.js +14 -38
- package/scripts/check-ttp-upstream.js +8 -40
- package/scripts/check-version-bump.js +9 -61
- package/scripts/check-version-tags.js +20 -121
- package/scripts/predeploy.js +38 -184
- package/scripts/refresh-manifest-snapshot.js +16 -38
- package/scripts/refresh-mitre-atlas.js +3 -8
- package/scripts/refresh-mitre-attack.js +1 -8
- package/scripts/refresh-mitre-d3fend.js +3 -9
- package/scripts/refresh-mitre-ics-attack.js +3 -8
- package/scripts/refresh-reverse-refs.js +27 -94
- package/scripts/refresh-rfc-index.js +2 -10
- package/scripts/refresh-sbom.js +31 -161
- package/scripts/refresh-upstream-catalogs.js +40 -137
- package/scripts/release.js +69 -232
- package/scripts/run-e2e-scenarios.js +24 -71
- package/scripts/sync-manifest-metadata.js +10 -34
- package/scripts/sync-package-description.js +8 -17
- package/scripts/validate-vendor-online.js +13 -44
- package/scripts/verify-shipped-tarball.js +35 -140
package/scripts/bootstrap.js
CHANGED
|
@@ -2,45 +2,17 @@
|
|
|
2
2
|
'use strict';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* Bootstrap ceremony. The mode is auto-detected, so a downstream consumer
|
|
6
|
+
* running it cannot invalidate the maintainer's signing key:
|
|
6
7
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
8
|
+
* public key only verify only — generates nothing, rewrites nothing
|
|
9
|
+
* private key present re-sign every skill, then verify
|
|
10
|
+
* no keys, or --init generate an Ed25519 keypair, sign, verify
|
|
10
11
|
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* key + signed manifest. Running `npm run bootstrap` here just confirms
|
|
14
|
-
* the working tree is intact. No keypair is generated; no signatures are
|
|
15
|
-
* rewritten. This is the safe default.
|
|
12
|
+
* The private key never leaves the maintainer's machine. keys/public.pem is the
|
|
13
|
+
* one tracked artifact, committed after first init.
|
|
16
14
|
*
|
|
17
|
-
*
|
|
18
|
-
* after editing skill content. Re-signs every skill with the existing
|
|
19
|
-
* private key, then verifies.
|
|
20
|
-
*
|
|
21
|
-
* 3. First-maintainer init (no keys/public.pem, OR --init explicitly passed)
|
|
22
|
-
* — GENERATE + SIGN + VERIFY. Used once when a maintainer sets up signing
|
|
23
|
-
* for a brand-new clone. Generates an Ed25519 keypair, signs every skill,
|
|
24
|
-
* and verifies. The new public key is committed; the private key stays in
|
|
25
|
-
* .keys/ (gitignored).
|
|
26
|
-
*
|
|
27
|
-
* The private key never leaves the maintainer's machine. The public key in
|
|
28
|
-
* keys/public.pem is the one tracked artifact and is committed by the
|
|
29
|
-
* maintainer after first init.
|
|
30
|
-
*
|
|
31
|
-
* Subprocesses use execFileSync (no shell) with argument arrays — there is no
|
|
32
|
-
* user input on the path, and avoiding the shell removes the injection surface
|
|
33
|
-
* regardless.
|
|
34
|
-
*
|
|
35
|
-
* Usage:
|
|
36
|
-
* node scripts/bootstrap.js Auto-detect mode and run.
|
|
37
|
-
* node scripts/bootstrap.js --init Force first-maintainer init (generate
|
|
38
|
-
* keypair + sign + verify).
|
|
39
|
-
* node scripts/bootstrap.js --force Re-run even if marker exists.
|
|
40
|
-
* node scripts/bootstrap.js --help Print this help text.
|
|
41
|
-
*
|
|
42
|
-
* Dependencies: Node 24 stdlib only. package.json has no runtime deps and
|
|
43
|
-
* this script keeps it that way.
|
|
15
|
+
* Node stdlib only: package.json has no runtime deps and this keeps it so.
|
|
44
16
|
*/
|
|
45
17
|
|
|
46
18
|
const fs = require('node:fs');
|
|
@@ -113,8 +85,8 @@ function run(label, scriptPath, scriptArgs) {
|
|
|
113
85
|
const pretty = `node ${path.relative(ROOT, scriptPath)} ${scriptArgs.join(' ')}`.trim();
|
|
114
86
|
console.log(`[bootstrap] ${label}: ${pretty}`);
|
|
115
87
|
try {
|
|
116
|
-
// execFileSync
|
|
117
|
-
//
|
|
88
|
+
// execFileSync, never a shell: the path and args are repo constants passed
|
|
89
|
+
// as an array, so no injection surface exists.
|
|
118
90
|
childProcess.execFileSync(process.execPath, [scriptPath, ...scriptArgs], {
|
|
119
91
|
cwd: ROOT,
|
|
120
92
|
stdio: 'inherit'
|
|
@@ -153,10 +125,8 @@ function main() {
|
|
|
153
125
|
const mode = detectMode(args);
|
|
154
126
|
|
|
155
127
|
if (mode === 'verify-only') {
|
|
156
|
-
//
|
|
157
|
-
//
|
|
158
|
-
// integrity — never generates or signs, which would invalidate the
|
|
159
|
-
// upstream maintainer's signing chain.
|
|
128
|
+
// Confirms tree integrity and nothing more. Generating or signing here
|
|
129
|
+
// would invalidate the upstream maintainer's signing chain.
|
|
160
130
|
console.log('[bootstrap] Detected downstream-consumer state:');
|
|
161
131
|
console.log(' - keys/public.pem present (shipped by maintainer)');
|
|
162
132
|
console.log(' - .keys/private.pem absent');
|
|
@@ -170,8 +140,6 @@ function main() {
|
|
|
170
140
|
}
|
|
171
141
|
|
|
172
142
|
if (mode === 'resign') {
|
|
173
|
-
// Maintainer re-sign path. Private key already exists; re-sign every
|
|
174
|
-
// skill against the current content and verify.
|
|
175
143
|
console.log('[bootstrap] Detected maintainer re-sign state (private key present).');
|
|
176
144
|
console.log('[bootstrap] Re-signing every skill with the existing private key.');
|
|
177
145
|
console.log();
|
package/scripts/build-indexes.js
CHANGED
|
@@ -1,56 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* to answer routine cross-reference questions.
|
|
8
|
-
*
|
|
9
|
-
* Outputs (17 total):
|
|
10
|
-
* xref.json — inverted index over cwe/d3fend/framework_gap/
|
|
11
|
-
* atlas/attack/rfc/dlp citations
|
|
12
|
-
* trigger-table.json — flat trigger → [skills]
|
|
13
|
-
* chains.json — pre-computed cross-walks per CVE and per CWE
|
|
14
|
-
* jurisdiction-map.json — jurisdiction → skills that reference it
|
|
15
|
-
* handoff-dag.json — cross-skill mention graph
|
|
16
|
-
* summary-cards.json — per-skill 100-word abstract
|
|
17
|
-
* section-offsets.json — per-skill byte/line offsets of every H2
|
|
18
|
-
* token-budget.json — approximate token cost per skill + section
|
|
19
|
-
* recipes.json — curated multi-skill recipes
|
|
20
|
-
* jurisdiction-clocks.json — normalized obligation × hours matrix
|
|
21
|
-
* did-ladders.json — canonical defense-in-depth ladders
|
|
22
|
-
* theater-fingerprints.json — structured compliance-theater pattern records
|
|
23
|
-
* currency.json — pre-computed skill currency snapshot
|
|
24
|
-
* frequency.json — citation-count tables per catalog field
|
|
25
|
-
* activity-feed.json — "what changed when" feed
|
|
26
|
-
* catalog-summaries.json — compact per-catalog summary cards
|
|
27
|
-
* stale-content.json — persisted stale-content findings
|
|
28
|
-
* _meta.json — SHA-256 of every source file for staleness
|
|
29
|
-
*
|
|
30
|
-
* Flags:
|
|
31
|
-
* (default) build all outputs
|
|
32
|
-
* --only <names> build only the comma-separated outputs (and
|
|
33
|
-
* anything they depend on)
|
|
34
|
-
* --changed build only outputs whose declared deps changed
|
|
35
|
-
* since the last _meta.json snapshot. Safe in CI:
|
|
36
|
-
* identical inputs always produce identical outputs.
|
|
37
|
-
* --parallel run independent builders concurrently via
|
|
38
|
-
* Promise.all (I/O concurrency, no worker threads).
|
|
39
|
-
* For CPU-bound fan-out, callers can compose with
|
|
40
|
-
* lib/worker-pool.js directly.
|
|
41
|
-
* --quiet suppress per-output log lines
|
|
42
|
-
*
|
|
43
|
-
* Re-build conditions:
|
|
44
|
-
* _meta.json records sha256 of every source file. validate-indexes
|
|
45
|
-
* (predeploy gate) re-hashes those and fails if any source changed
|
|
46
|
-
* after the last build. --changed reads that same table to decide what
|
|
47
|
-
* to rebuild.
|
|
48
|
-
*
|
|
49
|
-
* Index file naming convention: leading underscore marks them as derived
|
|
50
|
-
* (mirroring `_meta` in catalog files), so anyone scanning `data/` for
|
|
51
|
-
* primary data filters them out.
|
|
52
|
-
*
|
|
53
|
-
* Node 24 stdlib only — zero npm deps.
|
|
3
|
+
* Produces the pre-computed indexes under `data/_indexes/`. Identical inputs must
|
|
4
|
+
* always produce identical outputs: --changed and the validate-indexes predeploy
|
|
5
|
+
* gate both key on the sha256 table in _meta.json, so a no-op run must not
|
|
6
|
+
* re-stamp.
|
|
54
7
|
*/
|
|
55
8
|
|
|
56
9
|
const fs = require("fs");
|
|
@@ -69,20 +22,13 @@ function sha256(buf) {
|
|
|
69
22
|
}
|
|
70
23
|
|
|
71
24
|
function writeJson(name, obj) {
|
|
72
|
-
//
|
|
73
|
-
// a truncated JSON output on disk. Write to a temp sibling and rename — rename
|
|
74
|
-
// is atomic on POSIX and Windows, so a reader (or the next build's
|
|
75
|
-
// parse-check) only ever sees the complete old or complete new file.
|
|
25
|
+
// Temp sibling then rename, so a crash mid-write cannot leave a truncated index.
|
|
76
26
|
const abs = path.join(IDX, name);
|
|
77
|
-
//
|
|
78
|
-
// a pid can't race the same tmp path, and the in-repo temp name isn't
|
|
79
|
-
// predictable. Mirrors lib/citation-resolve.js cachePut.
|
|
27
|
+
// The random suffix keeps a --parallel fan-out sharing one pid off this path.
|
|
80
28
|
const tmp = `${abs}.tmp-${process.pid}.${crypto.randomBytes(4).toString("hex")}`;
|
|
81
29
|
fs.writeFileSync(tmp, JSON.stringify(obj, null, 2) + "\n", "utf8");
|
|
82
|
-
//
|
|
83
|
-
//
|
|
84
|
-
// EPERM/EACCES/EBUSY. Retry with short backoff so a build doesn't fail on a
|
|
85
|
-
// lock that clears in milliseconds; on POSIX the first attempt always wins.
|
|
30
|
+
// On Windows a sync client or indexer holding the target open makes the rename
|
|
31
|
+
// transiently EPERM/EACCES/EBUSY, so it is retried with backoff.
|
|
86
32
|
let lastErr;
|
|
87
33
|
for (let attempt = 0; attempt < 10; attempt++) {
|
|
88
34
|
try { fs.renameSync(tmp, abs); return; }
|
|
@@ -92,7 +38,6 @@ function writeJson(name, obj) {
|
|
|
92
38
|
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20 * (attempt + 1));
|
|
93
39
|
}
|
|
94
40
|
}
|
|
95
|
-
// Persistent failure: drop the temp sibling so it doesn't orphan, then surface.
|
|
96
41
|
try { fs.unlinkSync(tmp); } catch { /* best effort */ }
|
|
97
42
|
throw lastErr;
|
|
98
43
|
}
|
|
@@ -133,15 +78,9 @@ Examples:
|
|
|
133
78
|
`);
|
|
134
79
|
}
|
|
135
80
|
|
|
136
|
-
//
|
|
137
|
-
|
|
138
|
-
//
|
|
139
|
-
// cache of these, but the skill frontmatter is the authoritative source — the
|
|
140
|
-
// linter and staleness gate read frontmatter. Overlaying the parsed
|
|
141
|
-
// frontmatter onto each skill record here means the indexes reflect the skill
|
|
142
|
-
// bodies even when the manifest cache has drifted (e.g. dropping UK-CAF / AU
|
|
143
|
-
// control mappings from framework_gaps). Array fields are overlaid only when
|
|
144
|
-
// present in frontmatter; description is a scalar.
|
|
81
|
+
// Skill frontmatter, not the manifest cache, is authoritative for these fields,
|
|
82
|
+
// so the indexes stay aligned with the skill bodies when the cache drifts. A
|
|
83
|
+
// field is overlaid only when frontmatter carries it.
|
|
145
84
|
const FRONTMATTER_ARRAY_FIELDS = [
|
|
146
85
|
"framework_gaps", "d3fend_refs", "cwe_refs", "atlas_refs",
|
|
147
86
|
"attack_refs", "rfc_refs", "triggers", "data_deps",
|
|
@@ -169,11 +108,8 @@ function loadSources() {
|
|
|
169
108
|
const skillBodies = {};
|
|
170
109
|
for (const s of manifest.skills) skillBodies[s.name] = fs.readFileSync(ABS(s.path), "utf8");
|
|
171
110
|
|
|
172
|
-
//
|
|
173
|
-
//
|
|
174
|
-
// dlp_refs, etc.). Downstream builders read cross-reference arrays from
|
|
175
|
-
// these records, so this is the single point that keeps the indexes aligned
|
|
176
|
-
// with the skill bodies.
|
|
111
|
+
// Frontmatter first, manifest cache for the fields frontmatter doesn't carry
|
|
112
|
+
// (signatures, dlp_refs). Every downstream builder reads these records.
|
|
177
113
|
const skills = manifest.skills.map((s) => authoritativeSkill(s, skillBodies[s.name]));
|
|
178
114
|
const skillNames = new Set(skills.map((s) => s.name));
|
|
179
115
|
|
|
@@ -196,15 +132,10 @@ function loadSources() {
|
|
|
196
132
|
return ctx;
|
|
197
133
|
}
|
|
198
134
|
|
|
199
|
-
//
|
|
200
|
-
//
|
|
201
|
-
//
|
|
202
|
-
//
|
|
203
|
-
// output. The --changed planner walks every changed source and
|
|
204
|
-
// flags every output whose deps match.
|
|
205
|
-
// dependsOn: names of other outputs that must be built first (used by
|
|
206
|
-
// chains.json which composes CVE + CWE halves, and
|
|
207
|
-
// token-budget.json which consumes section-offsets).
|
|
135
|
+
// Registry entry: { name, file, deps, build, dependsOn?: [name, ...] }
|
|
136
|
+
// deps predicates over a relative source path; --changed rebuilds an
|
|
137
|
+
// output when any changed source matches one.
|
|
138
|
+
// dependsOn output names built first, because this builder reads their file.
|
|
208
139
|
|
|
209
140
|
function isAnySkillBody(p) { return p.startsWith("skills/") && p.endsWith("/skill.md"); }
|
|
210
141
|
function isManifest(p) { return p === "manifest.json"; }
|
|
@@ -291,12 +222,8 @@ const OUTPUTS = [
|
|
|
291
222
|
for (const s of ctx.skills) {
|
|
292
223
|
const body = ctx.skillBodies[s.name];
|
|
293
224
|
for (const code of codes) {
|
|
294
|
-
//
|
|
295
|
-
//
|
|
296
|
-
// "US-based") and control/countermeasure id grammar (`\bCA\b` matches
|
|
297
|
-
// inside `D3-CA`, `\bSA\b` inside `SA-12`), polluting coverage
|
|
298
|
-
// (Indonesia landed on 41/51 skills). These jurisdictions are mapped
|
|
299
|
-
// via the curated NAME_TO_CODE regulation-name table below instead.
|
|
225
|
+
// Bare 2-letter ISO codes never free-text match: `\bCA\b` collides with
|
|
226
|
+
// prose and with control-id grammar. They reach the map via NAME_TO_CODE.
|
|
300
227
|
if (code.length <= 2) continue;
|
|
301
228
|
const re = new RegExp("\\b" + code + "\\b");
|
|
302
229
|
if (re.test(body) && !out[code].skills.includes(s.name)) out[code].skills.push(s.name);
|
|
@@ -361,9 +288,8 @@ const OUTPUTS = [
|
|
|
361
288
|
entry_types: ["CVE", "CWE"],
|
|
362
289
|
},
|
|
363
290
|
};
|
|
364
|
-
//
|
|
365
|
-
//
|
|
366
|
-
// reflect the same operator-queryable, curated truth.
|
|
291
|
+
// Drafts are skipped so this agrees with the reverse-ref index
|
|
292
|
+
// (scripts/refresh-reverse-refs.js), which excludes them too.
|
|
367
293
|
for (const cveId of Object.keys(ctx.cveCatalog).filter((k) => !k.startsWith("_") && ctx.cveCatalog[k] && ctx.cveCatalog[k]._draft !== true)) {
|
|
368
294
|
const cve = ctx.cveCatalog[cveId];
|
|
369
295
|
const referencingSkills = ctx.skills
|
|
@@ -399,7 +325,6 @@ const OUTPUTS = [
|
|
|
399
325
|
referencing_skills: referencingSkills, chain: hydrated,
|
|
400
326
|
};
|
|
401
327
|
}
|
|
402
|
-
// CWE half (delegated to builder)
|
|
403
328
|
const cweChains = buildCweChains({
|
|
404
329
|
skills: ctx.skills, cweCatalog: ctx.cweCatalog, atlasTtps: ctx.atlasTtps,
|
|
405
330
|
cveCatalog: ctx.cveCatalog, frameworkGaps: ctx.frameworkGaps,
|
|
@@ -437,8 +362,7 @@ const OUTPUTS = [
|
|
|
437
362
|
dependsOn: ["section-offsets"], // needs the produced index
|
|
438
363
|
build: (ctx) => {
|
|
439
364
|
const { buildTokenBudget } = require("./builders/token-budget");
|
|
440
|
-
//
|
|
441
|
-
// dependency planner). Read it back for the token splitter.
|
|
365
|
+
// Already on disk: the dependency planner builds section-offsets first.
|
|
442
366
|
const sectionOffsets = readJson(path.join(IDX, "section-offsets.json"));
|
|
443
367
|
return buildTokenBudget({ root: ctx.root, skills: ctx.skills, sectionOffsets });
|
|
444
368
|
},
|
|
@@ -541,8 +465,6 @@ const OUTPUTS = [
|
|
|
541
465
|
},
|
|
542
466
|
];
|
|
543
467
|
|
|
544
|
-
// --- Plan + run --------------------------------------------------------
|
|
545
|
-
|
|
546
468
|
function loadPriorMeta() {
|
|
547
469
|
const p = path.join(IDX, "_meta.json");
|
|
548
470
|
if (!fs.existsSync(p)) return null;
|
|
@@ -552,9 +474,7 @@ function loadPriorMeta() {
|
|
|
552
474
|
function liveSourceSet(ctx) {
|
|
553
475
|
const out = new Set();
|
|
554
476
|
out.add("manifest.json");
|
|
555
|
-
// README.md
|
|
556
|
-
// it must be a hashed source — otherwise a README edit is invisible to
|
|
557
|
-
// --changed and the validate-indexes freshness gate.
|
|
477
|
+
// README.md feeds the stale-content builder, so it has to be a hashed source.
|
|
558
478
|
if (fs.existsSync(ABS("README.md"))) out.add("README.md");
|
|
559
479
|
for (const c of ctx.catalogFiles) out.add(c);
|
|
560
480
|
for (const s of ctx.skills) out.add(s.path);
|
|
@@ -562,9 +482,7 @@ function liveSourceSet(ctx) {
|
|
|
562
482
|
}
|
|
563
483
|
|
|
564
484
|
function changedSources(ctx, priorMeta) {
|
|
565
|
-
//
|
|
566
|
-
// every source if there's no prior meta. Also accounts for new + removed
|
|
567
|
-
// source files (which always force a rebuild).
|
|
485
|
+
// Every source when there is no prior meta; an appeared or disappeared one counts.
|
|
568
486
|
if (!priorMeta || !priorMeta.source_hashes) {
|
|
569
487
|
return [...liveSourceSet(ctx)];
|
|
570
488
|
}
|
|
@@ -575,7 +493,6 @@ function changedSources(ctx, priorMeta) {
|
|
|
575
493
|
const h = sha256(fs.readFileSync(ABS(p)));
|
|
576
494
|
if (priorMeta.source_hashes[p] !== h) changed.push(p);
|
|
577
495
|
}
|
|
578
|
-
// Any source that disappeared since last build counts as a change.
|
|
579
496
|
for (const p of recorded) if (!live.has(p)) changed.push(p);
|
|
580
497
|
return changed;
|
|
581
498
|
}
|
|
@@ -590,12 +507,8 @@ function outputsAffectedBy(changedPaths) {
|
|
|
590
507
|
return affected;
|
|
591
508
|
}
|
|
592
509
|
|
|
593
|
-
//
|
|
594
|
-
//
|
|
595
|
-
// left unparseable must be regenerated even when every source is byte-identical
|
|
596
|
-
// — otherwise the no-op path leaves a broken tree while writeMeta records it as
|
|
597
|
-
// fresh (and reports a 0 count for the file it never rebuilt). Mirrors the
|
|
598
|
-
// existence/parse check in lib/validate-indexes.js verifyOutputs.
|
|
510
|
+
// A deleted, truncated or unparseable output is a rebuild trigger on its own:
|
|
511
|
+
// --changed sees only source-hash deltas, so writeMeta would record it as fresh.
|
|
599
512
|
function outputsMissingOrCorrupt() {
|
|
600
513
|
const broken = new Set();
|
|
601
514
|
for (const o of OUTPUTS) {
|
|
@@ -610,7 +523,6 @@ function outputsMissingOrCorrupt() {
|
|
|
610
523
|
}
|
|
611
524
|
|
|
612
525
|
function withDependencyClosure(names) {
|
|
613
|
-
// Pull in any dependsOn entries (e.g. token-budget needs section-offsets).
|
|
614
526
|
const closure = new Set(names);
|
|
615
527
|
let added = true;
|
|
616
528
|
while (added) {
|
|
@@ -644,12 +556,10 @@ function topoOrder(names) {
|
|
|
644
556
|
}
|
|
645
557
|
|
|
646
558
|
async function runBuilders(ctx, names, opts) {
|
|
647
|
-
// Build the dependency-respecting execution plan, then dispatch.
|
|
648
559
|
const order = topoOrder(names);
|
|
649
560
|
const log = (s) => opts.quiet || console.log(s);
|
|
650
561
|
|
|
651
|
-
//
|
|
652
|
-
// outputs depending on those, etc. This is the parallelization unit.
|
|
562
|
+
// A level holds outputs with no unbuilt dependsOn; --parallel fans out over one.
|
|
653
563
|
const remaining = new Set(order);
|
|
654
564
|
const levels = [];
|
|
655
565
|
while (remaining.size > 0) {
|
|
@@ -693,13 +603,9 @@ function writeMeta(ctx, results, opts = {}) {
|
|
|
693
603
|
const computedSourceHashes = {};
|
|
694
604
|
for (const p of sourceFiles) computedSourceHashes[p] = sha256(fs.readFileSync(ABS(p)));
|
|
695
605
|
|
|
696
|
-
// A `--only` build
|
|
697
|
-
//
|
|
698
|
-
//
|
|
699
|
-
// that case, so validate-indexes still detects drift and demands a full
|
|
700
|
-
// rebuild — overwriting them with current hashes would fail OPEN, recording
|
|
701
|
-
// stale outputs as fresh. (--changed rebuilds exactly the change-affected
|
|
702
|
-
// outputs and a full build rebuilds all, so their current hashes are accurate.)
|
|
606
|
+
// A `--only` build touches an arbitrary subset, so keeping the prior hashes
|
|
607
|
+
// lets validate-indexes still demand a full rebuild. Writing current hashes
|
|
608
|
+
// here fails open, recording stale outputs as fresh.
|
|
703
609
|
const partial = !!(opts && opts.only);
|
|
704
610
|
const sourceHashes = (partial && prior && prior.source_hashes && typeof prior.source_hashes === "object")
|
|
705
611
|
? prior.source_hashes
|
|
@@ -707,15 +613,8 @@ function writeMeta(ctx, results, opts = {}) {
|
|
|
707
613
|
|
|
708
614
|
const outputsList = OUTPUTS.map((o) => o.file).sort();
|
|
709
615
|
|
|
710
|
-
//
|
|
711
|
-
//
|
|
712
|
-
// validate-indexes keys freshness on `source_hashes`/`outputs`, never on the
|
|
713
|
-
// timestamp, so re-stamping a no-op run with a fresh wall-clock value adds
|
|
714
|
-
// no freshness signal and only produces a spurious git diff — contradicting
|
|
715
|
-
// the documented "identical inputs always produce identical outputs"
|
|
716
|
-
// contract for --changed (and the idempotence the predeploy gate relies on).
|
|
717
|
-
// Reuse the prior timestamp on a genuine no-op; mint a new one only when the
|
|
718
|
-
// hashed surface actually moved (or there is no prior meta to inherit from).
|
|
616
|
+
// `generated_at` carries over when the source hashes and the output set are
|
|
617
|
+
// identical to the last build, so a no-op run produces no git diff.
|
|
719
618
|
const sourcesUnchanged = prior && prior.source_hashes &&
|
|
720
619
|
JSON.stringify(prior.source_hashes) === JSON.stringify(sourceHashes) &&
|
|
721
620
|
JSON.stringify(prior.outputs || []) === JSON.stringify(outputsList);
|
|
@@ -723,8 +622,7 @@ function writeMeta(ctx, results, opts = {}) {
|
|
|
723
622
|
? prior.generated_at
|
|
724
623
|
: new Date().toISOString();
|
|
725
624
|
|
|
726
|
-
//
|
|
727
|
-
// (covers --only / --changed runs that didn't rebuild every output).
|
|
625
|
+
// In-memory result when this run built it, otherwise from disk.
|
|
728
626
|
function readBack(name) {
|
|
729
627
|
if (results[name]) return results[name];
|
|
730
628
|
const o = OUTPUTS.find((x) => x.name === name);
|
|
@@ -764,20 +662,16 @@ function writeMeta(ctx, results, opts = {}) {
|
|
|
764
662
|
source_hashes: sourceHashes,
|
|
765
663
|
skill_count: ctx.skills.length,
|
|
766
664
|
catalog_count: ctx.catalogFiles.length,
|
|
767
|
-
//
|
|
768
|
-
//
|
|
769
|
-
// deleted/truncated/corrupted OUTPUT (an index file removed from a clean
|
|
770
|
-
// source tree would otherwise pass the freshness gate as "current").
|
|
665
|
+
// Source hashes alone miss a deleted output: removed from a clean source
|
|
666
|
+
// tree, it would pass the freshness gate as current.
|
|
771
667
|
outputs: outputsList,
|
|
772
668
|
index_stats: {
|
|
773
669
|
xref_entries: xrefStats,
|
|
774
670
|
trigger_table_entries: Object.keys(trigger).length,
|
|
775
671
|
chains_cve_entries: cveChainCount,
|
|
776
672
|
chains_cwe_entries: cweChainCount,
|
|
777
|
-
//
|
|
778
|
-
//
|
|
779
|
-
// count of jurisdictions actually indexed. (`jurisdiction_clocks` below
|
|
780
|
-
// reports the clocks-only subset.)
|
|
673
|
+
// Counted off jurisdiction-map, which holds every code; `jurisdiction_clocks`
|
|
674
|
+
// below is the clocks-only subset.
|
|
781
675
|
jurisdictions_indexed: Object.keys(readBack("jurisdiction-map") || {}).length || Object.keys(jurisdictionClocks.by_jurisdiction || {}).length,
|
|
782
676
|
handoff_dag_nodes: handoff.nodes?.length || 0,
|
|
783
677
|
summary_cards: Object.keys(summaryCards.skills || {}).length,
|
|
@@ -806,7 +700,6 @@ async function main() {
|
|
|
806
700
|
const ctx = loadSources();
|
|
807
701
|
const log = (s) => opts.quiet || console.log(s);
|
|
808
702
|
|
|
809
|
-
// Decide which outputs to build.
|
|
810
703
|
let chosen;
|
|
811
704
|
if (opts.only) {
|
|
812
705
|
const wanted = opts.only.split(",").map((s) => s.trim()).filter(Boolean);
|
|
@@ -822,22 +715,15 @@ async function main() {
|
|
|
822
715
|
const changed = changedSources(ctx, prior);
|
|
823
716
|
log(`changed sources: ${changed.length}`);
|
|
824
717
|
const affected = outputsAffectedBy(changed);
|
|
825
|
-
//
|
|
826
|
-
// A deleted/corrupt output is a rebuild trigger independent of source
|
|
827
|
-
// hashes — without this, --changed reports "nothing to do" and writeMeta
|
|
828
|
-
// would claim freshness for a tree that's actually broken.
|
|
718
|
+
// Without this union, --changed reports "nothing to do" on a corrupt output.
|
|
829
719
|
const broken = outputsMissingOrCorrupt();
|
|
830
720
|
for (const name of broken) affected.add(name);
|
|
831
721
|
if (broken.size > 0) log(`missing/corrupt outputs to regenerate: ${[...broken].sort().join(", ")}`);
|
|
832
722
|
chosen = withDependencyClosure(affected);
|
|
833
723
|
if (chosen.size === 0) {
|
|
834
724
|
log("build-indexes: no outputs need rebuilding (sources unchanged)");
|
|
835
|
-
//
|
|
836
|
-
//
|
|
837
|
-
// writeMeta preserves the prior `generated_at` when the hashed surface is
|
|
838
|
-
// byte-identical, so a genuine no-op leaves the file unchanged — no
|
|
839
|
-
// spurious timestamp-only diff. Re-stamping would add no freshness signal
|
|
840
|
-
// (validate-indexes keys on source_hashes/outputs, not the timestamp).
|
|
725
|
+
// Still rewritten, so the hash table and outputs list self-heal against the
|
|
726
|
+
// live source set; a true no-op leaves the file byte-identical.
|
|
841
727
|
writeMeta(ctx, {}, opts);
|
|
842
728
|
return;
|
|
843
729
|
}
|
|
@@ -1,17 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* across skills and catalogs, sorted by date. Lightweight RSS for the
|
|
7
|
-
* skill corpus that consumers can poll without diff-ing the manifest.
|
|
8
|
-
*
|
|
9
|
-
* Combines:
|
|
10
|
-
* - per-skill last_threat_review
|
|
11
|
-
* - per-catalog _meta.last_updated
|
|
12
|
-
* - manifest threat_review_date + atlas_version_date when present
|
|
13
|
-
*
|
|
14
|
-
* Output sorted descending by date.
|
|
3
|
+
* Builds `data/_indexes/activity-feed.json`: a "what changed when" feed across
|
|
4
|
+
* skills and catalogs, sorted descending by date, so a consumer can poll it
|
|
5
|
+
* rather than diff the manifest.
|
|
15
6
|
*/
|
|
16
7
|
|
|
17
8
|
const fs = require("fs");
|
|
@@ -49,8 +40,7 @@ function buildActivityFeed({ root, manifest, skills, catalogFiles }) {
|
|
|
49
40
|
});
|
|
50
41
|
}
|
|
51
42
|
} catch {
|
|
52
|
-
//
|
|
53
|
-
// is unlikely.
|
|
43
|
+
// Skip malformed JSON; lint runs before build-indexes.
|
|
54
44
|
}
|
|
55
45
|
}
|
|
56
46
|
|
|
@@ -1,15 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* file, a compact summary: purpose, entry count, version pin (where applicable),
|
|
7
|
-
* source confidence, TLP, last-updated date. Consumers can load this single
|
|
8
|
-
* file (~3-4 KB) instead of every _meta block to learn what catalogs are
|
|
9
|
-
* available and how fresh they are.
|
|
10
|
-
*
|
|
11
|
-
* Curated human-readable purpose strings: keep these in lockstep with the
|
|
12
|
-
* canonical catalog README in `docs/data-catalogs.md` if/when added.
|
|
3
|
+
* Builds `data/_indexes/catalog-summaries.json`: one compact card per
|
|
4
|
+
* data/<catalog>.json, so a consumer learns what catalogs exist and how fresh
|
|
5
|
+
* they are from a single small file rather than every _meta block.
|
|
13
6
|
*/
|
|
14
7
|
|
|
15
8
|
const fs = require("fs");
|
|
@@ -1,29 +1,18 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* `orchestrator currency` to produce the same answer.
|
|
9
|
-
*
|
|
10
|
-
* The reference date is deterministic so the file hash stays stable until
|
|
11
|
-
* skills or the reference change — which is what `validate-indexes`
|
|
12
|
-
* requires. The orchestrator's interactive `currency` command remains the
|
|
13
|
-
* source-of-truth for live decay; this index is the snapshot view.
|
|
14
|
-
*
|
|
15
|
-
* Decay formula matches pipeline.js _currencyScore() exactly:
|
|
16
|
-
* >180 days → -30, >90 → -20, >60 → -10, >30 → -5
|
|
17
|
-
* -5 per forward_watch entry
|
|
3
|
+
* Builds `data/_indexes/currency.json` — per-skill currency scored against
|
|
4
|
+
* manifest.threat_review_date rather than today, so the file hash stays
|
|
5
|
+
* stable until skills or the reference change, which is what
|
|
6
|
+
* `validate-indexes` requires. The orchestrator's `currency` command is the
|
|
7
|
+
* live-decay view; this index is the snapshot.
|
|
18
8
|
*/
|
|
19
9
|
|
|
20
10
|
const fs = require("fs");
|
|
21
11
|
const path = require("path");
|
|
22
12
|
|
|
23
13
|
function currencyScore(daysSinceReview, _forwardWatchCount) {
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
// Param retained for ABI compatibility with callers.
|
|
14
|
+
// Scoring matches orchestrator/pipeline.js _currencyScore(). forward_watch
|
|
15
|
+
// count is a maintenance signal, not staleness, so the param is unused.
|
|
27
16
|
let score = 100;
|
|
28
17
|
if (daysSinceReview > 180) score -= 30;
|
|
29
18
|
else if (daysSinceReview > 90) score -= 20;
|
|
@@ -43,8 +32,6 @@ function parseFrontmatterForwardWatchCount(body) {
|
|
|
43
32
|
const m = body.match(/^---\n([\s\S]*?)\n---/);
|
|
44
33
|
if (!m) return 0;
|
|
45
34
|
const fm = m[1];
|
|
46
|
-
// Counts top-level "forward_watch:" list items. Lines starting with " - "
|
|
47
|
-
// immediately after a "forward_watch:" line until the next non-indented key.
|
|
48
35
|
const lines = fm.split(/\r?\n/);
|
|
49
36
|
let inFw = false;
|
|
50
37
|
let count = 0;
|
|
@@ -1,29 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
3
|
+
* Builds the CWE-keyed half of `data/_indexes/chains.json`, in the same
|
|
4
|
+
* hydrated cross-walk shape as the CVE-keyed half.
|
|
4
5
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* see every skill / catalog dimension that touches the weakness class.
|
|
9
|
-
*
|
|
10
|
-
* Per-CWE shape:
|
|
11
|
-
* {
|
|
12
|
-
* name: CWE catalog entry name
|
|
13
|
-
* category: CWE catalog category (memory-safety / injection / etc.)
|
|
14
|
-
* referencing_skills: every skill listing this CWE in cwe_refs
|
|
15
|
-
* chain: {
|
|
16
|
-
* atlas, attack_refs, framework_gaps, d3fend, rfc_refs, dlp_refs
|
|
17
|
-
* }
|
|
18
|
-
* related_cves: CVEs in cve-catalog.json whose framework gaps
|
|
19
|
-
* surface via skills that also cite this CWE
|
|
20
|
-
* }
|
|
21
|
-
*
|
|
22
|
-
* The CWE → CVE link is indirect — CWEs aren't currently stamped on each
|
|
23
|
-
* CVE entry directly. The skill bodies are the connective tissue. So we
|
|
24
|
-
* compute it the same way the CVE chains builder does: skills cite CWEs,
|
|
25
|
-
* skills cite framework_gaps, framework_gaps surface evidence_cves. The
|
|
26
|
-
* intersection through the skill graph gives the related-CVE set.
|
|
6
|
+
* The CWE → CVE link is indirect: CWE ids are not stamped on CVE entries, so
|
|
7
|
+
* related_cves is computed through the skill graph — skills cite CWEs, skills
|
|
8
|
+
* cite framework_gaps, framework_gaps surface evidence_cves.
|
|
27
9
|
*/
|
|
28
10
|
|
|
29
11
|
function buildCweChains({ skills, cweCatalog, atlasTtps, cveCatalog, frameworkGaps, d3fendCatalog, rfcCatalog }) {
|
|
@@ -37,7 +19,6 @@ function buildCweChains({ skills, cweCatalog, atlasTtps, cveCatalog, frameworkGa
|
|
|
37
19
|
.filter((s) => (s.cwe_refs || []).includes(cweId))
|
|
38
20
|
.map((s) => s.name);
|
|
39
21
|
|
|
40
|
-
// Aggregate cross-refs from those skills.
|
|
41
22
|
const accum = {
|
|
42
23
|
atlas_refs: new Set(),
|
|
43
24
|
attack_refs: new Set(),
|
|
@@ -54,7 +35,6 @@ function buildCweChains({ skills, cweCatalog, atlasTtps, cveCatalog, frameworkGa
|
|
|
54
35
|
}
|
|
55
36
|
}
|
|
56
37
|
|
|
57
|
-
// Hydrate the cross-walk dimensions for the AI consumer.
|
|
58
38
|
const hydrated = {
|
|
59
39
|
atlas: [...accum.atlas_refs].sort().map((a) => ({
|
|
60
40
|
id: a,
|
|
@@ -80,11 +60,8 @@ function buildCweChains({ skills, cweCatalog, atlasTtps, cveCatalog, frameworkGa
|
|
|
80
60
|
dlp_refs: [...accum.dlp_refs].sort(),
|
|
81
61
|
};
|
|
82
62
|
|
|
83
|
-
//
|
|
84
|
-
//
|
|
85
|
-
// (_draft) CVEs so this CWE half agrees with the CVE half (build-indexes.js)
|
|
86
|
-
// and the reverse-ref index — all three must reflect the same curated,
|
|
87
|
-
// operator-queryable truth.
|
|
63
|
+
// Draft (_draft) CVEs are skipped so this half agrees with the CVE half
|
|
64
|
+
// (build-indexes.js) and the reverse-ref index.
|
|
88
65
|
const relatedCves = new Set();
|
|
89
66
|
for (const gap of accum.framework_gaps) {
|
|
90
67
|
for (const ev of (frameworkGaps[gap]?.evidence_cves || [])) {
|