@clear-capabilities/agentic-security-scanner 0.175.0 → 0.177.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/bin/agentic-security.js +206 -28
- package/dist/1919.index.js +1 -1
- package/dist/2066.index.js +851 -0
- package/dist/2199.index.js +1 -1
- package/dist/2584.index.js +3350 -0
- package/dist/2923.index.js +13 -0
- package/dist/2946.index.js +81 -0
- package/dist/3196.index.js +27 -0
- package/dist/3727.index.js +292 -0
- package/dist/37d64df006576248d0d8.json +286 -0
- package/dist/4168.index.js +509 -0
- package/dist/4281.index.js +993 -0
- package/dist/5099.index.js +96 -0
- package/dist/5456.index.js +291 -0
- package/dist/5616.index.js +1471 -0
- package/dist/5620.index.js +2494 -0
- package/dist/5817.index.js +714 -0
- package/dist/5937.index.js +672 -0
- package/dist/6333.index.js +98 -0
- package/dist/6655.index.js +3221 -0
- package/dist/7838.index.js +491 -974
- package/dist/818.index.js +4 -3
- package/dist/83.index.js +159 -0
- package/dist/8551.index.js +82 -0
- package/dist/8633.index.js +93 -0
- package/dist/8714.index.js +76 -76
- package/dist/8825.index.js +84 -0
- package/dist/932.index.js +186 -0
- package/dist/9560.index.js +358 -19
- package/dist/9800.index.js +6 -1
- package/dist/agentic-security.mjs +10 -10
- package/dist/agentic-security.mjs.sha256 +1 -1
- package/dist/b4289e728659450b5e19.js +32 -0
- package/package.json +8 -4
- package/src/artifact/CLAUDE.md +4 -0
- package/src/artifact/artifact-evidence.js +63 -0
- package/src/artifact/cosign.js +61 -0
- package/src/artifact/engine-source.js +72 -0
- package/src/artifact/oci-layout.js +47 -0
- package/src/artifact/registry.js +213 -0
- package/src/capabilities/tool-registry.js +2 -0
- package/src/lineage/CLAUDE.md +4 -0
- package/src/lineage/identity-token.js +11 -0
- package/src/lineage/observation-adapters.js +13 -0
- package/src/lineage/runtime-edge-states.js +139 -0
- package/src/lineage/runtime-explorer-view.js +82 -0
- package/src/lineage/runtime-observation.js +11 -0
- package/src/mcp/CLAUDE.md +3 -1
- package/src/mcp/proof-tools.js +132 -0
- package/src/mcp/server.js +1 -1
- package/src/mcp/tools.js +3 -1
- package/src/posture/CLAUDE.md +23 -0
- package/src/posture/artifact-registry.js +4 -0
- package/src/posture/custom-rules.js +54 -8
- package/src/posture/dnr/attack-facts.js +31 -0
- package/src/posture/dnr/cli.js +138 -0
- package/src/posture/dnr/detection-ir.js +123 -0
- package/src/posture/dnr/exploit-correlation.js +25 -0
- package/src/posture/dnr/exposure-window.js +134 -0
- package/src/posture/dnr/hunt.js +84 -0
- package/src/posture/dnr/log-reader.js +189 -0
- package/src/posture/dnr/matchers.js +90 -0
- package/src/posture/dnr/outputs.js +262 -0
- package/src/posture/dnr/payload-classes.js +120 -0
- package/src/posture/dnr/redact.js +25 -0
- package/src/posture/dnr/store.js +57 -0
- package/src/posture/dnr/validate.js +64 -0
- package/src/posture/dnr/view.js +51 -0
- package/src/posture/evaluation/lab/arm-ollama.js +88 -0
- package/src/posture/evaluation/lab/arms.js +142 -0
- package/src/posture/evaluation/lab/cli.js +135 -0
- package/src/posture/evaluation/lab/dataset.js +139 -0
- package/src/posture/evaluation/lab/materialize.js +44 -0
- package/src/posture/evaluation/lab/neutralize.js +144 -0
- package/src/posture/evaluation/lab/report.js +194 -0
- package/src/posture/evaluation/lab/run.js +225 -0
- package/src/posture/evaluation/lab/score.js +135 -0
- package/src/posture/evaluation/lab/scoreboard.js +48 -0
- package/src/posture/evaluation/lab/stats.js +41 -0
- package/src/posture/evaluation/lab/store.js +53 -0
- package/src/posture/fleet/cli.js +260 -63
- package/src/posture/fleet/discover-edges.js +135 -0
- package/src/posture/fleet/enterprise-graph.js +189 -0
- package/src/posture/fleet/fleet-graph.js +9 -4
- package/src/posture/fleet/fleet-manifest.js +50 -5
- package/src/posture/fleet/job-runner.js +153 -0
- package/src/posture/fleet/job-store.js +138 -0
- package/src/posture/fleet/lineage-summary.js +55 -0
- package/src/posture/fleet/org-health.js +56 -0
- package/src/posture/fleet/scan-cache.js +112 -0
- package/src/posture/fleet/scan-child.js +32 -0
- package/src/posture/proof-fabric/cli.js +34 -6
- package/src/posture/proof-fabric/container.js +57 -0
- package/src/posture/proof-fabric/fix-proof.js +2 -1
- package/src/posture/proof-fabric/generate-python.js +129 -0
- package/src/posture/proof-fabric/generate.js +214 -0
- package/src/posture/proof-fabric/reproducer.js +3 -0
- package/src/posture/proof-fabric/runner.js +44 -13
- package/src/posture/proof-fabric/siblings.js +77 -0
- package/src/posture/routing/policy-router.js +103 -0
- package/src/posture/rule-compiler/cli.js +101 -9
- package/src/posture/rule-compiler/corpus.js +13 -9
- package/src/posture/rule-compiler/ir.js +52 -9
- package/src/posture/rule-compiler/mutation.js +72 -1
- package/src/posture/rule-compiler/promotion.js +21 -5
- package/src/posture/rule-compiler/proposal.js +137 -0
- package/src/posture/rule-compiler/regression.js +95 -0
- package/src/posture/rule-compiler/store.js +45 -1
- package/src/posture/rule-pack-signing.js +4 -2
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// CLI surface: `agentic-security rules synthesize|evaluate|promote|status|rollback`.
|
|
1
|
+
// CLI surface: `agentic-security rules synthesize|evaluate|promote|status|rollback|cost|regress`.
|
|
2
2
|
//
|
|
3
3
|
// Distinct from the pre-existing `labs synthesize-rule` (posture/rule-synthesis.js), which proposes SUPPRESSION
|
|
4
4
|
// rules from repeated false-positive triage verdicts — a different input, a different output and a different
|
|
@@ -9,12 +9,17 @@
|
|
|
9
9
|
// it never decides promotion).
|
|
10
10
|
import fs from 'node:fs';
|
|
11
11
|
import { statePath } from '../state-dir.js';
|
|
12
|
+
import { packTrust, signPublishedRule } from './store.js';
|
|
13
|
+
import { sharedFalsePositiveCorpus, falsePositiveHits, regressRules } from './regression.js';
|
|
12
14
|
import { draftRuleIR, validateRuleIR, ruleIRDigest } from './ir.js';
|
|
13
15
|
import { checkEligibility } from './eligibility.js';
|
|
14
16
|
import { buildCorpus } from './corpus.js';
|
|
15
17
|
import { gatePromotion, checkLeakage, PROMOTION_DEFAULTS } from './promotion.js';
|
|
16
18
|
import { recordTransition, stateOf, historyFor } from './lifecycle.js';
|
|
17
|
-
import { loadLedger, saveLedger, saveIR, loadIR, saveEvaluation, writeFixtures, publishCustomRule, unpublishCustomRule } from './store.js';
|
|
19
|
+
import { loadLedger, saveLedger, saveIR, loadIR, saveEvaluation, writeFixtures, publishCustomRule, unpublishCustomRule, saveCorpus, loadCorpus, saveSynthesis, loadSynthesis, loadProvenance, saveProvenance } from './store.js';
|
|
20
|
+
import { synthesizeCandidate } from './proposal.js';
|
|
21
|
+
import { mutatorCoverage } from './mutation.js';
|
|
22
|
+
import { sha256 } from '../proof-fabric/reproducer.js';
|
|
18
23
|
|
|
19
24
|
function loadFinding(root, id) {
|
|
20
25
|
const p = statePath(root, 'last-scan.json');
|
|
@@ -35,7 +40,9 @@ export async function cmdRules(args, root) {
|
|
|
35
40
|
if (sub === 'promote') return cmdPromote(args, root);
|
|
36
41
|
if (sub === 'status') return cmdStatus(args, root);
|
|
37
42
|
if (sub === 'rollback') return cmdRollback(args, root);
|
|
38
|
-
|
|
43
|
+
if (sub === 'cost') return cmdCost(args, root);
|
|
44
|
+
if (sub === 'regress') return cmdRegress(args, root);
|
|
45
|
+
console.error('usage: rules synthesize <finding-id> --spec <file.json> [--auto] [--max-attempts N] | evaluate <rule-id> | promote <rule-id> | status [rule-id] | rollback <rule-id> | cost <rule-id>');
|
|
39
46
|
return 4;
|
|
40
47
|
}
|
|
41
48
|
|
|
@@ -51,7 +58,22 @@ async function cmdSynthesize(args, root) {
|
|
|
51
58
|
const elig = checkEligibility(finding, { proofState: args.flags['proof-state'] || null });
|
|
52
59
|
if (!elig.eligible) { console.error(`Not eligible for rule synthesis: ${elig.reasons.join('; ')}`); return 4; }
|
|
53
60
|
|
|
54
|
-
|
|
61
|
+
// The candidate pattern is the model's or the person's, when they gave one. Without one, or with --auto, the synthesis loop proposes
|
|
62
|
+
// from the vulnerable and patched code, scores each attempt on the real corpus and its mutations, and revises (see proposal.js).
|
|
63
|
+
let finalSpec = spec; let synthesis = null;
|
|
64
|
+
const corpusSpec = { originalCode: spec.originalCode, patchedCode: spec.patchedCode ?? null, nearbyCleanCode: spec.nearbyCleanCode ?? [], protectedVariantsCode: spec.protectedVariantsCode ?? [] };
|
|
65
|
+
const leak = checkLeakage(JSON.stringify(corpusSpec), { benchmarkNames: args.flags['benchmark-names'] ? String(args.flags['benchmark-names']).split(',') : [], fileNames: spec.fileNames ?? [] });
|
|
66
|
+
if (!leak.ok) { console.error(`Refusing: the material for this rule shows benchmark or answer-key signals (${leak.offendingTerms.join('; ')}). Nothing was written.`); return 4; }
|
|
67
|
+
if (args.flags.auto || (!spec.match && !spec.sink)) {
|
|
68
|
+
if (!isStr(spec.originalCode)) { console.error('Automatic proposal needs originalCode (the vulnerable snippet) in the spec.'); return 4; }
|
|
69
|
+
const { id, title, severity, cwe, family, languages, remediation } = spec;
|
|
70
|
+
synthesis = synthesizeCandidate({ base: { id, title, severity, cwe, family, languages, remediation, originatingFindingId: finding.stableId ?? finding.id, evidenceState: elig.evidenceState }, corpusSpec, proposed: spec.match ?? null, maxAttempts: Number(args.flags['max-attempts']) || 6, leakageOk: true });
|
|
71
|
+
for (const a of synthesis.attempts) console.log(` attempt ${a.n}: ${a.strategy.padEnd(34)} ${a.valid ? `precision ${fmtPct(a.score.precision)} recall ${fmtPct(a.score.recall)} protected-failures ${a.score.protectedFailures}${a.pass ? ' PASS' : ''}` : `invalid (${a.errors[0]})`}`);
|
|
72
|
+
if (!synthesis.best) { console.error('No valid candidate could be proposed from this material.'); return 1; }
|
|
73
|
+
finalSpec = { ...spec, ...(synthesis.best.match ? { match: synthesis.best.match } : {}), ...(synthesis.best.structure ?? {}) };
|
|
74
|
+
console.log(synthesis.best.pass ? ` candidate meets the promotion gate on this corpus (not yet recorded as VALIDATED).` : ` no attempt met the gate; the best is kept as a CANDIDATE only: ${synthesis.best.reasons.join('; ')}`);
|
|
75
|
+
}
|
|
76
|
+
const ir = draftRuleIR({ ...finalSpec, originatingFindingId: finding.stableId ?? finding.id, originatingCommit: spec.originatingCommit ?? null, evidenceState: elig.evidenceState });
|
|
55
77
|
const errs = validateRuleIR(ir);
|
|
56
78
|
if (errs.length) { console.error(`Invalid rule: ${errs.join('; ')}`); return 4; }
|
|
57
79
|
|
|
@@ -61,6 +83,8 @@ async function cmdSynthesize(args, root) {
|
|
|
61
83
|
const cand = recordTransition(ledger, { ruleId: ir.id, from: 'DRAFT', to: 'CANDIDATE', reason: 'eligibility passed', evidence: { eligibility: elig }, actor: args.flags.actor || 'agent' });
|
|
62
84
|
saveLedger(root, ledger);
|
|
63
85
|
saveIR(root, { ...ir, state: 'CANDIDATE' });
|
|
86
|
+
saveCorpus(root, ir.id, { ...corpusSpec, fileNames: spec.fileNames ?? [], discoveryCost: spec.discoveryCost ?? (args.flags['discovery-cost-usd'] ? { usd: Number(args.flags['discovery-cost-usd']), tokens: Number(args.flags['discovery-tokens']) || null, model: spec.model ?? null } : null) });
|
|
87
|
+
if (synthesis) saveSynthesis(root, ir.id, { attempts: synthesis.attempts, best: synthesis.best });
|
|
64
88
|
console.log(`Rule ${ir.id}: DRAFT -> CANDIDATE (digest ${ruleIRDigest(ir).slice(0, 12)})`);
|
|
65
89
|
if (!cand.ok) console.error(cand.reason);
|
|
66
90
|
return 0;
|
|
@@ -71,17 +95,35 @@ async function cmdEvaluate(args, root) {
|
|
|
71
95
|
if (!id) { console.error('usage: rules evaluate <rule-id> --corpus <file.json>'); return 4; }
|
|
72
96
|
const ir = loadIR(root, id);
|
|
73
97
|
if (!ir) { console.error(`No rule IR found for ${id}. Run rules synthesize first.`); return 4; }
|
|
74
|
-
const corpusSpec = readJsonArg(args.flags.corpus);
|
|
98
|
+
const corpusSpec = readJsonArg(args.flags.corpus) ?? loadCorpus(root, id);
|
|
75
99
|
if (!corpusSpec) { console.error('No corpus supplied. Pass --corpus <file.json> with originalCode (and optionally patchedCode/nearbyCleanCode/protectedVariantsCode).'); return 4; }
|
|
76
100
|
const corpus = buildCorpus(ir, corpusSpec);
|
|
77
|
-
|
|
101
|
+
// the corpus a rule is evaluated on is the evidence for it: keep it, so `rules regress` and the shared false-positive corpus use the SAME code
|
|
102
|
+
if (args.flags.corpus) saveCorpus(root, id, { ...(loadCorpus(root, id) ?? {}), ...corpusSpec });
|
|
103
|
+
const leakage = checkLeakage(JSON.stringify({ ...corpusSpec, fileNames: undefined, discoveryCost: undefined }), { benchmarkNames: args.flags['benchmark-names'] ? String(args.flags['benchmark-names']).split(',') : [], fileNames: corpusSpec.fileNames ?? [] });
|
|
78
104
|
const gate = gatePromotion({ ir, corpus, leakageCheck: leakage, thresholds: args.flags.thresholds ? readJsonArg(args.flags.thresholds) : PROMOTION_DEFAULTS });
|
|
79
105
|
|
|
80
106
|
const ledger = loadLedger(root);
|
|
107
|
+
// the shared false-positive corpus: the safe code of every other promoted rule, plus the operator's own files. Part of the gate.
|
|
108
|
+
const fpItems = sharedFalsePositiveCorpus(root, ledger, id);
|
|
109
|
+
const fpHits = falsePositiveHits(ir, fpItems);
|
|
110
|
+
gate.fpCorpus = { items: fpItems.length, hits: fpHits.map((h) => h.label) };
|
|
111
|
+
if (fpHits.length) { gate.pass = false; gate.reasons.push(`fires on ${fpHits.length} of ${fpItems.length} item(s) in the shared false-positive corpus: ${fpHits.slice(0, 3).map((h) => h.label).join(', ')}${fpHits.length > 3 ? ', ...' : ''}`); }
|
|
81
112
|
const cur = stateOf(ledger, id);
|
|
82
113
|
if (gate.pass && cur === 'CANDIDATE') recordTransition(ledger, { ruleId: id, from: 'CANDIDATE', to: 'VALIDATED', reason: 'promotion gate passed', evidence: { score: gate.evaluation.score }, actor: args.flags.actor || 'agent' });
|
|
83
114
|
saveLedger(root, ledger);
|
|
84
115
|
saveEvaluation(root, id, gate);
|
|
116
|
+
const corpusHash = (list) => sha256(JSON.stringify(list.map((x) => [x.label, x.code]).sort()));
|
|
117
|
+
const prev = loadProvenance(root, id);
|
|
118
|
+
saveProvenance(root, id, {
|
|
119
|
+
schema: 'agentic-security/rule-provenance', version: 1, ruleId: id, ruleVersion: 1, ruleHash: ruleIRDigest(ir),
|
|
120
|
+
originatingFinding: ir.provenance.originatingFindingId, originatingCommit: ir.provenance.originatingCommit, evidenceState: ir.provenance.evidenceState,
|
|
121
|
+
generatingModel: ir.provenance.model, promptConfigVersion: ir.provenance.promptConfigVersion,
|
|
122
|
+
positiveCorpusHash: corpusHash(corpus.positives), negativeCorpusHash: corpusHash(corpus.negatives),
|
|
123
|
+
benchmarkResults: { precision: gate.evaluation.score.precision, recall: gate.evaluation.score.recall, tp: gate.evaluation.score.tp, fp: gate.evaluation.score.fp, fn: gate.evaluation.score.fn, tn: gate.evaluation.score.tn, protectedFailures: gate.evaluation.score.protectedFailures, thresholds: gate.evaluation.thresholds, gate: gate.pass ? 'passed' : 'failed', reasons: gate.reasons },
|
|
124
|
+
leakageCheck: leakage, mutatorCoverage: mutatorCoverage(corpusSpec.originalCode), discoveryCost: corpusSpec.discoveryCost ?? prev?.discoveryCost ?? null,
|
|
125
|
+
promotionEvent: prev?.promotionEvent ?? null, evaluatedAt: new Date().toISOString(),
|
|
126
|
+
});
|
|
85
127
|
if (args.flags.json) console.log(JSON.stringify(gate, null, 2));
|
|
86
128
|
else {
|
|
87
129
|
console.log(`Rule ${id}: ${gate.pass ? 'VALIDATED' : 'gate FAILED'}`);
|
|
@@ -91,6 +133,7 @@ async function cmdEvaluate(args, root) {
|
|
|
91
133
|
if (gate.pass) { const f = writeFixtures(root, ir, corpus); if (f.ok) console.log(` regression fixtures written: ${f.written.length} file(s) under .agentic-security/rule-compiler/${id}/fixtures/`); }
|
|
92
134
|
return gate.pass ? 0 : 1;
|
|
93
135
|
}
|
|
136
|
+
function isStr(v) { return typeof v === 'string' && v.length > 0; }
|
|
94
137
|
const fmtPct = (n) => (n == null ? 'unmeasured' : `${(n * 100).toFixed(1)}%`);
|
|
95
138
|
|
|
96
139
|
async function cmdPromote(args, root) {
|
|
@@ -101,15 +144,39 @@ async function cmdPromote(args, root) {
|
|
|
101
144
|
if (stateOf(ledger, id) !== 'VALIDATED') { console.error(`Rule ${id} is not VALIDATED (run rules evaluate first); refusing to promote.`); return 4; }
|
|
102
145
|
const ir = loadIR(root, id);
|
|
103
146
|
if (!ir) { console.error(`No rule IR found for ${id}.`); return 4; }
|
|
147
|
+
// other rules may have been promoted since this one was evaluated: hold it to the false-positive corpus as it is NOW
|
|
148
|
+
const fpNow = falsePositiveHits(ir, sharedFalsePositiveCorpus(root, ledger, id));
|
|
149
|
+
if (fpNow.length) { console.error(`Promotion refused: ${id} fires on ${fpNow.length} item(s) of the shared false-positive corpus (${fpNow.slice(0, 3).map((h) => h.label).join(', ')}). Re-evaluate or narrow the rule.`); return 1; }
|
|
104
150
|
const pub = publishCustomRule(root, ir);
|
|
105
151
|
if (!pub.ok) { console.error(`Promotion refused: ${pub.reason}`); return 4; }
|
|
106
152
|
const t = recordTransition(ledger, { ruleId: id, from: 'VALIDATED', to: 'PROMOTED', reason: args.flags.reason || 'promoted', actor: args.flags.actor || 'operator' });
|
|
107
153
|
saveLedger(root, ledger);
|
|
154
|
+
const prov = loadProvenance(root, id);
|
|
155
|
+
if (prov) saveProvenance(root, id, { ...prov, promotionEvent: { at: new Date().toISOString(), actor: args.flags.actor || 'operator', reason: args.flags.reason || 'promoted', ruleHash: ruleIRDigest(ir) } });
|
|
108
156
|
console.log(`Rule ${id}: VALIDATED -> PROMOTED. Published to ${pub.file}.`);
|
|
157
|
+
// Publishing is not trusting. Rule packs are signature-checked so a rule dropped into a repository cannot run in someone's CI; sign this
|
|
158
|
+
// one with the operator's key when there is one, and say plainly whether the scanner will load it.
|
|
159
|
+
const key = args.flags['sign-key'] ?? process.env.AGENTIC_SECURITY_PRIVATE_KEY;
|
|
160
|
+
if (key) { const sg = signPublishedRule(pub.file, key); console.log(sg.ok ? ` signed: ${pub.file}.sig` : ` could not sign (${sg.reason}); the rule is published unsigned`); }
|
|
161
|
+
const trust = packTrust(root, id);
|
|
162
|
+
console.log(trust.live ? ` The scanner will load it (${trust.via}).` : ` NOT LIVE YET: the scanner will refuse this rule (${trust.reason}). To make it live: ${trust.remediation}.`);
|
|
109
163
|
if (!t.ok) console.error(t.reason);
|
|
110
164
|
return 0;
|
|
111
165
|
}
|
|
112
166
|
|
|
167
|
+
/** `rules regress [rule-id]`: re-score every PROMOTED rule against its own corpus and the shared false-positive corpus. Exit 1 on any regression. */
|
|
168
|
+
async function cmdRegress(args, root) {
|
|
169
|
+
const ledger = loadLedger(root);
|
|
170
|
+
if (ledger.tampered) { console.error(`Refusing: rule lifecycle ledger failed its integrity check (${ledger.tamperReason}).`); return 4; }
|
|
171
|
+
const only = args._[2] ?? null;
|
|
172
|
+
const results = regressRules(root, ledger, { only, thresholds: args.flags.thresholds ? readJsonArg(args.flags.thresholds) : undefined });
|
|
173
|
+
if (args.flags.json) console.log(JSON.stringify({ results }, null, 2));
|
|
174
|
+
else if (!results.length) console.log(only ? `No promoted rule ${only}.` : 'No rule is PROMOTED; nothing to regress.');
|
|
175
|
+
else for (const r of results) { console.log(`${r.id}: ${r.status.toUpperCase()}${r.precision != null ? ` precision ${fmtPct(r.precision)} recall ${fmtPct(r.recall)} false-positive corpus hits ${r.fpCorpusHits}` : ''}`); for (const m of r.reasons) console.log(` - ${m}`); }
|
|
176
|
+
if (only && !results.length) return 4;
|
|
177
|
+
return results.every((r) => r.status === 'ok') ? 0 : 1;
|
|
178
|
+
}
|
|
179
|
+
|
|
113
180
|
async function cmdRollback(args, root) {
|
|
114
181
|
const id = args._[2];
|
|
115
182
|
if (!id) { console.error('usage: rules rollback <rule-id> [--reason text]'); return 4; }
|
|
@@ -130,12 +197,37 @@ async function cmdStatus(args, root) {
|
|
|
130
197
|
const h = historyFor(ledger, id);
|
|
131
198
|
if (!h.length) { console.error(`No history for rule ${id}.`); return 4; }
|
|
132
199
|
if (args.flags.json) console.log(JSON.stringify(h, null, 2));
|
|
133
|
-
else { console.log(`Rule ${id}: ${stateOf(ledger, id)}`);
|
|
200
|
+
else { console.log(`Rule ${id}: ${stateOf(ledger, id)}`);
|
|
201
|
+
if (stateOf(ledger, id) === 'PROMOTED') { const tr = packTrust(root, id); console.log(tr.live ? ` live: yes (${tr.via})` : ` live: NO, the scanner refuses it (${tr.reason}). ${tr.remediation}.`); }
|
|
202
|
+
for (const e of h) console.log(` ${e.at} ${e.from ?? '(new)'} -> ${e.to} (${e.reason || ''})`);
|
|
203
|
+
const syn = loadSynthesis(root, id);
|
|
204
|
+
if (syn?.attempts?.length) { console.log(` synthesis: ${syn.attempts.length} attempt(s), best ${syn.best ? `${syn.best.strategy}${syn.best.pass ? ' (met the gate)' : ' (did not meet the gate)'}` : 'none'}`); }
|
|
205
|
+
}
|
|
134
206
|
return 0;
|
|
135
207
|
}
|
|
136
208
|
const ids = [...new Set(ledger.entries.map((e) => e.ruleId))];
|
|
137
|
-
if (args.flags.json) { console.log(JSON.stringify(ids.map((rid) => ({ id: rid, state: stateOf(ledger, rid) })), null, 2)); return 0; }
|
|
209
|
+
if (args.flags.json) { console.log(JSON.stringify(ids.map((rid) => ({ id: rid, state: stateOf(ledger, rid), ...(stateOf(ledger, rid) === 'PROMOTED' ? { live: packTrust(root, rid).live } : {}) })), null, 2)); return 0; }
|
|
138
210
|
if (!ids.length) { console.log('No rules in the lifecycle ledger yet.'); return 0; }
|
|
139
|
-
for (const rid of ids) console.log(` ${rid.padEnd(40)} ${stateOf(ledger, rid)}`);
|
|
211
|
+
for (const rid of ids) console.log(` ${rid.padEnd(40)} ${stateOf(ledger, rid)}${stateOf(ledger, rid) === 'PROMOTED' && !packTrust(root, rid).live ? ' (NOT LIVE: the scanner refuses it)' : ''}`);
|
|
212
|
+
return 0;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// PRD 3 business metric: what the first, model-assisted discovery cost, against what every later deterministic detection costs.
|
|
216
|
+
// The discovery cost is whatever the agent run reported for it (recorded at synthesis); a deterministic detection makes no model
|
|
217
|
+
// call and is $0 of model spend. The count of later detections is read from the LAST scan only, so it is a floor, not a history.
|
|
218
|
+
async function cmdCost(args, root) {
|
|
219
|
+
const id = args._[2];
|
|
220
|
+
if (!id) { console.error('usage: rules cost <rule-id>'); return 4; }
|
|
221
|
+
const prov = loadProvenance(root, id);
|
|
222
|
+
if (!prov) { console.error(`No provenance for ${id}. Run rules evaluate first.`); return 4; }
|
|
223
|
+
let detections = null;
|
|
224
|
+
try { const last = JSON.parse(fs.readFileSync(statePath(root, 'last-scan.json'), 'utf8')); detections = (last.findings || []).filter((f) => String(f.id || '').startsWith(`custom:rule-compiler/${id}:`)).length; } catch { /* no scan */ }
|
|
225
|
+
const first = prov.discoveryCost;
|
|
226
|
+
const out = { ruleId: id, firstDiscovery: first ? { usd: first.usd ?? null, tokens: first.tokens ?? null, model: first.model ?? null } : null, laterDeterministicDetections: detections, laterDeterministicModelUsd: 0, state: prov.promotionEvent ? 'promoted' : 'not promoted' };
|
|
227
|
+
if (args.flags.json) { console.log(JSON.stringify(out, null, 2)); return 0; }
|
|
228
|
+
console.log(`Rule ${id} (${out.state})`);
|
|
229
|
+
console.log(first ? ` first discovery: ${first.usd != null ? `$${first.usd}` : 'cost not recorded'}${first.tokens ? `, ${first.tokens} tokens` : ''}${first.model ? `, ${first.model}` : ''}` : ' first discovery: cost was not recorded for this rule');
|
|
230
|
+
console.log(` later detections by this rule in the last scan: ${detections ?? 'no scan available'}, each at $0 of model spend`);
|
|
231
|
+
if (first?.usd != null && detections) console.log(` discovery cost spread over ${detections + 1} detection(s): $${(first.usd / (detections + 1)).toFixed(4)} each, falling with every further one`);
|
|
140
232
|
return 0;
|
|
141
233
|
}
|
|
@@ -9,17 +9,13 @@
|
|
|
9
9
|
// the score a rule gets here is the score it will actually produce once promoted.
|
|
10
10
|
import { generateVariants } from './mutation.js';
|
|
11
11
|
import { compileToCustomRule } from './ir.js';
|
|
12
|
-
import { runRule } from '../custom-rules.js';
|
|
12
|
+
import { runRule, normalizeCustomRule } from '../custom-rules.js';
|
|
13
13
|
|
|
14
|
+
/** The rule exactly as the scanner would load it: through the loader's own normalizer, not a second implementation. */
|
|
14
15
|
function toRuntimeRule(ir) {
|
|
15
16
|
const { ok, doc, errs } = compileToCustomRule(ir);
|
|
16
17
|
if (!ok) throw new Error(`cannot build corpus for an invalid rule IR: ${errs.join('; ')}`);
|
|
17
|
-
|
|
18
|
-
return {
|
|
19
|
-
id: doc.id, title: doc.title, severity: doc.severity, cwe: doc.cwe, languages: doc.languages, message: doc.message,
|
|
20
|
-
regexes, requireAll: !!doc.match.allOf, windowLines: doc.match.window ?? 5,
|
|
21
|
-
notMatch: doc.match.notMatch ? new RegExp(doc.match.notMatch) : null,
|
|
22
|
-
};
|
|
18
|
+
return normalizeCustomRule(doc);
|
|
23
19
|
}
|
|
24
20
|
|
|
25
21
|
const EXT = { javascript: 'x.js', typescript: 'x.ts', python: 'x.py', go: 'x.go', ruby: 'x.rb', java: 'x.java', rust: 'x.rs', csharp: 'x.cs', php: 'x.php', any: 'x.txt' };
|
|
@@ -33,8 +29,16 @@ export function buildCorpus(ir, { originalCode, patchedCode = null, nearbyCleanC
|
|
|
33
29
|
negatives.push({ label: 'patched', code: patchedCode });
|
|
34
30
|
for (const v of generateVariants(patchedCode)) negatives.push({ label: `patched-sibling:${v.mutator}`, code: v.code });
|
|
35
31
|
}
|
|
36
|
-
|
|
37
|
-
|
|
32
|
+
// The safe side is mutated too (rename, wrapper, equivalent API, ALTERNATE SANITIZER, framework variant, ...): a rule keyed to one
|
|
33
|
+
// sanitizer's name or one framework's accessor fires on the sibling that uses another, and that is a protected-negative failure.
|
|
34
|
+
nearbyCleanCode.forEach((code, i) => {
|
|
35
|
+
negatives.push({ label: `nearby-clean-${i}`, code });
|
|
36
|
+
for (const v of generateVariants(code)) negatives.push({ label: `nearby-clean-${i}-sibling:${v.mutator}`, code: v.code });
|
|
37
|
+
});
|
|
38
|
+
protectedVariantsCode.forEach((code, i) => {
|
|
39
|
+
negatives.push({ label: `protected-${i}`, code });
|
|
40
|
+
for (const v of generateVariants(code)) negatives.push({ label: `protected-${i}-sibling:${v.mutator}`, code: v.code });
|
|
41
|
+
});
|
|
38
42
|
|
|
39
43
|
return { positives, negatives };
|
|
40
44
|
}
|
|
@@ -4,9 +4,20 @@
|
|
|
4
4
|
// regex/allOf/notMatch/window engine; this module only adds PROVENANCE metadata around it.
|
|
5
5
|
//
|
|
6
6
|
// A rule's match semantics are deliberately the same closed vocabulary custom-rules.js
|
|
7
|
-
// already supports (pattern | allOf+window | notMatch
|
|
8
|
-
// introduced, so a promoted rule is exactly as auditable as a hand-written one.
|
|
7
|
+
// already supports (pattern | allOf+window | notMatch, plus the structural fields below): no
|
|
8
|
+
// separate matching engine is introduced, so a promoted rule is exactly as auditable as a hand-written one.
|
|
9
|
+
//
|
|
10
|
+
// The structural fields are REAL, not annotations: each is a regular expression and each compiles to a constraint the
|
|
11
|
+
// runtime matcher enforces.
|
|
12
|
+
// source, propagator, sink together they become a windowed co-occurrence (allOf) anchored on the SINK; a sink alone is the pattern
|
|
13
|
+
// sanitizer, dominatingGuard a hit is dropped when either matches within `guardWindow` lines before (or on) it
|
|
14
|
+
// frameworkConstraint a file-level precondition (the file must contain it, wherever it is)
|
|
15
|
+
// exclusions a hit is dropped when its own line matches one
|
|
16
|
+
// languages the rule only runs on files of those languages
|
|
17
|
+
// An explicit `match` always wins over a derived one. An AST pattern is NOT a field: the existing engine has no AST matcher for rules to
|
|
18
|
+
// compile to, and a field the matcher ignored would be the old mistake (an annotation that looks like a constraint).
|
|
9
19
|
import { sha256 } from '../proof-fabric/reproducer.js';
|
|
20
|
+
import { normalizeCustomRule } from '../custom-rules.js';
|
|
10
21
|
|
|
11
22
|
const RULE_IR_SCHEMA = 'agentic-security/rule-ir';
|
|
12
23
|
const RULE_IR_VERSION = 1;
|
|
@@ -28,8 +39,9 @@ export function validateRuleIR(ir) {
|
|
|
28
39
|
if (!isStr(ir.family)) errs.push('family required');
|
|
29
40
|
if (!Array.isArray(ir.languages) || !ir.languages.length || !ir.languages.every((l) => VALID_LANGS.has(l))) errs.push('languages must be a non-empty array from the supported set');
|
|
30
41
|
const m = ir.match;
|
|
31
|
-
if (
|
|
32
|
-
else
|
|
42
|
+
if (m == null && !isStr(ir.sink)) errs.push('match required (or at least a sink, from which one is derived)');
|
|
43
|
+
else if (m != null && typeof m !== 'object') errs.push('match must be an object');
|
|
44
|
+
else if (m) {
|
|
33
45
|
const hasPattern = isStr(m.pattern);
|
|
34
46
|
const hasAllOf = Array.isArray(m.allOf) && m.allOf.length > 0 && m.allOf.every(isStr);
|
|
35
47
|
if (!hasPattern && !hasAllOf) errs.push('match needs a pattern or a non-empty allOf');
|
|
@@ -39,12 +51,20 @@ export function validateRuleIR(ir) {
|
|
|
39
51
|
try { new RegExp(re); } catch { errs.push(`invalid regex: ${re}`); }
|
|
40
52
|
}
|
|
41
53
|
}
|
|
42
|
-
//
|
|
43
|
-
// never
|
|
54
|
+
// Structural fields: each is a regular expression the matcher enforces (see the header). Free text is rejected, so a field can
|
|
55
|
+
// never look like a constraint while being ignored.
|
|
44
56
|
for (const k of ['sink', 'source', 'propagator', 'sanitizer', 'dominatingGuard', 'frameworkConstraint']) {
|
|
45
|
-
if (ir[k]
|
|
57
|
+
if (ir[k] == null) continue;
|
|
58
|
+
if (!isStr(ir[k])) { errs.push(`${k} must be a regular-expression string when present`); continue; }
|
|
59
|
+
try { new RegExp(ir[k]); } catch { errs.push(`${k} is not a valid regular expression`); }
|
|
60
|
+
}
|
|
61
|
+
if ((ir.source || ir.propagator) && !ir.sink && !ir.match) errs.push('a source or propagator needs a sink');
|
|
62
|
+
if (ir.window != null && !(Number.isInteger(ir.window) && ir.window > 0 && ir.window <= 200)) errs.push('window must be an integer in 1..200');
|
|
63
|
+
if (ir.guardWindow != null && !(Number.isInteger(ir.guardWindow) && ir.guardWindow > 0 && ir.guardWindow <= 200)) errs.push('guardWindow must be an integer in 1..200');
|
|
64
|
+
if (ir.exclusions != null) {
|
|
65
|
+
if (!(Array.isArray(ir.exclusions) && ir.exclusions.every(isStr))) errs.push('exclusions must be an array of regular-expression strings');
|
|
66
|
+
else for (const re of ir.exclusions) { try { new RegExp(re); } catch { errs.push(`invalid exclusion regex: ${re}`); } }
|
|
46
67
|
}
|
|
47
|
-
if (ir.exclusions != null && !(Array.isArray(ir.exclusions) && ir.exclusions.every(isStr))) errs.push('exclusions must be an array of strings');
|
|
48
68
|
if (!ir.provenance || typeof ir.provenance !== 'object') errs.push('provenance required');
|
|
49
69
|
else {
|
|
50
70
|
if (!isStr(ir.provenance.originatingFindingId)) errs.push('provenance.originatingFindingId required');
|
|
@@ -74,14 +94,37 @@ export function draftRuleIR({ id, title, severity, cwe, family, languages, match
|
|
|
74
94
|
export function compileToCustomRule(ir) {
|
|
75
95
|
const errs = validateRuleIR(ir);
|
|
76
96
|
if (errs.length) return { ok: false, errs };
|
|
97
|
+
const alt = (...parts) => { const p = parts.filter(isStr); return p.length ? p.map((x) => `(?:${x})`).join('|') : null; };
|
|
98
|
+
// the base match: explicit wins; otherwise source -> propagator -> sink as a windowed co-occurrence, or a sink alone
|
|
99
|
+
let base = ir.match;
|
|
100
|
+
if (!base) {
|
|
101
|
+
// the SINK leads: the matcher anchors on the first pattern, and the finding, the guard window and the line exclusions all belong at
|
|
102
|
+
// the sink, where the flaw manifests. The window is symmetric, so the order of the others does not affect co-occurrence.
|
|
103
|
+
const chain = [ir.sink, ir.propagator, ir.source].filter(isStr);
|
|
104
|
+
base = chain.length > 1 ? { allOf: chain, window: ir.window ?? 8 } : { pattern: ir.sink };
|
|
105
|
+
}
|
|
106
|
+
const guard = alt(ir.sanitizer, ir.dominatingGuard);
|
|
77
107
|
const doc = {
|
|
78
108
|
id: `rule-compiler/${ir.id}`,
|
|
79
109
|
title: ir.title,
|
|
80
110
|
severity: ir.severity,
|
|
81
111
|
cwe: ir.cwe,
|
|
82
112
|
languages: ir.languages,
|
|
83
|
-
match: {
|
|
113
|
+
match: {
|
|
114
|
+
...(base.pattern ? { pattern: base.pattern } : {}),
|
|
115
|
+
...(base.allOf ? { allOf: base.allOf, window: base.window ?? 5 } : {}),
|
|
116
|
+
...(base.notMatch ? { notMatch: base.notMatch } : {}),
|
|
117
|
+
...(isStr(ir.frameworkConstraint) ? { fileRequires: [ir.frameworkConstraint] } : {}),
|
|
118
|
+
...(guard ? { guardedBy: guard, guardWindow: ir.guardWindow ?? base.window ?? 5 } : {}),
|
|
119
|
+
...(Array.isArray(ir.exclusions) && ir.exclusions.length ? { excludeLine: ir.exclusions } : {}),
|
|
120
|
+
},
|
|
84
121
|
message: ir.remediation || `${ir.title} (${ir.cwe}, compiled from ${ir.provenance.originatingFindingId})`,
|
|
85
122
|
};
|
|
123
|
+
// The loader has its own safety checks (nested quantifiers and so on). A rule it would refuse must not be reported as compiled,
|
|
124
|
+
// or it would score well here and silently never fire in a scan.
|
|
125
|
+
const origError = console.error; let loaderSaid = '';
|
|
126
|
+
console.error = (...a) => { loaderSaid = a.join(' '); };
|
|
127
|
+
let normalized; try { normalized = normalizeCustomRule(doc); } finally { console.error = origError; }
|
|
128
|
+
if (!normalized) return { ok: false, errs: [`the scanner's rule loader would reject this rule${loaderSaid ? `: ${loaderSaid.replace(/^agentic-security:\s*/, '').slice(0, 160)}` : ''}`] };
|
|
86
129
|
return { ok: true, doc };
|
|
87
130
|
}
|
|
@@ -29,6 +29,7 @@ export function variableRename(code, { suffix = '_v2' } = {}) {
|
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
export function wrapInFunction(code, { name = 'wrapped_v2' } = {}) {
|
|
32
|
+
if (!String(code).trim()) return null;
|
|
32
33
|
const indented = code.split('\n').map((l) => (l ? ` ${l}` : l)).join('\n');
|
|
33
34
|
return `function ${name}() {\n${indented}\n}\n${name}();\n`;
|
|
34
35
|
}
|
|
@@ -46,7 +47,71 @@ function insertIndirection(code) {
|
|
|
46
47
|
|
|
47
48
|
function _hash(s) { let h = 0; for (const c of s) h = (h * 31 + c.charCodeAt(0)) | 0; return h; }
|
|
48
49
|
|
|
49
|
-
|
|
50
|
+
// ---- the four mutators the PRD lists beyond rename / wrapper / indirection --------------------------------------------------
|
|
51
|
+
// Each is a deterministic, table-driven textual rewrite that returns null when it does not apply. Tables are deliberately
|
|
52
|
+
// small and conservative: an entry is here only when both sides are the same API family used the same way, so the claim "same
|
|
53
|
+
// vulnerability, different shape" stays true. They are applied to the vulnerable side (does the rule still fire?) and to the safe
|
|
54
|
+
// side (does it still decline?); the safe side is where a rule keyed to ONE sanitizer's name is caught.
|
|
55
|
+
|
|
56
|
+
// equivalent APIs: the same operation through a sibling entry point
|
|
57
|
+
const EQUIVALENT_APIS = Object.freeze([
|
|
58
|
+
[/\bchild_process\.exec\(/, 'child_process.execSync('], [/\bexecSync\(/, 'exec('], [/\bos\.system\(/, 'os.popen('],
|
|
59
|
+
[/\bdb\.query\(/, 'db.execute('], [/\bconnection\.query\(/, 'connection.execute('], [/\bcursor\.execute\(/, 'cursor.executemany('],
|
|
60
|
+
[/\bfs\.readFile\(/, 'fs.readFileSync('], [/\bfs\.readFileSync\(/, 'fs.readFile('], [/\bfs\.writeFile\(/, 'fs.writeFileSync('],
|
|
61
|
+
[/\bhttp\.get\(/, 'https.get('], [/\baxios\.get\(/, 'axios.request('], [/\bres\.send\(/, 'res.end('],
|
|
62
|
+
[/\bos\.Open\(/, 'os.OpenFile('], [/\bsubprocess\.call\(/, 'subprocess.run('], [/\bsubprocess\.run\(/, 'subprocess.call('],
|
|
63
|
+
]);
|
|
64
|
+
export function equivalentApi(code) {
|
|
65
|
+
for (const [from, to] of EQUIVALENT_APIS) if (from.test(code)) return code.replace(from, to);
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// alternate sanitizers: a different, equally valid way to neutralize the same input
|
|
70
|
+
const ALTERNATE_SANITIZERS = Object.freeze([
|
|
71
|
+
[/\bescapeHtml\(/, 'validator.escape('], [/\bvalidator\.escape\(/, 'escapeHtml('], [/\bDOMPurify\.sanitize\(/, 'sanitizeHtml('], [/\bsanitizeHtml\(/, 'DOMPurify.sanitize('],
|
|
72
|
+
[/\bshlex\.quote\(/, 'pipes.quote('], [/\bhtml\.escape\(/, 'markupsafe.escape('], [/\bhtml\.EscapeString\(/, 'template.HTMLEscapeString('],
|
|
73
|
+
[/\bencodeURIComponent\(/, 'encodeURI('], [/\bpath\.basename\(/, 'path.parse('], [/\bparseInt\(/, 'Number('], [/\bNumber\(/, 'parseInt('],
|
|
74
|
+
[/\bstrconv\.Atoi\(/, 'strconv.ParseInt('], [/\bint\(/, 'abs('],
|
|
75
|
+
]);
|
|
76
|
+
export function alternateSanitizer(code) {
|
|
77
|
+
for (const [from, to] of ALTERNATE_SANITIZERS) if (from.test(code)) return code.replace(from, to);
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// framework variants: the same untrusted input reached through another framework's accessor
|
|
82
|
+
const FRAMEWORK_VARIANTS = Object.freeze([
|
|
83
|
+
[/\breq\.body\./g, 'ctx.request.body.'], [/\breq\.query\./g, 'ctx.query.'], [/\breq\.params\./g, 'ctx.params.'],
|
|
84
|
+
[/\brequest\.args\.get\(/g, 'request.form.get('], [/\brequest\.form\.get\(/g, 'request.args.get('], [/\$_GET\[/g, '$_REQUEST['], [/\$_POST\[/g, '$_REQUEST['],
|
|
85
|
+
[/\br\.URL\.Query\(\)\.Get\(/g, 'r.FormValue('], [/\brequest\.getParameter\(/g, 'request.getHeader('],
|
|
86
|
+
]);
|
|
87
|
+
export function frameworkVariant(code) {
|
|
88
|
+
for (const [from, to] of FRAMEWORK_VARIANTS) if (from.test(code)) { from.lastIndex = 0; return code.replace(from, to); }
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// reordering: swap the first pair of ADJACENT simple statements that share no identifier, so neither feeds the other. This is a
|
|
93
|
+
// syntactic independence test, not a data-flow proof; it is applied to rule-corpus snippets, where a swap of two unrelated
|
|
94
|
+
// statements is what a different author's version of the same code would plausibly look like.
|
|
95
|
+
const CONTROL = /^\s*(?:if|else|for|while|switch|case|return|try|catch|finally|do|break|continue|throw|function|def|class|import|from|package|\}|\{|@|\/\/|#)\b|^\s*[}{]\s*$/;
|
|
96
|
+
export function reorderStatements(code) {
|
|
97
|
+
const lines = code.split('\n');
|
|
98
|
+
const simple = (l) => l.trim() && !CONTROL.test(l) && /[;)]\s*$|^[^=]*=[^=].*[^{]$/.test(l.trim()) && !/[{]\s*$/.test(l);
|
|
99
|
+
const ids = (l) => new Set([...l.matchAll(/\b[A-Za-z_][A-Za-z0-9_]*\b/g)].map((m) => m[0]).filter((x) => !RESERVED.has(x)));
|
|
100
|
+
for (let i = 0; i + 1 < lines.length; i++) {
|
|
101
|
+
if (!simple(lines[i]) || !simple(lines[i + 1])) continue;
|
|
102
|
+
const a = ids(lines[i]); const b = ids(lines[i + 1]);
|
|
103
|
+
if ([...a].some((x) => b.has(x))) continue;
|
|
104
|
+
const out = [...lines]; [out[i], out[i + 1]] = [out[i + 1], out[i]];
|
|
105
|
+
return out.join('\n');
|
|
106
|
+
}
|
|
107
|
+
return null;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const MUTATORS = Object.freeze({
|
|
111
|
+
'variable-rename': variableRename, 'wrap-in-function': wrapInFunction, 'insert-indirection': insertIndirection,
|
|
112
|
+
'equivalent-api': equivalentApi, 'reorder-statements': reorderStatements, 'alternate-sanitizer': alternateSanitizer, 'framework-variant': frameworkVariant,
|
|
113
|
+
});
|
|
114
|
+
export const MUTATOR_NAMES = Object.freeze(Object.keys(MUTATORS));
|
|
50
115
|
|
|
51
116
|
/** Every mutator applied to `code`; mutators that don't apply (return null) are omitted, never silently skipped without saying so. */
|
|
52
117
|
export function generateVariants(code) {
|
|
@@ -58,3 +123,9 @@ export function generateVariants(code) {
|
|
|
58
123
|
}
|
|
59
124
|
return out;
|
|
60
125
|
}
|
|
126
|
+
|
|
127
|
+
/** Which of the seven mutators produced a variant for this code, and which did not (so a report can say what was not exercised). */
|
|
128
|
+
export function mutatorCoverage(code) {
|
|
129
|
+
const applied = new Set(generateVariants(code).map((v) => v.mutator));
|
|
130
|
+
return { applied: MUTATOR_NAMES.filter((n) => applied.has(n)), notApplicable: MUTATOR_NAMES.filter((n) => !applied.has(n)) };
|
|
131
|
+
}
|
|
@@ -13,14 +13,30 @@ export const PROMOTION_DEFAULTS = Object.freeze({ minPrecision: 0.95, minRecall:
|
|
|
13
13
|
// checker rather than reaching across that isolation boundary.
|
|
14
14
|
const ADVISORY_ID = /\b(?:CVE|GHSA|CWE)-[A-Za-z0-9-]+\b/i;
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
16
|
+
// What a synthesis agent must never see (PRD 3, anti-cheating): benchmark answer keys, hidden test labels, CWE-bearing hidden file
|
|
17
|
+
// names, vulnerability comments and protected test metadata. These defaults are always applied; a caller adds its own benchmark names.
|
|
18
|
+
const DEFAULT_BENCHMARK_NAMES = Object.freeze(['owasp benchmark', 'juliet', 'sard test', 'cve-replay', 'corpus-baseline', 'answer key', 'ground truth', 'adjudicated', 'hidden test', 'expectedresults']);
|
|
19
|
+
const CWE_FILENAME = /(^|[\\/])[^\\/]*cwe[-_]?\d+[^\\/]*$/i;
|
|
20
|
+
const ANSWER_KEY_FILENAME = /(^|[\\/])(?:labels?|answers?|expected|ground[-_]?truth|answer[-_]?key|oracle|result)[^\\/]*\.(?:json|jsonl|ya?ml|csv|txt)$/i;
|
|
21
|
+
const VULNERABILITY_COMMENT = /(?:\/\/|#|\/\*|^\s*\*)\s*(?:POTENTIAL FLAW|FLAW\b|INCIDENTAL|BAD\b|GOOD\b|vulnerab\w*|insecure|exploit\w*)/im;
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Anti-cheating gate. Fails closed: any hit makes `ok` false, and the synthesis loop will not report a pass.
|
|
25
|
+
* @param {string} promptText everything the synthesis agent would be shown (corpus code, spec text)
|
|
26
|
+
* @param {{benchmarkNames?: string[], fileNames?: string[]}} opts
|
|
27
|
+
*/
|
|
28
|
+
export function checkLeakage(promptText, { benchmarkNames = [], fileNames = [] } = {}) {
|
|
19
29
|
const hay = String(promptText ?? '');
|
|
30
|
+
const lower = hay.toLowerCase();
|
|
20
31
|
const offending = [];
|
|
21
|
-
for (const name of benchmarkNames) if (name &&
|
|
32
|
+
for (const name of [...DEFAULT_BENCHMARK_NAMES, ...benchmarkNames]) if (name && lower.includes(String(name).toLowerCase())) offending.push(`benchmark name: ${name}`);
|
|
22
33
|
const m = ADVISORY_ID.exec(hay);
|
|
23
|
-
if (m) offending.push(m[0]);
|
|
34
|
+
if (m) offending.push(`identifier: ${m[0]}`);
|
|
35
|
+
if (VULNERABILITY_COMMENT.test(hay)) offending.push('a comment that describes the vulnerability');
|
|
36
|
+
for (const f of fileNames) {
|
|
37
|
+
if (CWE_FILENAME.test(String(f))) offending.push(`weakness-bearing file name: ${f}`);
|
|
38
|
+
if (ANSWER_KEY_FILENAME.test(String(f))) offending.push(`answer-key file name: ${f}`);
|
|
39
|
+
}
|
|
24
40
|
return { ok: offending.length === 0, offendingTerms: offending };
|
|
25
41
|
}
|
|
26
42
|
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
// The synthesis loop for the rule compiler (PRD 3, requirement 5): propose, compile, test positives, test negatives, mutation test,
|
|
2
|
+
// score, revise, repeat with bounded attempts, emit a candidate.
|
|
3
|
+
//
|
|
4
|
+
// "Models propose rules. Evidence promotes rules." A model's proposal (a `match` in the spec) is tried first when there is one.
|
|
5
|
+
// When there is none, or it fails, this module proposes deterministically from the two pieces of evidence it has: the vulnerable
|
|
6
|
+
// code and its patched twin. The lines that exist only in the vulnerable version say what the flaw looks like; the calls that exist
|
|
7
|
+
// only in the patched version say what a safe version looks like (and become the `notMatch` that keeps the rule quiet on safe files).
|
|
8
|
+
// No model is consulted here and nothing here promotes a rule: the loop returns attempts and a best candidate, and promotion stays
|
|
9
|
+
// with the gate and the lifecycle ledger.
|
|
10
|
+
//
|
|
11
|
+
// Every attempt is scored against the real corpus (positives with their mutations, negatives with theirs) by the same matcher the
|
|
12
|
+
// rule will run on once promoted. An attempt that does not clear the gate is revised from WHY it failed: low recall generalizes,
|
|
13
|
+
// a negative that fires adds the safe version's names to `notMatch`.
|
|
14
|
+
import { draftRuleIR, validateRuleIR } from './ir.js';
|
|
15
|
+
import { buildCorpus } from './corpus.js';
|
|
16
|
+
import { evaluateRule } from './promotion.js';
|
|
17
|
+
|
|
18
|
+
const esc = (s) => s.replace(/[.*+?^${}()|[\]\\\/]/g, '\\$&');
|
|
19
|
+
const KEYWORDS = new Set(['if', 'else', 'for', 'while', 'return', 'function', 'const', 'let', 'var', 'new', 'await', 'async', 'try', 'catch', 'throw', 'typeof', 'def', 'class', 'import', 'from', 'with', 'in', 'of', 'and', 'or', 'not', 'is', 'func', 'fn', 'public', 'private', 'static', 'void', 'int', 'string']);
|
|
20
|
+
const SOURCES = [
|
|
21
|
+
String.raw`req(?:uest)?\.(?:body|query|params|headers|cookies)\b`, String.raw`ctx\.(?:query|params|request\.body)\b`, String.raw`request\.(?:args|form|values|json|GET|POST)\b`,
|
|
22
|
+
String.raw`\$_(?:GET|POST|REQUEST|COOKIE)\b`, String.raw`process\.argv\b`, String.raw`os\.Args\b`, String.raw`\bgetParameter\s*\(`, String.raw`r\.(?:URL\.Query\(\)|FormValue)\b`,
|
|
23
|
+
];
|
|
24
|
+
|
|
25
|
+
const meaningful = (l) => { const t = l.trim(); return t.length >= 8 && !/^(?:\/\/|#|\/\*|\*|[{}()\];,]+$)/.test(t); };
|
|
26
|
+
|
|
27
|
+
/** Lines (trimmed) present only in `a`. */
|
|
28
|
+
function onlyIn(a, b) {
|
|
29
|
+
const bs = new Set(String(b ?? '').split('\n').map((l) => l.trim()));
|
|
30
|
+
return String(a ?? '').split('\n').filter((l) => meaningful(l) && !bs.has(l.trim())).map((l) => l.trim());
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Call names (a.b.c) in some code, in order of appearance, without keywords. */
|
|
34
|
+
function callNames(code) {
|
|
35
|
+
const out = [];
|
|
36
|
+
for (const m of String(code).matchAll(/\b([A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*)\s*\(/g)) if (!KEYWORDS.has(m[1]) && !out.includes(m[1])) out.push(m[1]);
|
|
37
|
+
return out;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** A regular expression for one source line: the call names stay, strings and the other identifiers become classes. */
|
|
41
|
+
export function generalizeLine(line) {
|
|
42
|
+
const keep = new Set(callNames(line));
|
|
43
|
+
let out = ''; let i = 0;
|
|
44
|
+
while (i < line.length) {
|
|
45
|
+
const c = line[i];
|
|
46
|
+
if (/\s/.test(c)) { const before = line[i - 1] ?? ''; while (/\s/.test(line[i] ?? '')) i++; out += /[\w$]/.test(before) && /[\w$]/.test(line[i] ?? '') ? String.raw`\s+` : String.raw`\s*`; continue; }
|
|
47
|
+
if (c === '"' || c === "'" || c === '`') { const q = c; let j = i + 1; while (j < line.length && line[j] !== q) j += line[j] === '\\' ? 2 : 1; out += q === '`' ? '`[^`]*`' : `(?:'[^']*'|"[^"]*")`; i = j + 1; continue; }
|
|
48
|
+
const id = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*/.exec(line.slice(i));
|
|
49
|
+
if (id) { const tok = id[0]; const isCall = line[i + tok.length] === '(' || keep.has(tok); out += isCall || KEYWORDS.has(tok) ? esc(tok) : String.raw`[A-Za-z_$][\w$.]*`; i += tok.length; continue; }
|
|
50
|
+
const num = /^\d+(?:\.\d+)?/.exec(line.slice(i));
|
|
51
|
+
if (num) { out += String.raw`\d+(?:\.\d+)?`; i += num[0].length; continue; }
|
|
52
|
+
out += esc(c); i++;
|
|
53
|
+
}
|
|
54
|
+
return out;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const sinkOf = (line) => callNames(line).filter((n) => !/^(?:console|logger|log)\b/.test(n))[0] ?? null;
|
|
58
|
+
const interpolation = String.raw`(?:\+|\$\{|%s|%d|\.format\s*\(|\bf["']|\|\|)`;
|
|
59
|
+
|
|
60
|
+
/** Ordered proposals, most specific first. Each is {strategy, match}; `match.notMatch` is added by `withSafeNames`. */
|
|
61
|
+
export function proposeCandidates({ vulnerableCode, patchedCode = null }) {
|
|
62
|
+
const vuln = onlyIn(vulnerableCode, patchedCode);
|
|
63
|
+
const lines = vuln.length ? vuln : String(vulnerableCode).split('\n').filter(meaningful).map((l) => l.trim()).filter((l) => callNames(l).length);
|
|
64
|
+
const out = []; const seen = new Set();
|
|
65
|
+
const add = (strategy, match, structure = null) => { const k = JSON.stringify([match, structure]); if (!seen.has(k)) { seen.add(k); out.push({ strategy, ...(match ? { match } : {}), ...(structure ? { structure } : {}) }); } };
|
|
66
|
+
for (const l of lines.slice(0, 3)) add('line-shape', { pattern: generalizeLine(l) });
|
|
67
|
+
for (const l of lines.slice(0, 3)) {
|
|
68
|
+
const sink = sinkOf(l);
|
|
69
|
+
if (sink) add('sink-with-interpolation', { pattern: `${esc(sink)}\\s*\\([^)\\n]*${interpolation}` });
|
|
70
|
+
}
|
|
71
|
+
const sink = lines.map(sinkOf).find(Boolean);
|
|
72
|
+
const src = SOURCES.find((s) => new RegExp(s).test(vulnerableCode));
|
|
73
|
+
if (sink && src) add('source-and-sink', { allOf: [`${esc(sink)}\\s*\\(`, src], window: 6 });
|
|
74
|
+
// the same evidence as an explicit structure: a source and a sink the matcher enforces, so a sanitizer found in the patched code can
|
|
75
|
+
// later be added as a LOCAL guard (dropping one hit) instead of a file-level exclusion (silencing the whole file)
|
|
76
|
+
if (sink && src) add('structural', null, { source: src, sink: `${esc(sink)}\\s*\\(`, window: 6 });
|
|
77
|
+
if (sink) add('sink-call', { pattern: `${esc(sink)}\\s*\\(` });
|
|
78
|
+
return out;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Names that appear in the patched code and not in the vulnerable code: the safe version's vocabulary. */
|
|
82
|
+
export function safeNames({ vulnerableCode, patchedCode }) {
|
|
83
|
+
if (!patchedCode) return [];
|
|
84
|
+
const v = new Set(callNames(vulnerableCode));
|
|
85
|
+
const names = callNames(patchedCode).filter((n) => !v.has(n));
|
|
86
|
+
if (/\?['"`]?\s*[,)]/.test(patchedCode) && !/\?['"`]?\s*[,)]/.test(vulnerableCode)) names.push('?');
|
|
87
|
+
return names;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const PLACEHOLDER_RE = '\\?[\'"`]?\\s*[,)]';
|
|
91
|
+
const safeRegex = (names) => (names.length ? names.map((n) => (n === '?' ? PLACEHOLDER_RE : esc(n))).join('|') : null);
|
|
92
|
+
|
|
93
|
+
const summary = (score) => ({ precision: score.precision, recall: score.recall, tp: score.tp, fp: score.fp, fn: score.fn, tn: score.tn, protectedFailures: score.protectedFailures });
|
|
94
|
+
const quality = (e) => (e.pass ? 10 : 0) + (e.score.precision ?? 0) * (e.score.recall ?? 0) - e.score.protectedFailures * 0.5;
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* @param {object} o
|
|
98
|
+
* @param {object} o.base the IR fields that are not the match: id, title, severity, cwe, family, languages, remediation, evidence
|
|
99
|
+
* @param {object} o.corpusSpec { originalCode, patchedCode?, nearbyCleanCode?, protectedVariantsCode? }
|
|
100
|
+
* @param {object|null} o.proposed a match supplied by a model or a person, tried first
|
|
101
|
+
* @returns {{attempts: object[], best: object|null, exhausted: boolean}}
|
|
102
|
+
*/
|
|
103
|
+
export function synthesizeCandidate({ base, corpusSpec, proposed = null, maxAttempts = 6, thresholds, leakageOk = true }) {
|
|
104
|
+
const cap = Math.max(1, Math.min(12, maxAttempts));
|
|
105
|
+
const queue = [];
|
|
106
|
+
if (proposed) queue.push({ strategy: 'proposed', match: proposed });
|
|
107
|
+
for (const p of proposeCandidates({ vulnerableCode: corpusSpec.originalCode, patchedCode: corpusSpec.patchedCode })) queue.push(p);
|
|
108
|
+
const names = safeNames({ vulnerableCode: corpusSpec.originalCode, patchedCode: corpusSpec.patchedCode });
|
|
109
|
+
const attempts = []; const tried = new Set();
|
|
110
|
+
const tryOne = (strategy, match, structure = null) => {
|
|
111
|
+
const key = JSON.stringify([match, structure]);
|
|
112
|
+
if (tried.has(key) || attempts.length >= cap) return null;
|
|
113
|
+
tried.add(key);
|
|
114
|
+
const ir = draftRuleIR({ ...base, ...(match ? { match } : {}), ...(structure ?? {}) });
|
|
115
|
+
const errs = validateRuleIR(ir);
|
|
116
|
+
if (errs.length) { attempts.push({ n: attempts.length + 1, strategy, match: match ?? null, structure, valid: false, errors: errs, pass: false, reasons: errs }); return null; }
|
|
117
|
+
const corpus = buildCorpus(ir, corpusSpec);
|
|
118
|
+
const ev = evaluateRule(ir, corpus, thresholds);
|
|
119
|
+
const rec = { n: attempts.length + 1, strategy, match: match ?? null, structure, valid: true, pass: ev.pass && leakageOk, reasons: ev.reasons, score: summary(ev.score), ir, corpusSizes: { positives: corpus.positives.length, negatives: corpus.negatives.length } };
|
|
120
|
+
attempts.push(rec);
|
|
121
|
+
return { rec, ev };
|
|
122
|
+
};
|
|
123
|
+
for (let qi = 0; qi < queue.length && attempts.length < cap; qi++) {
|
|
124
|
+
const q = queue[qi];
|
|
125
|
+
const first = tryOne(q.strategy, q.match, q.structure);
|
|
126
|
+
if (!first) continue;
|
|
127
|
+
if (first.rec.pass) break;
|
|
128
|
+
// revise from the failure: a negative that fired means the safe version is not excluded yet
|
|
129
|
+
const fired = first.ev.score.negatives.filter((n) => n.fired);
|
|
130
|
+
const rx = safeRegex(names);
|
|
131
|
+
if (fired.length && rx && q.structure && !q.structure.sanitizer) { const r = tryOne(`${q.strategy}+sanitizer`, null, { ...q.structure, sanitizer: rx }); if (r?.rec.pass) break; }
|
|
132
|
+
else if (fired.length && rx && q.match && !q.match.notMatch) { const r = tryOne(`${q.strategy}+exclude-safe`, { ...q.match, notMatch: rx }); if (r?.rec.pass) break; }
|
|
133
|
+
}
|
|
134
|
+
const valid = attempts.filter((a) => a.valid);
|
|
135
|
+
const best = valid.length ? [...valid].sort((a, b) => quality({ pass: b.pass, score: b.score }) - quality({ pass: a.pass, score: a.score }))[0] : null;
|
|
136
|
+
return { attempts: attempts.map(({ ir, ...rest }) => rest), best: best ? { strategy: best.strategy, match: best.match, structure: best.structure, pass: best.pass, score: best.score, reasons: best.reasons } : null, exhausted: !attempts.some((a) => a.pass) };
|
|
137
|
+
}
|