@clear-capabilities/agentic-security-scanner 0.161.0 → 0.163.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -0
- package/bin/agentic-security.js +98 -2
- package/dist/1340.index.js +1871 -0
- package/dist/3477.index.js +727 -0
- package/dist/3499.index.js +23 -3
- package/dist/4113.index.js +59 -8
- package/dist/5014.index.js +833 -0
- package/dist/5756.index.js +2 -2
- package/dist/6257.index.js +1 -1
- package/dist/6907.index.js +248 -0
- package/dist/6994.index.js +1 -1
- package/dist/7033.index.js +33 -5
- package/dist/7439.index.js +1822 -0
- package/dist/7838.index.js +38 -2
- package/dist/8798.index.js +1 -1
- package/dist/9207.index.js +5 -0
- package/dist/9560.index.js +2336 -27
- package/dist/957.index.js +1 -1
- package/dist/{1905.index.js → 9999.index.js} +306 -13
- package/dist/agentic-security.mjs +12 -12
- package/dist/agentic-security.mjs.sha256 +1 -1
- package/package.json +39 -7
- package/src/capabilities/CLAUDE.md +114 -0
- package/src/capabilities/address.js +73 -0
- package/src/capabilities/attack-coverage.js +93 -0
- package/src/capabilities/decide.js +256 -0
- package/src/capabilities/delegate.js +69 -0
- package/src/capabilities/egress.js +20 -0
- package/src/capabilities/hook-advice.js +50 -0
- package/src/capabilities/index.js +32 -0
- package/src/capabilities/manifest.js +344 -0
- package/src/capabilities/outbound.js +189 -0
- package/src/capabilities/paths.js +57 -0
- package/src/capabilities/probes.js +211 -0
- package/src/capabilities/proxy.js +182 -0
- package/src/capabilities/reasons.js +67 -0
- package/src/capabilities/receipts.js +356 -0
- package/src/capabilities/records.js +58 -0
- package/src/capabilities/recovery.js +367 -0
- package/src/capabilities/report.js +81 -0
- package/src/capabilities/runner.js +236 -0
- package/src/capabilities/secrets.js +26 -0
- package/src/capabilities/tool-gate.js +119 -0
- package/src/capabilities/tool-registry.js +60 -0
- package/src/dataflow/CLAUDE.md +29 -0
- package/src/dataflow/catalog.js +21 -0
- package/src/dataflow/engine.js +20 -0
- package/src/dataflow/guard-dominance.js +352 -0
- package/src/discovery/CLAUDE.md +18 -0
- package/src/discovery/index.js +43 -1
- package/src/discovery/lenses.js +18 -0
- package/src/discovery/memory.js +7 -5
- package/src/engine.js +80 -15
- package/src/fix/apply-fix-service.js +33 -3
- package/src/integrations/index.js +5 -2
- package/src/ir/CLAUDE.md +14 -0
- package/src/ir/implicit-handler-params.js +74 -0
- package/src/ir/parser-cs.js +20 -0
- package/src/ir/parser-go.js +30 -13
- package/src/ir/parser-java.js +11 -0
- package/src/language/nix-eval-isolation.js +33 -5
- package/src/lineage/CLAUDE.md +31 -0
- package/src/lineage/deployment/adapter-cloud.js +202 -0
- package/src/lineage/deployment/adapter-compose.js +90 -0
- package/src/lineage/deployment/adapter-kit.js +142 -0
- package/src/lineage/deployment/adapter-kubernetes.js +247 -0
- package/src/lineage/deployment/attack-paths.js +443 -0
- package/src/lineage/deployment/boundaries-run.js +223 -0
- package/src/lineage/deployment/boundary-graph.js +567 -0
- package/src/lineage/deployment/cross-repo.js +253 -0
- package/src/lineage/deployment/drift.js +383 -0
- package/src/lineage/deployment/ingest.js +221 -0
- package/src/lineage/deployment/projection.js +244 -0
- package/src/lineage/deployment/trace-correlation.js +541 -0
- package/src/lineage/source-registry.js +19 -0
- package/src/mcp/CLAUDE.md +14 -3
- package/src/mcp/invariant-tools.js +64 -0
- package/src/mcp/portfolio-tools.js +79 -0
- package/src/mcp/redact.js +15 -0
- package/src/mcp/server.js +24 -3
- package/src/mcp/tools.js +30 -5
- package/src/posture/CLAUDE.md +505 -0
- package/src/posture/accuracy-scorecard.js +95 -4
- package/src/posture/assurance/baseline-inventory.js +218 -0
- package/src/posture/assurance/baseline.js +322 -0
- package/src/posture/assurance/bounded-io.js +145 -0
- package/src/posture/assurance/config.js +299 -0
- package/src/posture/assurance/contracts.js +315 -0
- package/src/posture/assurance/identity.js +62 -0
- package/src/posture/assurance/migrations.js +259 -0
- package/src/posture/assurance/rollback.js +214 -0
- package/src/posture/assurance/rollout.js +200 -0
- package/src/posture/assurance/schema-kit.js +111 -0
- package/src/posture/assurance/verification-record.js +258 -0
- package/src/posture/attestation.js +5 -0
- package/src/posture/autopilot.js +28 -1
- package/src/posture/business-logic.js +26 -4
- package/src/posture/evaluation/bakeoff.js +148 -0
- package/src/posture/evaluation/custody.js +200 -0
- package/src/posture/evaluation/deployment-ablation.js +416 -0
- package/src/posture/evaluation/dev-cases.js +65 -0
- package/src/posture/evaluation/economics.js +85 -0
- package/src/posture/evaluation/gates.js +161 -0
- package/src/posture/evaluation/grouping.js +101 -0
- package/src/posture/evaluation/interval.js +132 -0
- package/src/posture/evaluation/invariant-ablation.js +325 -0
- package/src/posture/evaluation/labels.js +200 -0
- package/src/posture/evaluation/protocol.js +308 -0
- package/src/posture/evaluation/release-gate.js +209 -0
- package/src/posture/evaluation/report.js +158 -0
- package/src/posture/evaluation/runner.js +273 -0
- package/src/posture/evaluation/scan-child.js +80 -0
- package/src/posture/evaluation/score.js +174 -0
- package/src/posture/evaluation/synthetic.js +90 -0
- package/src/posture/evaluation/why-missed.js +219 -0
- package/src/posture/evidence-bundle.js +93 -7
- package/src/posture/execution-proof.js +11 -1
- package/src/posture/fix-verify-loop.js +20 -6
- package/src/posture/fix-verify.js +37 -2
- package/src/posture/fleet.js +11 -2
- package/src/posture/invariants/cli.js +95 -0
- package/src/posture/invariants/coverage.js +164 -0
- package/src/posture/invariants/export.js +178 -0
- package/src/posture/invariants/expressions.js +46 -0
- package/src/posture/invariants/lifecycle.js +289 -0
- package/src/posture/invariants/project-input.js +55 -0
- package/src/posture/invariants/repair.js +340 -0
- package/src/posture/invariants/run.js +45 -0
- package/src/posture/invariants/scenarios.js +311 -0
- package/src/posture/invariants/schema.js +167 -0
- package/src/posture/invariants/shrink.js +407 -0
- package/src/posture/invariants/state-assertions.js +165 -0
- package/src/posture/logic-claims.js +12 -0
- package/src/posture/model-routing.js +40 -0
- package/src/posture/oracles/adapters.js +531 -0
- package/src/posture/oracles/conformance.js +276 -0
- package/src/posture/oracles/oracle.js +453 -0
- package/src/posture/oracles/registry.js +76 -0
- package/src/posture/oracles/scenario-classes.js +223 -0
- package/src/posture/portfolio/backend.js +279 -0
- package/src/posture/portfolio/bundle.js +314 -0
- package/src/posture/portfolio/cli.js +142 -0
- package/src/posture/portfolio/manifest.js +263 -0
- package/src/posture/portfolio/progress.js +267 -0
- package/src/posture/portfolio/resume.js +130 -0
- package/src/posture/portfolio/retention.js +207 -0
- package/src/posture/portfolio/scheduler.js +488 -0
- package/src/posture/portfolio/signing.js +155 -0
- package/src/posture/portfolio/wording.js +59 -0
- package/src/posture/portfolio/work-units.js +405 -0
- package/src/posture/replay/replay.js +250 -0
- package/src/posture/routing/adapter.js +140 -0
- package/src/posture/routing/calibration.js +278 -0
- package/src/posture/routing/canary.js +141 -0
- package/src/posture/routing/control.js +35 -0
- package/src/posture/routing/decide.js +211 -0
- package/src/posture/routing/drift.js +137 -0
- package/src/posture/routing/economics.js +185 -0
- package/src/posture/routing/feedback.js +207 -0
- package/src/posture/routing/index.js +31 -0
- package/src/posture/routing/outcomes.js +281 -0
- package/src/posture/routing/promotion.js +291 -0
- package/src/posture/routing/receipts.js +59 -0
- package/src/posture/routing/report.js +142 -0
- package/src/posture/routing/shadow.js +53 -0
- package/src/posture/specification-mining.js +26 -0
- package/src/posture/verification/advisory-state.js +71 -0
- package/src/posture/verification/emit.js +257 -0
- package/src/posture/verification/hypothesis-promotion.js +211 -0
- package/src/posture/verification/patch-negative.js +226 -0
- package/src/posture/verification/patch-promotion.js +113 -0
- package/src/posture/verification/projection.js +183 -0
- package/src/posture/verification/repair-records.js +124 -0
- package/src/posture/verifier.js +16 -0
- package/src/report/index.js +100 -1
- package/src/sandbox/CLAUDE.md +51 -0
- package/src/sandbox/backend-namespace.js +7 -0
- package/src/sandbox/backend-userspace.js +151 -34
- package/src/sandbox/control-probes.js +215 -0
- package/src/sandbox/supervise.js +201 -0
- package/src/sandbox/trust-boundary.js +113 -0
- package/src/sandbox/trust-domains.js +171 -0
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
// Real-code quality gates over a frozen protocol (section 5 of the programme PRD).
|
|
2
|
+
//
|
|
3
|
+
// This module can answer `pass` ONLY when every population minimum is met by
|
|
4
|
+
// real, independently adjudicated, non-synthetic labels inside the sealed split,
|
|
5
|
+
// AND the measured value clears the threshold registered in the protocol. In
|
|
6
|
+
// every other case it answers `insufficient-population` or `unmeasured`, with the
|
|
7
|
+
// reason. It never answers `pass` for a smaller convenient sample, for synthetic
|
|
8
|
+
// data, or for a threshold it was handed by the caller: thresholds are read from
|
|
9
|
+
// the protocol, and a protocol that fails its hash check is rejected outright.
|
|
10
|
+
//
|
|
11
|
+
// The statuses are deliberately not boolean: `unmeasured` is not `fail`, and
|
|
12
|
+
// neither is a pass.
|
|
13
|
+
|
|
14
|
+
import { validateProtocol, CORE_LANGUAGES } from './protocol.js';
|
|
15
|
+
import { isScoreable } from './labels.js';
|
|
16
|
+
import { groupTargets } from './grouping.js';
|
|
17
|
+
import { groupedBootstrap, f1Of } from './interval.js';
|
|
18
|
+
|
|
19
|
+
export const GATE_STATUSES = Object.freeze(['pass', 'fail', 'insufficient-population', 'unmeasured']);
|
|
20
|
+
|
|
21
|
+
const gate = (id, status, extra = {}) => ({ id, status, ...extra });
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* @param {object} o
|
|
25
|
+
* @param {object} o.protocol frozen protocol (thresholds and splits come from here)
|
|
26
|
+
* @param {object} o.score a score (score.js) of the SEALED split of a run bound to this protocol
|
|
27
|
+
* @param {object[]} o.defects all defect labels
|
|
28
|
+
* @param {object[]} o.negatives all negative labels
|
|
29
|
+
* @param {object} [o.run] the sealed-split run record the score came from; enables the integrity gates (leaks, denominator)
|
|
30
|
+
* @param {object} [o.baseline] the last verified release score; enables the no-regression gate
|
|
31
|
+
*/
|
|
32
|
+
export function evaluateGates({ protocol, score, defects = [], negatives = [], run = null, baseline = null }) {
|
|
33
|
+
const v = validateProtocol(protocol);
|
|
34
|
+
if (!v.ok) return { ok: false, rejected: true, errors: v.errors, overall: 'rejected', gates: [] };
|
|
35
|
+
if (score && score.protocolHash !== protocol.protocolHash) {
|
|
36
|
+
return { ok: false, rejected: true, errors: [{ code: 'PROTOCOL_MISMATCH', path: 'score.protocolHash', message: 'the score was produced under a different protocol; it cannot be judged against these thresholds' }], overall: 'rejected', gates: [] };
|
|
37
|
+
}
|
|
38
|
+
const t = protocol.thresholds;
|
|
39
|
+
const gates = [];
|
|
40
|
+
const sealed = new Set(protocol.splits.sealed);
|
|
41
|
+
|
|
42
|
+
const realD = defects.filter((l) => isScoreable(l) && !l.synthetic && sealed.has(l.targetId));
|
|
43
|
+
const realN = negatives.filter((l) => isScoreable(l) && !l.synthetic && sealed.has(l.targetId));
|
|
44
|
+
const count = (xs, key, val) => xs.filter((x) => x[key] === val).length;
|
|
45
|
+
const shortfalls = [];
|
|
46
|
+
for (const lang of CORE_LANGUAGES) {
|
|
47
|
+
const p = count(realD, 'language', lang); const n = count(realN, 'language', lang);
|
|
48
|
+
if (p < t.minPositivesPerCoreLanguage) shortfalls.push(`${lang}: ${p}/${t.minPositivesPerCoreLanguage} positives`);
|
|
49
|
+
if (n < t.minNegativesPerCoreLanguage) shortfalls.push(`${lang}: ${n}/${t.minNegativesPerCoreLanguage} negatives`);
|
|
50
|
+
}
|
|
51
|
+
const familyShort = [...new Set(protocol.scope.families)].filter((f) => count(realD, 'family', f) < t.minPositivesPerFamily);
|
|
52
|
+
const populationOk = shortfalls.length === 0 && familyShort.length === 0;
|
|
53
|
+
const popDetail = {
|
|
54
|
+
reason: protocol.synthetic
|
|
55
|
+
? 'the protocol is synthetic: a synthetic population can exercise the machinery and can never satisfy a real-code gate'
|
|
56
|
+
: `population minimums not met: ${shortfalls.slice(0, 6).join('; ')}${shortfalls.length > 6 ? `; and ${shortfalls.length - 6} more` : ''}${familyShort.length ? `; ${familyShort.length} famil${familyShort.length === 1 ? 'y' : 'ies'} below ${t.minPositivesPerFamily} sealed positives` : ''}`,
|
|
57
|
+
realSealedPositives: realD.length, realSealedNegatives: realN.length,
|
|
58
|
+
};
|
|
59
|
+
const blocked = protocol.synthetic || !populationOk || !score;
|
|
60
|
+
const unmet = (id, threshold) => gate(id, score ? 'insufficient-population' : 'unmeasured', {
|
|
61
|
+
threshold, measured: null, ...(score ? popDetail : { reason: 'no sealed-split score exists' }),
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
const e2e = score?.endToEnd;
|
|
65
|
+
const decide = (id, threshold, measured, cmp = (m, th) => m >= th) => {
|
|
66
|
+
if (blocked) return unmet(id, threshold);
|
|
67
|
+
if (measured === null || measured === undefined) return gate(id, 'unmeasured', { threshold, measured: null, reason: 'value undefined on this population' });
|
|
68
|
+
return gate(id, cmp(measured, threshold) ? 'pass' : 'fail', { threshold, measured });
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
gates.push(decide('overall-micro-f1', t.overallMicroF1, e2e?.micro?.f1));
|
|
72
|
+
gates.push(decide('overall-macro-f1', t.overallMacroF1, e2e?.macroF1));
|
|
73
|
+
gates.push(decide('pooled-precision', t.pooledPrecision, e2e?.micro?.precision));
|
|
74
|
+
gates.push(decide('pooled-recall', t.pooledRecall, e2e?.micro?.recall));
|
|
75
|
+
gates.push(decide('completion', t.completion, score?.completion?.rate));
|
|
76
|
+
for (const lang of CORE_LANGUAGES) gates.push(decide(`per-language-f1:${lang}`, t.perLanguageF1, e2e?.byLanguage?.[lang]?.f1 ?? null));
|
|
77
|
+
gates.push(lowerBoundGate({ protocol, score, blocked, popDetail }));
|
|
78
|
+
for (const g of integrityGates({ protocol, score, run, defects, negatives })) gates.push(g);
|
|
79
|
+
if (baseline) gates.push(regressionGate({ score, baseline, blocked, popDetail }));
|
|
80
|
+
|
|
81
|
+
const overall = gates.every((g) => g.status === 'pass') ? 'pass'
|
|
82
|
+
: gates.some((g) => g.status === 'fail') ? 'fail'
|
|
83
|
+
: gates.some((g) => g.status === 'insufficient-population') ? 'insufficient-population' : 'unmeasured';
|
|
84
|
+
return { ok: true, rejected: false, errors: [], protocolHash: protocol.protocolHash, synthetic: protocol.synthetic, overall, gates, population: popDetail };
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// ---------------------------------------------------------------- the interval gate
|
|
88
|
+
|
|
89
|
+
/** The lowest per-language F1 lower bound (grouped bootstrap) against the registered bound. Unmeasured, never assumed. */
|
|
90
|
+
function lowerBoundGate({ protocol, score, blocked, popDetail }) {
|
|
91
|
+
const id = 'per-language-f1-lower-bound';
|
|
92
|
+
const threshold = protocol.thresholds.perLanguageF1LowerBound;
|
|
93
|
+
if (!score) return gate(id, 'unmeasured', { threshold, measured: null, reason: 'no sealed-split score exists' });
|
|
94
|
+
if (blocked) return gate(id, 'insufficient-population', { threshold, measured: null, reason: popDetail.reason });
|
|
95
|
+
if (!score.perTarget || typeof score.perTarget !== 'object') return gate(id, 'unmeasured', { threshold, measured: null, reason: 'the score carries no per-target tallies, so no grouped interval can be computed' });
|
|
96
|
+
const { groupOf } = groupTargets(protocol.targets);
|
|
97
|
+
const units = Object.entries(score.perTarget).map(([tid, c]) => ({ id: tid, ...c }));
|
|
98
|
+
const perLanguage = {};
|
|
99
|
+
for (const lang of CORE_LANGUAGES) {
|
|
100
|
+
perLanguage[lang] = groupedBootstrap({
|
|
101
|
+
units: units.filter((u) => u.language === lang), groupOf, statistic: (a) => f1Of(a.total), name: `lang:${lang}:f1`, protocolHash: protocol.protocolHash,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
const measured = Object.values(perLanguage).filter((iv) => iv.status === 'measured');
|
|
105
|
+
const unmeasured = Object.entries(perLanguage).filter(([, iv]) => iv.status !== 'measured');
|
|
106
|
+
const lows = measured.map((iv) => iv.low);
|
|
107
|
+
const min = lows.length ? Math.min(...lows) : null;
|
|
108
|
+
const detail = Object.fromEntries(Object.entries(perLanguage).map(([l, iv]) => [l, { status: iv.status, low: iv.low, high: iv.high, groups: iv.groups }]));
|
|
109
|
+
if (min !== null && min < threshold) return gate(id, 'fail', { threshold, measured: min, method: protocol.thresholds.intervalMethod, perLanguage: detail, reason: 'at least one language has an F1 lower bound under the registered bound' });
|
|
110
|
+
if (unmeasured.length) return gate(id, 'unmeasured', { threshold, measured: min, method: protocol.thresholds.intervalMethod, perLanguage: detail, reason: `${unmeasured.length} language${unmeasured.length === 1 ? '' : 's'} without enough independent groups for an interval: ${unmeasured.slice(0, 4).map(([l]) => l).join(', ')}` });
|
|
111
|
+
return gate(id, 'pass', { threshold, measured: min, method: protocol.thresholds.intervalMethod, perLanguage: detail });
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// ---------------------------------------------------------------- integrity gates
|
|
115
|
+
|
|
116
|
+
const LEAK_REASON = /leakage control/i;
|
|
117
|
+
|
|
118
|
+
/** Gates that judge the evaluation's own integrity. They are measurable on any population, so they never read insufficient-population. */
|
|
119
|
+
function integrityGates({ protocol, score, run, defects, negatives }) {
|
|
120
|
+
// Without a run record there is nothing to check; release-gate.js refuses to produce a release verdict without one.
|
|
121
|
+
if (!run) return [];
|
|
122
|
+
const leaks = (run.outcomes || []).filter((o) => o.status === 'quarantined' && LEAK_REASON.test(o.failureReason || ''));
|
|
123
|
+
const out = [gate('answer-key-leaks', leaks.length === 0 ? 'pass' : 'fail', { threshold: 0, measured: leaks.length, ...(leaks.length ? { reason: `${leaks.length} target workspace(s) carried label or answer-key material`, targets: leaks.map((o) => o.targetId).sort() } : {}) })];
|
|
124
|
+
|
|
125
|
+
// The denominator is the frozen population, not whatever the run happened to cover.
|
|
126
|
+
const problems = [];
|
|
127
|
+
if (run.protocolHash !== protocol.protocolHash) problems.push('run is bound to a different protocol');
|
|
128
|
+
const have = new Set((run.outcomes || []).map((o) => `${o.targetId}|${o.variant}`));
|
|
129
|
+
for (const t of protocol.targets.filter((x) => protocol.splits.sealed.includes(x.id))) {
|
|
130
|
+
if (!have.has(`${t.id}|pre`)) problems.push(`sealed target ${t.id} has no pre outcome`);
|
|
131
|
+
if (t.postCommit && !have.has(`${t.id}|post`)) problems.push(`sealed target ${t.id} has no post outcome`);
|
|
132
|
+
}
|
|
133
|
+
const sealed = new Set(protocol.splits.sealed);
|
|
134
|
+
const positives = defects.filter((l) => isScoreable(l) && sealed.has(l.targetId)).length;
|
|
135
|
+
const negs = negatives.filter((l) => isScoreable(l) && sealed.has(l.targetId)).length;
|
|
136
|
+
if (score?.endToEnd?.micro && Number.isFinite(score.endToEnd.micro.tp)) {
|
|
137
|
+
const m = score.endToEnd.micro;
|
|
138
|
+
if (m.tp + m.fn !== positives) problems.push(`scored positives ${m.tp + m.fn} differ from the ${positives} adjudicated sealed positives`);
|
|
139
|
+
const negScored = m.tn + m.fp;
|
|
140
|
+
const failed = (score.negativeFailures || []).length;
|
|
141
|
+
if (Array.isArray(score.negativeFailures) && negScored + failed < negs && negScored + failed !== negs) problems.push(`scored negatives ${negScored + failed} differ from the ${negs} adjudicated sealed negatives`);
|
|
142
|
+
}
|
|
143
|
+
out.push(gate('denominator-intact', problems.length === 0 ? 'pass' : 'fail', { threshold: null, measured: problems.length, ...(problems.length ? { reason: problems.slice(0, 5).join('; '), problems: problems.slice(0, 20) } : {}) }));
|
|
144
|
+
return out;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** Compare to the last VERIFIED release score; a drop in micro F1 or in any language's F1 fails. */
|
|
148
|
+
function regressionGate({ score, baseline, blocked, popDetail }) {
|
|
149
|
+
const id = 'no-regression-vs-baseline';
|
|
150
|
+
const cur = score?.endToEnd; const base = baseline?.endToEnd;
|
|
151
|
+
if (!cur || !base) return gate(id, 'unmeasured', { threshold: 'baseline', measured: null, reason: 'a score or a baseline is missing' });
|
|
152
|
+
const regressions = [];
|
|
153
|
+
const cmp = (name, c, b) => { if (typeof c === 'number' && typeof b === 'number' && c < b) regressions.push({ name, baseline: b, current: c }); };
|
|
154
|
+
cmp('micro-f1', cur.micro?.f1, base.micro?.f1);
|
|
155
|
+
cmp('micro-precision', cur.micro?.precision, base.micro?.precision);
|
|
156
|
+
cmp('micro-recall', cur.micro?.recall, base.micro?.recall);
|
|
157
|
+
for (const [lang, b] of Object.entries(base.byLanguage || {})) cmp(`f1:${lang}`, cur.byLanguage?.[lang]?.f1, b?.f1);
|
|
158
|
+
if (regressions.length) return gate(id, 'fail', { threshold: 'baseline', measured: regressions.length, regressions, reason: 'the current engine scores below the recorded baseline' });
|
|
159
|
+
if (blocked) return gate(id, 'insufficient-population', { threshold: 'baseline', measured: null, reason: popDetail.reason });
|
|
160
|
+
return gate(id, 'pass', { threshold: 'baseline', measured: 0 });
|
|
161
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
// Leakage-safe grouping and splitting (QA-001.AC02).
|
|
2
|
+
//
|
|
3
|
+
// Related samples must land on the same side of the development/sealed line, or
|
|
4
|
+
// the sealed set is not independent of what detector work already saw. Two
|
|
5
|
+
// samples are related when they share any of: a vulnerable/fixed pair, an
|
|
6
|
+
// upstream project (a fork resolves to its parent), an advisory id (duplicate
|
|
7
|
+
// advisories), a commit, or a near-identical template fingerprint. Relations are
|
|
8
|
+
// transitive (union-find), so a chain A~B~C is one group.
|
|
9
|
+
//
|
|
10
|
+
// Pure and deterministic: group ids and split assignment depend only on the
|
|
11
|
+
// targets and an explicit salt, never on iteration order or a clock.
|
|
12
|
+
|
|
13
|
+
import * as crypto from 'node:crypto';
|
|
14
|
+
|
|
15
|
+
/** `https://GitHub.com/Org/Repo.git/` and `git@github.com:org/repo` normalise to `github.com/org/repo`. */
|
|
16
|
+
export function normalizeUpstream(url) {
|
|
17
|
+
if (typeof url !== 'string' || !url.trim()) return null;
|
|
18
|
+
let u = url.trim().toLowerCase();
|
|
19
|
+
u = u.replace(/^[a-z+]+:\/\//, '').replace(/^git@/, '').replace(/^[^@/]+@/, '');
|
|
20
|
+
u = u.replace(':', '/').replace(/\/+$/, '').replace(/\.git$/, '');
|
|
21
|
+
return u || null;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
class UnionFind {
|
|
25
|
+
constructor(ids) { this.parent = new Map(ids.map((i) => [i, i])); }
|
|
26
|
+
find(x) { let r = x; while (this.parent.get(r) !== r) r = this.parent.get(r); this.parent.set(x, r); return r; }
|
|
27
|
+
union(a, b) { const ra = this.find(a); const rb = this.find(b); if (ra !== rb) this.parent.set(ra < rb ? rb : ra, ra < rb ? ra : rb); }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const KEYS = (t) => {
|
|
31
|
+
const k = [];
|
|
32
|
+
if (t.pairId) k.push(['pair', t.pairId]);
|
|
33
|
+
const up = normalizeUpstream(t.upstream);
|
|
34
|
+
if (up) k.push(['upstream', up]);
|
|
35
|
+
const fork = normalizeUpstream(t.forkOf);
|
|
36
|
+
if (fork) k.push(['upstream', fork]);
|
|
37
|
+
for (const a of t.advisoryIds || []) k.push(['advisory', String(a).toUpperCase()]);
|
|
38
|
+
for (const c of [t.preCommit, t.postCommit]) if (c) k.push(['commit', c]);
|
|
39
|
+
if (t.templateFingerprint) k.push(['template', t.templateFingerprint]);
|
|
40
|
+
return k;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Group targets by every relation above. Returns `{ groups, groupOf }` where each
|
|
45
|
+
* group is `{ groupId, members[], reasons[] }`, members sorted, groups sorted.
|
|
46
|
+
*/
|
|
47
|
+
export function groupTargets(targets) {
|
|
48
|
+
const list = (targets || []).filter((t) => t && t.id);
|
|
49
|
+
const uf = new UnionFind(list.map((t) => t.id));
|
|
50
|
+
const byKey = new Map();
|
|
51
|
+
for (const t of list) {
|
|
52
|
+
for (const [kind, val] of KEYS(t)) {
|
|
53
|
+
const key = `${kind}:${val}`;
|
|
54
|
+
if (byKey.has(key)) uf.union(byKey.get(key).id, t.id);
|
|
55
|
+
else byKey.set(key, t);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
const members = new Map();
|
|
59
|
+
for (const t of list) { const r = uf.find(t.id); if (!members.has(r)) members.set(r, []); members.get(r).push(t.id); }
|
|
60
|
+
// Which relation kinds actually linked two or more members of each group.
|
|
61
|
+
const kindCount = new Map();
|
|
62
|
+
for (const t of list) {
|
|
63
|
+
for (const [kind, val] of KEYS(t)) {
|
|
64
|
+
const gk = `${uf.find(t.id)}|${kind}:${val}`;
|
|
65
|
+
kindCount.set(gk, (kindCount.get(gk) || 0) + 1);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
const groups = [];
|
|
69
|
+
const groupOf = {};
|
|
70
|
+
for (const [root, ids] of members) {
|
|
71
|
+
ids.sort();
|
|
72
|
+
const reasons = new Set();
|
|
73
|
+
for (const [gk, n] of kindCount) if (n > 1 && gk.startsWith(`${root}|`)) reasons.add(gk.slice(root.length + 1).split(':')[0]);
|
|
74
|
+
const groupId = `grp:${crypto.createHash('sha256').update(ids.join('\n')).digest('hex').slice(0, 16)}`;
|
|
75
|
+
groups.push({ groupId, members: ids, reasons: [...reasons].sort() });
|
|
76
|
+
for (const id of ids) groupOf[id] = groupId;
|
|
77
|
+
}
|
|
78
|
+
groups.sort((a, b) => a.groupId.localeCompare(b.groupId));
|
|
79
|
+
return { groups, groupOf };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Deterministically assign whole groups to development or sealed. A group's side
|
|
84
|
+
* is a function of its id and the salt only, so adding an unrelated target never
|
|
85
|
+
* moves an existing group, and no group can straddle the line.
|
|
86
|
+
*/
|
|
87
|
+
export function assignSplits(groups, { sealedFraction = 0.3, salt = '' } = {}) {
|
|
88
|
+
const dev = []; const sealed = [];
|
|
89
|
+
for (const g of groups) {
|
|
90
|
+
const h = crypto.createHash('sha256').update(`${salt}|${g.groupId}`).digest();
|
|
91
|
+
const u = h.readUInt32BE(0) / 0x100000000;
|
|
92
|
+
(u < sealedFraction ? sealed : dev).push(...g.members);
|
|
93
|
+
}
|
|
94
|
+
return { dev: dev.sort(), sealed: sealed.sort() };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Groups whose members are spread over both sides. Empty means the split is leakage-safe. */
|
|
98
|
+
export function splitStraddles(groups, splits) {
|
|
99
|
+
const dev = new Set(splits?.dev || []); const sealed = new Set(splits?.sealed || []);
|
|
100
|
+
return (groups || []).filter((g) => g.members.some((m) => dev.has(m)) && g.members.some((m) => sealed.has(m)));
|
|
101
|
+
}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
// Documented 95% interval methods (QA-004.AC01).
|
|
2
|
+
//
|
|
3
|
+
// Two methods, each registered by name in the protocol (`thresholds.intervalMethod`) before any result is observed:
|
|
4
|
+
//
|
|
5
|
+
// grouped-bootstrap-95 For F1, precision and recall, and for the micro and macro aggregates. Resamples GROUPS, not cases:
|
|
6
|
+
// targets that share a pair, upstream, advisory, commit or template (grouping.js) are one resampling
|
|
7
|
+
// unit, because treating near-duplicates as independent draws makes the interval narrower than the
|
|
8
|
+
// evidence supports. The percentile method over `replicates` resamples (default 2000), driven by a
|
|
9
|
+
// seeded generator, so the same score and protocol produce the same interval to the last digit.
|
|
10
|
+
// The seed is derived from the protocol hash: nobody picks a seed after seeing a result.
|
|
11
|
+
// wilson-95 For a single proportion (the completion rate), where there is no group structure to resample.
|
|
12
|
+
//
|
|
13
|
+
// When there are too few groups for resampling to mean anything (`minGroups`, default 10), the interval is `unmeasured` with the
|
|
14
|
+
// reason, never a degenerate zero-width or a made-up range. An undefined statistic in a resample (no positives drawn) is skipped
|
|
15
|
+
// and counted, and a method that skips too many resamples is itself reported `unmeasured`.
|
|
16
|
+
//
|
|
17
|
+
// Pure: no fs, no clock, no Math.random.
|
|
18
|
+
|
|
19
|
+
import * as crypto from 'node:crypto';
|
|
20
|
+
|
|
21
|
+
export const INTERVAL_METHODS = Object.freeze({
|
|
22
|
+
'grouped-bootstrap-95': Object.freeze({
|
|
23
|
+
level: 0.95,
|
|
24
|
+
unit: 'target group (grouping.js union-find over pair, upstream, advisory, commit, template)',
|
|
25
|
+
estimator: 'percentile',
|
|
26
|
+
replicates: 2000,
|
|
27
|
+
minGroups: 10,
|
|
28
|
+
maxUndefinedFraction: 0.1,
|
|
29
|
+
seed: 'first 32 bits of sha256(protocolHash + "|" + statisticName)',
|
|
30
|
+
}),
|
|
31
|
+
'wilson-95': Object.freeze({ level: 0.95, unit: 'case', estimator: 'wilson-score', z: 1.959964 }),
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
/** mulberry32: a small, well-distributed 32-bit generator. Deterministic for a given seed. */
|
|
35
|
+
export function seededRng(seed) {
|
|
36
|
+
let a = seed >>> 0;
|
|
37
|
+
return () => {
|
|
38
|
+
a = (a + 0x6D2B79F5) >>> 0;
|
|
39
|
+
let t = a;
|
|
40
|
+
t = Math.imul(t ^ (t >>> 15), t | 1);
|
|
41
|
+
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
|
42
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** A 32-bit seed that depends only on the frozen protocol and the statistic's name. */
|
|
47
|
+
export function seedFor(protocolHash, statisticName) {
|
|
48
|
+
return crypto.createHash('sha256').update(`${protocolHash}|${statisticName}`).digest().readUInt32BE(0);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const quantile = (sorted, q) => {
|
|
52
|
+
if (!sorted.length) return null;
|
|
53
|
+
const pos = (sorted.length - 1) * q;
|
|
54
|
+
const lo = Math.floor(pos); const hi = Math.ceil(pos);
|
|
55
|
+
return sorted[lo] + (sorted[hi] - sorted[lo]) * (pos - lo);
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
const ZERO = () => ({ tp: 0, fn: 0, fp: 0, tn: 0 });
|
|
59
|
+
const add = (a, b) => { a.tp += b.tp; a.fn += b.fn; a.fp += b.fp; a.tn += b.tn; };
|
|
60
|
+
|
|
61
|
+
export const f1Of = (c) => (c.tp + c.fp + c.fn === 0 ? null : (2 * c.tp) / (2 * c.tp + c.fp + c.fn));
|
|
62
|
+
export const precisionOf = (c) => (c.tp + c.fp > 0 ? c.tp / (c.tp + c.fp) : null);
|
|
63
|
+
export const recallOf = (c) => (c.tp + c.fn > 0 ? c.tp / (c.tp + c.fn) : null);
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Grouped percentile bootstrap.
|
|
67
|
+
* @param {object} o
|
|
68
|
+
* @param {{id: string, language: string, tp: number, fn: number, fp: number, tn: number}[]} o.units per-target tallies
|
|
69
|
+
* @param {Object<string,string>} o.groupOf target id -> group id (a target with no entry is its own group)
|
|
70
|
+
* @param {(agg: {total: object, byLanguage: Object<string,object>}) => (number|null)} o.statistic
|
|
71
|
+
* @param {string} o.name the statistic's name (feeds the seed)
|
|
72
|
+
* @param {string} o.protocolHash
|
|
73
|
+
*/
|
|
74
|
+
export function groupedBootstrap({ units, groupOf = {}, statistic, name, protocolHash, replicates = INTERVAL_METHODS['grouped-bootstrap-95'].replicates, minGroups = INTERVAL_METHODS['grouped-bootstrap-95'].minGroups, level = 0.95 }) {
|
|
75
|
+
const method = 'grouped-bootstrap-95';
|
|
76
|
+
const groups = new Map();
|
|
77
|
+
for (const u of units || []) {
|
|
78
|
+
const g = groupOf[u.id] || `solo:${u.id}`;
|
|
79
|
+
if (!groups.has(g)) groups.set(g, []);
|
|
80
|
+
groups.get(g).push(u);
|
|
81
|
+
}
|
|
82
|
+
const G = [...groups.keys()].sort();
|
|
83
|
+
const aggOf = (members) => {
|
|
84
|
+
const total = ZERO(); const byLanguage = {};
|
|
85
|
+
for (const u of members) {
|
|
86
|
+
add(total, u);
|
|
87
|
+
byLanguage[u.language] = byLanguage[u.language] || ZERO();
|
|
88
|
+
add(byLanguage[u.language], u);
|
|
89
|
+
}
|
|
90
|
+
return { total, byLanguage };
|
|
91
|
+
};
|
|
92
|
+
const all = aggOf(units || []);
|
|
93
|
+
const point = statistic(all);
|
|
94
|
+
const base = { method, level, replicates, groups: G.length, units: (units || []).length, point };
|
|
95
|
+
if (G.length < minGroups) {
|
|
96
|
+
return { ...base, status: 'unmeasured', low: null, high: null, reason: `${G.length} independent group${G.length === 1 ? '' : 's'}, fewer than the ${minGroups} needed for a resampling interval to mean anything` };
|
|
97
|
+
}
|
|
98
|
+
const rng = seededRng(seedFor(protocolHash, name));
|
|
99
|
+
const perGroup = G.map((g) => aggOf(groups.get(g)));
|
|
100
|
+
const draws = [];
|
|
101
|
+
let undefinedDraws = 0;
|
|
102
|
+
for (let r = 0; r < replicates; r++) {
|
|
103
|
+
const total = ZERO(); const byLanguage = {};
|
|
104
|
+
for (let i = 0; i < G.length; i++) {
|
|
105
|
+
const pick = perGroup[Math.floor(rng() * G.length)];
|
|
106
|
+
add(total, pick.total);
|
|
107
|
+
for (const [lang, c] of Object.entries(pick.byLanguage)) { byLanguage[lang] = byLanguage[lang] || ZERO(); add(byLanguage[lang], c); }
|
|
108
|
+
}
|
|
109
|
+
const v = statistic({ total, byLanguage });
|
|
110
|
+
if (v === null || v === undefined || Number.isNaN(v)) undefinedDraws++; else draws.push(v);
|
|
111
|
+
}
|
|
112
|
+
const maxUndef = INTERVAL_METHODS[method].maxUndefinedFraction;
|
|
113
|
+
if (undefinedDraws / replicates > maxUndef) {
|
|
114
|
+
return { ...base, status: 'unmeasured', low: null, high: null, undefinedDraws, reason: `the statistic was undefined in ${undefinedDraws} of ${replicates} resamples (more than ${maxUndef * 100}%), so the interval would describe a different population` };
|
|
115
|
+
}
|
|
116
|
+
draws.sort((a, b) => a - b);
|
|
117
|
+
const alpha = (1 - level) / 2;
|
|
118
|
+
return { ...base, status: 'measured', low: quantile(draws, alpha), high: quantile(draws, 1 - alpha), undefinedDraws, seed: seedFor(protocolHash, name) };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Wilson score interval for a proportion. n = 0 is unmeasured. */
|
|
122
|
+
export function wilsonInterval(successes, n, z = INTERVAL_METHODS['wilson-95'].z) {
|
|
123
|
+
const method = 'wilson-95';
|
|
124
|
+
if (!Number.isInteger(successes) || !Number.isInteger(n) || n <= 0 || successes < 0 || successes > n) {
|
|
125
|
+
return { method, level: 0.95, status: 'unmeasured', point: null, low: null, high: null, successes, n, reason: 'no cases' };
|
|
126
|
+
}
|
|
127
|
+
const p = successes / n; const z2 = z * z;
|
|
128
|
+
const denom = 1 + z2 / n;
|
|
129
|
+
const centre = (p + z2 / (2 * n)) / denom;
|
|
130
|
+
const half = (z * Math.sqrt((p * (1 - p)) / n + z2 / (4 * n * n))) / denom;
|
|
131
|
+
return { method, level: 0.95, status: 'measured', point: p, low: Math.max(0, centre - half), high: Math.min(1, centre + half), successes, n };
|
|
132
|
+
}
|