@clear-capabilities/agentic-security-scanner 0.162.0 → 0.163.1

package/src/posture/evaluation/dev-cases.js

@@ -0,0 +1,65 @@
+// A DEVELOPMENT case suite for engine work (QA-005.AC03, QA-006), scored by the same frozen policy as everything else.
+//
+// `test/fixtures/engine-mechanisms/` holds small projects, each a vulnerable `pre/` tree and a patched `post/` tree, plus a `cases.json`
+// naming the reviewed location of each defect. This module turns them into a protocol, labels and a resolver so that
+// `runEvaluation` + `scoreRun` (same matching policy, same end-to-end rules) can answer "which of these defects does the engine
+// recover, and which patched trees stay silent" for ANY engine checkout. That makes a before/after comparison a measurement and not
+// an anecdote: run it once per engine revision and compare.
+//
+// What this is NOT: an independent population. The projects and the labels were written by the people who wrote the fix, so every
+// label is flagged `synthetic: true` and gates.js will never count them toward a real-code minimum. They exist to show that a
+// change recovers the mechanism it was made for, and that its patched twin stays quiet. They say nothing about real-code accuracy.
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { digestOf } from '../assurance/identity.js';
+import { digestTree, targetDigestOf } from './runner.js';
+import { freezeProtocol, CORE_LANGUAGES, PREREGISTERED_THRESHOLDS, DEFAULT_MATCHING } from './protocol.js';
+import { buildDefectLabel, buildNegativeLabel } from './labels.js';
+
+const LICENSE = 'development-fixture';
+const hex = (n) => n.toString(16).padStart(40, '0');
+const adjudication = (tag) => ({ status: 'adjudicated', independent: true, reviewers: [{ reviewerId: `dev-reviewer-a-${tag}`, verdict: 'confirmed' }, { reviewerId: `dev-reviewer-b-${tag}`, verdict: 'confirmed' }] });
+
+function readDevCases(fixturesDir) {
+  return JSON.parse(fs.readFileSync(path.join(fixturesDir, 'cases.json'), 'utf8')).cases;
+}
+
+/** @returns {{protocol, defects, negatives, freezeErrors}} everything flagged synthetic; every target in the development split. */
+export function buildDevSuite({ fixturesDir, cases = readDevCases(fixturesDir) }) {
+  const targets = cases.map((c, i) => ({
+    id: c.id, language: c.language, upstream: `example.invalid/dev/${c.id}`, pairId: `pair-${c.id}`, preCommit: hex(2 * i + 1), postCommit: hex(2 * i + 2), advisoryIds: [], license: LICENSE,
+    digest: targetDigestOf({ pre: digestTree(path.join(fixturesDir, c.id, 'pre')), post: digestTree(path.join(fixturesDir, c.id, 'post')) }),
+  }));
+  const draft = {
+    synthetic: true,
+    engine: { version: '0.0.0-dev-cases', bundleDigest: digestOf('dev-cases-engine') },
+    measurement: { commit: hex(0xabc), cleanTree: true },
+    tools: { node: 'dev' }, models: [], datasetLicenses: { [LICENSE]: 'authored for engine development; not real-world code' },
+    scope: { languages: [...CORE_LANGUAGES], families: [...new Set(cases.map((c) => c.family))].sort() },
+    matching: { ...DEFAULT_MATCHING },
+    limits: { perTargetTimeoutMs: 120000, spendCeilingUsd: 0, replicates: 3 },
+    thresholds: { ...PREREGISTERED_THRESHOLDS },
+    targets, splits: { dev: targets.map((t) => t.id).sort(), sealed: [] },
+    grouping: { method: 'union-find', keys: ['pair', 'upstream', 'advisory', 'commit', 'template'], salt: 'dev-cases' },
+  };
+  const frozen = freezeProtocol(draft);
+  const defects = []; const negatives = [];
+  cases.forEach((c, i) => {
+    const base = (tag) => ({ synthetic: true, adjudication: adjudication(`${tag}${i}`), proposedBy: { role: 'worker', id: 'dev-proposer' } });
+    const d = buildDefectLabel({
+      ...base('d'), targetId: c.id, rootCauseId: `development-root-cause-${c.id}`, affected: { commit: hex(2 * i + 1) },
+      language: c.language, family: c.family, cwe: c.cwe, location: { file: c.file, startLine: c.startLine, endLine: c.endLine },
+      evidence: [{ ref: `development-review-note-${c.id}`, producer: 'human' }],
+    });
+    defects.push(d);
+    negatives.push(buildNegativeLabel({
+      ...base('n'), targetId: c.id, variant: 'post', kind: 'patched', language: c.language, family: c.family,
+      scope: { files: [c.file] }, pairedDefectId: d.id, rationale: `development: the patched ${c.id} was reviewed as not exploitable`,
+    }));
+  });
+  return { protocol: frozen.ok ? frozen.protocol : null, freezeErrors: frozen.errors, defects, negatives, cases };
+}
+
+/** resolveTarget for the runner over the dev fixtures. */
+export const devResolver = (fixturesDir) => (target, variant) => ({ dir: path.join(fixturesDir, target.id, variant) });

package/src/posture/evaluation/economics.js

@@ -0,0 +1,85 @@
+// Security economics (QA-004.AC03): what finding and fixing defects cost, with the cost kinds kept apart.
+//
+// A ledger is a list of entries, each one thing that was spent:
+//   { id, kind, amountUsd | minutes, targetId, attemptId?, rootCauseId?, outcome? }
+// `kind` is one of
+//   model           model inference charged to the evaluation
+//   tool            tool and infrastructure compute (scanner time, sandbox, runner)
+//   cache           cache charges (writes and reads priced separately from fresh inference)
+//   human-review    reviewer MINUTES, never converted to dollars here (a rate is a policy choice, not a measurement)
+// `outcome` classifies the work the spend bought: confirmed | validated-fix | refuted | inconclusive | failed | duplicate.
+//
+// The rules that keep these figures honest:
+//   - the kinds are reported side by side and never silently summed; human minutes are not dollars;
+//   - an entry with no amount is UNMEASURED and counted as such, never read as zero;
+//   - cost per confirmed defect divides ALL machine spend (successful, failed, refuted and duplicate attempts alike) by the number
+//     of DISTINCT root causes independently confirmed, so failed attempts raise the price and a defect found twice is one defect;
+//   - cost per validated fix divides the same spend by distinct root causes with an independently validated repair;
+//   - with no confirmed defect the ratio is `null` (undefined), not 0 and not the total.
+//
+// Pure: no fs, no clock.
+
+import { SCHEMA_VERSION } from '../assurance/schema-kit.js';
+import { digestOf } from '../assurance/identity.js';
+
+const COST_KINDS = Object.freeze(['model', 'tool', 'cache', 'human-review']);
+export const OUTCOMES = Object.freeze(['confirmed', 'validated-fix', 'refuted', 'inconclusive', 'failed', 'duplicate']);
+const ECON_SCHEMA = 'agentic-security/evaluation-economics';
+
+function validateEntry(e, i) {
+  const errs = [];
+  if (!e || typeof e !== 'object') return [{ code: 'BAD_TYPE', path: `entries[${i}]`, message: 'must be an object' }];
+  if (!COST_KINDS.includes(e.kind)) errs.push({ code: 'BAD_ENUM', path: `entries[${i}].kind`, message: `kind must be one of ${COST_KINDS.join(', ')}` });
+  if (e.outcome !== undefined && !OUTCOMES.includes(e.outcome)) errs.push({ code: 'BAD_ENUM', path: `entries[${i}].outcome`, message: `outcome must be one of ${OUTCOMES.join(', ')}` });
+  const field = e.kind === 'human-review' ? 'minutes' : 'amountUsd';
+  const other = e.kind === 'human-review' ? 'amountUsd' : 'minutes';
+  if (e[other] !== undefined && e[other] !== null) errs.push({ code: 'RULE_VIOLATION', path: `entries[${i}].${other}`, message: `a ${e.kind} entry is measured in ${field}; ${other} would mix units` });
+  if (e[field] !== undefined && e[field] !== null && !(typeof e[field] === 'number' && Number.isFinite(e[field]) && e[field] >= 0)) errs.push({ code: 'BAD_VALUE', path: `entries[${i}].${field}`, message: `${field} must be a non-negative number or absent (unmeasured)` });
+  if ((e.outcome === 'confirmed' || e.outcome === 'validated-fix') && !e.rootCauseId) errs.push({ code: 'RULE_VIOLATION', path: `entries[${i}].rootCauseId`, message: 'a confirmed or validated outcome must name its root cause so duplicates cannot be counted twice' });
+  return errs;
+}
+
+/** @param {object[]} entries */
+export function accountCosts(entries) {
+  const errors = (entries || []).flatMap(validateEntry);
+  if (errors.length) return { ok: false, errors };
+
+  const totals = { model: { usd: 0, entries: 0, unmeasured: 0 }, tool: { usd: 0, entries: 0, unmeasured: 0 }, cache: { usd: 0, entries: 0, unmeasured: 0 }, humanReview: { minutes: 0, entries: 0, unmeasured: 0 } };
+  const slot = { model: 'model', tool: 'tool', cache: 'cache', 'human-review': 'humanReview' };
+  const confirmed = new Set(); const validated = new Set();
+  const byOutcome = {};
+  for (const e of entries || []) {
+    const t = totals[slot[e.kind]];
+    t.entries++;
+    const v = e.kind === 'human-review' ? e.minutes : e.amountUsd;
+    if (v === undefined || v === null) t.unmeasured++;
+    else if (e.kind === 'human-review') t.minutes += v; else t.usd += v;
+    const o = e.outcome || 'unclassified';
+    byOutcome[o] = byOutcome[o] || { usd: 0, minutes: 0, entries: 0 };
+    byOutcome[o].entries++;
+    if (v !== undefined && v !== null) { if (e.kind === 'human-review') byOutcome[o].minutes += v; else byOutcome[o].usd += v; }
+    if (e.outcome === 'confirmed') confirmed.add(e.rootCauseId);
+    if (e.outcome === 'validated-fix') { validated.add(e.rootCauseId); confirmed.add(e.rootCauseId); }
+  }
+  // Machine spend: every kind except reviewer minutes, INCLUDING failed, refuted, inconclusive and duplicate work.
+  const machineUsd = totals.model.usd + totals.tool.usd + totals.cache.usd;
+  const unmeasuredMachine = totals.model.unmeasured + totals.tool.unmeasured + totals.cache.unmeasured;
+  const div = (n, d) => (d > 0 ? n / d : null);
+  const out = {
+    schema: ECON_SCHEMA, schemaVersion: SCHEMA_VERSION, ok: true, errors: [],
+    totals,
+    separation: 'model, tool, cache and human-review figures are never combined; human review is in minutes, the rest in USD',
+    machineSpendUsd: machineUsd,
+    unmeasuredMachineEntries: unmeasuredMachine,
+    confirmedDefects: confirmed.size, validatedFixes: validated.size,
+    costPerConfirmedDefectUsd: div(machineUsd, confirmed.size),
+    costPerValidatedFixUsd: div(machineUsd, validated.size),
+    humanMinutesPerConfirmedDefect: div(totals.humanReview.minutes, confirmed.size),
+    humanMinutesPerValidatedFix: div(totals.humanReview.minutes, validated.size),
+    byOutcome,
+    note: 'ratios divide ALL machine spend (failed, refuted and duplicate attempts included) by DISTINCT independently confirmed root causes; a ratio over an incomplete ledger is a lower bound',
+    lowerBound: unmeasuredMachine > 0,
+  };
+  out.economicsHash = digestOf({ ...out, economicsHash: undefined });
+  return out;
+}

package/src/posture/evaluation/gates.js

@@ -0,0 +1,161 @@
+// Real-code quality gates over a frozen protocol (section 5 of the programme PRD).
+//
+// This module can answer `pass` ONLY when every population minimum is met by
+// real, independently adjudicated, non-synthetic labels inside the sealed split,
+// AND the measured value clears the threshold registered in the protocol. In
+// every other case it answers `insufficient-population` or `unmeasured`, with the
+// reason. It never answers `pass` for a smaller convenient sample, for synthetic
+// data, or for a threshold it was handed by the caller: thresholds are read from
+// the protocol, and a protocol that fails its hash check is rejected outright.
+//
+// The statuses are deliberately not boolean: `unmeasured` is not `fail`, and
+// neither is a pass.
+
+import { validateProtocol, CORE_LANGUAGES } from './protocol.js';
+import { isScoreable } from './labels.js';
+import { groupTargets } from './grouping.js';
+import { groupedBootstrap, f1Of } from './interval.js';
+
+export const GATE_STATUSES = Object.freeze(['pass', 'fail', 'insufficient-population', 'unmeasured']);
+
+const gate = (id, status, extra = {}) => ({ id, status, ...extra });
+
+/**
+ * @param {object} o
+ * @param {object} o.protocol  frozen protocol (thresholds and splits come from here)
+ * @param {object} o.score     a score (score.js) of the SEALED split of a run bound to this protocol
+ * @param {object[]} o.defects all defect labels
+ * @param {object[]} o.negatives all negative labels
+ * @param {object} [o.run]      the sealed-split run record the score came from; enables the integrity gates (leaks, denominator)
+ * @param {object} [o.baseline] the last verified release score; enables the no-regression gate
+ */
+export function evaluateGates({ protocol, score, defects = [], negatives = [], run = null, baseline = null }) {
+  const v = validateProtocol(protocol);
+  if (!v.ok) return { ok: false, rejected: true, errors: v.errors, overall: 'rejected', gates: [] };
+  if (score && score.protocolHash !== protocol.protocolHash) {
+    return { ok: false, rejected: true, errors: [{ code: 'PROTOCOL_MISMATCH', path: 'score.protocolHash', message: 'the score was produced under a different protocol; it cannot be judged against these thresholds' }], overall: 'rejected', gates: [] };
+  }
+  const t = protocol.thresholds;
+  const gates = [];
+  const sealed = new Set(protocol.splits.sealed);
+
+  const realD = defects.filter((l) => isScoreable(l) && !l.synthetic && sealed.has(l.targetId));
+  const realN = negatives.filter((l) => isScoreable(l) && !l.synthetic && sealed.has(l.targetId));
+  const count = (xs, key, val) => xs.filter((x) => x[key] === val).length;
+  const shortfalls = [];
+  for (const lang of CORE_LANGUAGES) {
+    const p = count(realD, 'language', lang); const n = count(realN, 'language', lang);
+    if (p < t.minPositivesPerCoreLanguage) shortfalls.push(`${lang}: ${p}/${t.minPositivesPerCoreLanguage} positives`);
+    if (n < t.minNegativesPerCoreLanguage) shortfalls.push(`${lang}: ${n}/${t.minNegativesPerCoreLanguage} negatives`);
+  }
+  const familyShort = [...new Set(protocol.scope.families)].filter((f) => count(realD, 'family', f) < t.minPositivesPerFamily);
+  const populationOk = shortfalls.length === 0 && familyShort.length === 0;
+  const popDetail = {
+    reason: protocol.synthetic
+      ? 'the protocol is synthetic: a synthetic population can exercise the machinery and can never satisfy a real-code gate'
+      : `population minimums not met: ${shortfalls.slice(0, 6).join('; ')}${shortfalls.length > 6 ? `; and ${shortfalls.length - 6} more` : ''}${familyShort.length ? `; ${familyShort.length} famil${familyShort.length === 1 ? 'y' : 'ies'} below ${t.minPositivesPerFamily} sealed positives` : ''}`,
+    realSealedPositives: realD.length, realSealedNegatives: realN.length,
+  };
+  const blocked = protocol.synthetic || !populationOk || !score;
+  const unmet = (id, threshold) => gate(id, score ? 'insufficient-population' : 'unmeasured', {
+    threshold, measured: null, ...(score ? popDetail : { reason: 'no sealed-split score exists' }),
+  });
+
+  const e2e = score?.endToEnd;
+  const decide = (id, threshold, measured, cmp = (m, th) => m >= th) => {
+    if (blocked) return unmet(id, threshold);
+    if (measured === null || measured === undefined) return gate(id, 'unmeasured', { threshold, measured: null, reason: 'value undefined on this population' });
+    return gate(id, cmp(measured, threshold) ? 'pass' : 'fail', { threshold, measured });
+  };
+
+  gates.push(decide('overall-micro-f1', t.overallMicroF1, e2e?.micro?.f1));
+  gates.push(decide('overall-macro-f1', t.overallMacroF1, e2e?.macroF1));
+  gates.push(decide('pooled-precision', t.pooledPrecision, e2e?.micro?.precision));
+  gates.push(decide('pooled-recall', t.pooledRecall, e2e?.micro?.recall));
+  gates.push(decide('completion', t.completion, score?.completion?.rate));
+  for (const lang of CORE_LANGUAGES) gates.push(decide(`per-language-f1:${lang}`, t.perLanguageF1, e2e?.byLanguage?.[lang]?.f1 ?? null));
+  gates.push(lowerBoundGate({ protocol, score, blocked, popDetail }));
+  for (const g of integrityGates({ protocol, score, run, defects, negatives })) gates.push(g);
+  if (baseline) gates.push(regressionGate({ score, baseline, blocked, popDetail }));
+
+  const overall = gates.every((g) => g.status === 'pass') ? 'pass'
+    : gates.some((g) => g.status === 'fail') ? 'fail'
+      : gates.some((g) => g.status === 'insufficient-population') ? 'insufficient-population' : 'unmeasured';
+  return { ok: true, rejected: false, errors: [], protocolHash: protocol.protocolHash, synthetic: protocol.synthetic, overall, gates, population: popDetail };
+}
+
+// ---------------------------------------------------------------- the interval gate
+
+/** The lowest per-language F1 lower bound (grouped bootstrap) against the registered bound. Unmeasured, never assumed. */
+function lowerBoundGate({ protocol, score, blocked, popDetail }) {
+  const id = 'per-language-f1-lower-bound';
+  const threshold = protocol.thresholds.perLanguageF1LowerBound;
+  if (!score) return gate(id, 'unmeasured', { threshold, measured: null, reason: 'no sealed-split score exists' });
+  if (blocked) return gate(id, 'insufficient-population', { threshold, measured: null, reason: popDetail.reason });
+  if (!score.perTarget || typeof score.perTarget !== 'object') return gate(id, 'unmeasured', { threshold, measured: null, reason: 'the score carries no per-target tallies, so no grouped interval can be computed' });
+  const { groupOf } = groupTargets(protocol.targets);
+  const units = Object.entries(score.perTarget).map(([tid, c]) => ({ id: tid, ...c }));
+  const perLanguage = {};
+  for (const lang of CORE_LANGUAGES) {
+    perLanguage[lang] = groupedBootstrap({
+      units: units.filter((u) => u.language === lang), groupOf, statistic: (a) => f1Of(a.total), name: `lang:${lang}:f1`, protocolHash: protocol.protocolHash,
+    });
+  }
+  const measured = Object.values(perLanguage).filter((iv) => iv.status === 'measured');
+  const unmeasured = Object.entries(perLanguage).filter(([, iv]) => iv.status !== 'measured');
+  const lows = measured.map((iv) => iv.low);
+  const min = lows.length ? Math.min(...lows) : null;
+  const detail = Object.fromEntries(Object.entries(perLanguage).map(([l, iv]) => [l, { status: iv.status, low: iv.low, high: iv.high, groups: iv.groups }]));
+  if (min !== null && min < threshold) return gate(id, 'fail', { threshold, measured: min, method: protocol.thresholds.intervalMethod, perLanguage: detail, reason: 'at least one language has an F1 lower bound under the registered bound' });
+  if (unmeasured.length) return gate(id, 'unmeasured', { threshold, measured: min, method: protocol.thresholds.intervalMethod, perLanguage: detail, reason: `${unmeasured.length} language${unmeasured.length === 1 ? '' : 's'} without enough independent groups for an interval: ${unmeasured.slice(0, 4).map(([l]) => l).join(', ')}` });
+  return gate(id, 'pass', { threshold, measured: min, method: protocol.thresholds.intervalMethod, perLanguage: detail });
+}
+
+// ---------------------------------------------------------------- integrity gates
+
+const LEAK_REASON = /leakage control/i;
+
+/** Gates that judge the evaluation's own integrity. They are measurable on any population, so they never read insufficient-population. */
+function integrityGates({ protocol, score, run, defects, negatives }) {
+  // Without a run record there is nothing to check; release-gate.js refuses to produce a release verdict without one.
+  if (!run) return [];
+  const leaks = (run.outcomes || []).filter((o) => o.status === 'quarantined' && LEAK_REASON.test(o.failureReason || ''));
+  const out = [gate('answer-key-leaks', leaks.length === 0 ? 'pass' : 'fail', { threshold: 0, measured: leaks.length, ...(leaks.length ? { reason: `${leaks.length} target workspace(s) carried label or answer-key material`, targets: leaks.map((o) => o.targetId).sort() } : {}) })];
+
+  // The denominator is the frozen population, not whatever the run happened to cover.
+  const problems = [];
+  if (run.protocolHash !== protocol.protocolHash) problems.push('run is bound to a different protocol');
+  const have = new Set((run.outcomes || []).map((o) => `${o.targetId}|${o.variant}`));
+  for (const t of protocol.targets.filter((x) => protocol.splits.sealed.includes(x.id))) {
+    if (!have.has(`${t.id}|pre`)) problems.push(`sealed target ${t.id} has no pre outcome`);
+    if (t.postCommit && !have.has(`${t.id}|post`)) problems.push(`sealed target ${t.id} has no post outcome`);
+  }
+  const sealed = new Set(protocol.splits.sealed);
+  const positives = defects.filter((l) => isScoreable(l) && sealed.has(l.targetId)).length;
+  const negs = negatives.filter((l) => isScoreable(l) && sealed.has(l.targetId)).length;
+  if (score?.endToEnd?.micro && Number.isFinite(score.endToEnd.micro.tp)) {
+    const m = score.endToEnd.micro;
+    if (m.tp + m.fn !== positives) problems.push(`scored positives ${m.tp + m.fn} differ from the ${positives} adjudicated sealed positives`);
+    const negScored = m.tn + m.fp;
+    const failed = (score.negativeFailures || []).length;
+    if (Array.isArray(score.negativeFailures) && negScored + failed < negs && negScored + failed !== negs) problems.push(`scored negatives ${negScored + failed} differ from the ${negs} adjudicated sealed negatives`);
+  }
+  out.push(gate('denominator-intact', problems.length === 0 ? 'pass' : 'fail', { threshold: null, measured: problems.length, ...(problems.length ? { reason: problems.slice(0, 5).join('; '), problems: problems.slice(0, 20) } : {}) }));
+  return out;
+}
+
+/** Compare to the last VERIFIED release score; a drop in micro F1 or in any language's F1 fails. */
+function regressionGate({ score, baseline, blocked, popDetail }) {
+  const id = 'no-regression-vs-baseline';
+  const cur = score?.endToEnd; const base = baseline?.endToEnd;
+  if (!cur || !base) return gate(id, 'unmeasured', { threshold: 'baseline', measured: null, reason: 'a score or a baseline is missing' });
+  const regressions = [];
+  const cmp = (name, c, b) => { if (typeof c === 'number' && typeof b === 'number' && c < b) regressions.push({ name, baseline: b, current: c }); };
+  cmp('micro-f1', cur.micro?.f1, base.micro?.f1);
+  cmp('micro-precision', cur.micro?.precision, base.micro?.precision);
+  cmp('micro-recall', cur.micro?.recall, base.micro?.recall);
+  for (const [lang, b] of Object.entries(base.byLanguage || {})) cmp(`f1:${lang}`, cur.byLanguage?.[lang]?.f1, b?.f1);
+  if (regressions.length) return gate(id, 'fail', { threshold: 'baseline', measured: regressions.length, regressions, reason: 'the current engine scores below the recorded baseline' });
+  if (blocked) return gate(id, 'insufficient-population', { threshold: 'baseline', measured: null, reason: popDetail.reason });
+  return gate(id, 'pass', { threshold: 'baseline', measured: 0 });
+}

package/src/posture/evaluation/grouping.js

@@ -0,0 +1,101 @@
+// Leakage-safe grouping and splitting (QA-001.AC02).
+//
+// Related samples must land on the same side of the development/sealed line, or
+// the sealed set is not independent of what detector work already saw. Two
+// samples are related when they share any of: a vulnerable/fixed pair, an
+// upstream project (a fork resolves to its parent), an advisory id (duplicate
+// advisories), a commit, or a near-identical template fingerprint. Relations are
+// transitive (union-find), so a chain A~B~C is one group.
+//
+// Pure and deterministic: group ids and split assignment depend only on the
+// targets and an explicit salt, never on iteration order or a clock.
+
+import * as crypto from 'node:crypto';
+
+/** `https://GitHub.com/Org/Repo.git/` and `git@github.com:org/repo` normalise to `github.com/org/repo`. */
+export function normalizeUpstream(url) {
+  if (typeof url !== 'string' || !url.trim()) return null;
+  let u = url.trim().toLowerCase();
+  u = u.replace(/^[a-z+]+:\/\//, '').replace(/^git@/, '').replace(/^[^@/]+@/, '');
+  u = u.replace(':', '/').replace(/\/+$/, '').replace(/\.git$/, '');
+  return u || null;
+}
+
+class UnionFind {
+  constructor(ids) { this.parent = new Map(ids.map((i) => [i, i])); }
+  find(x) { let r = x; while (this.parent.get(r) !== r) r = this.parent.get(r); this.parent.set(x, r); return r; }
+  union(a, b) { const ra = this.find(a); const rb = this.find(b); if (ra !== rb) this.parent.set(ra < rb ? rb : ra, ra < rb ? ra : rb); }
+}
+
+const KEYS = (t) => {
+  const k = [];
+  if (t.pairId) k.push(['pair', t.pairId]);
+  const up = normalizeUpstream(t.upstream);
+  if (up) k.push(['upstream', up]);
+  const fork = normalizeUpstream(t.forkOf);
+  if (fork) k.push(['upstream', fork]);
+  for (const a of t.advisoryIds || []) k.push(['advisory', String(a).toUpperCase()]);
+  for (const c of [t.preCommit, t.postCommit]) if (c) k.push(['commit', c]);
+  if (t.templateFingerprint) k.push(['template', t.templateFingerprint]);
+  return k;
+};
+
+/**
+ * Group targets by every relation above. Returns `{ groups, groupOf }` where each
+ * group is `{ groupId, members[], reasons[] }`, members sorted, groups sorted.
+ */
+export function groupTargets(targets) {
+  const list = (targets || []).filter((t) => t && t.id);
+  const uf = new UnionFind(list.map((t) => t.id));
+  const byKey = new Map();
+  for (const t of list) {
+    for (const [kind, val] of KEYS(t)) {
+      const key = `${kind}:${val}`;
+      if (byKey.has(key)) uf.union(byKey.get(key).id, t.id);
+      else byKey.set(key, t);
+    }
+  }
+  const members = new Map();
+  for (const t of list) { const r = uf.find(t.id); if (!members.has(r)) members.set(r, []); members.get(r).push(t.id); }
+  // Which relation kinds actually linked two or more members of each group.
+  const kindCount = new Map();
+  for (const t of list) {
+    for (const [kind, val] of KEYS(t)) {
+      const gk = `${uf.find(t.id)}|${kind}:${val}`;
+      kindCount.set(gk, (kindCount.get(gk) || 0) + 1);
+    }
+  }
+  const groups = [];
+  const groupOf = {};
+  for (const [root, ids] of members) {
+    ids.sort();
+    const reasons = new Set();
+    for (const [gk, n] of kindCount) if (n > 1 && gk.startsWith(`${root}|`)) reasons.add(gk.slice(root.length + 1).split(':')[0]);
+    const groupId = `grp:${crypto.createHash('sha256').update(ids.join('\n')).digest('hex').slice(0, 16)}`;
+    groups.push({ groupId, members: ids, reasons: [...reasons].sort() });
+    for (const id of ids) groupOf[id] = groupId;
+  }
+  groups.sort((a, b) => a.groupId.localeCompare(b.groupId));
+  return { groups, groupOf };
+}
+
+/**
+ * Deterministically assign whole groups to development or sealed. A group's side
+ * is a function of its id and the salt only, so adding an unrelated target never
+ * moves an existing group, and no group can straddle the line.
+ */
+export function assignSplits(groups, { sealedFraction = 0.3, salt = '' } = {}) {
+  const dev = []; const sealed = [];
+  for (const g of groups) {
+    const h = crypto.createHash('sha256').update(`${salt}|${g.groupId}`).digest();
+    const u = h.readUInt32BE(0) / 0x100000000;
+    (u < sealedFraction ? sealed : dev).push(...g.members);
+  }
+  return { dev: dev.sort(), sealed: sealed.sort() };
+}
+
+/** Groups whose members are spread over both sides. Empty means the split is leakage-safe. */
+export function splitStraddles(groups, splits) {
+  const dev = new Set(splits?.dev || []); const sealed = new Set(splits?.sealed || []);
+  return (groups || []).filter((g) => g.members.some((m) => dev.has(m)) && g.members.some((m) => sealed.has(m)));
+}

package/src/posture/evaluation/interval.js

@@ -0,0 +1,132 @@
+// Documented 95% interval methods (QA-004.AC01).
+//
+// Two methods, each registered by name in the protocol (`thresholds.intervalMethod`) before any result is observed:
+//
+//   grouped-bootstrap-95   For F1, precision and recall, and for the micro and macro aggregates. Resamples GROUPS, not cases:
+//                          targets that share a pair, upstream, advisory, commit or template (grouping.js) are one resampling
+//                          unit, because treating near-duplicates as independent draws makes the interval narrower than the
+//                          evidence supports. The percentile method over `replicates` resamples (default 2000), driven by a
+//                          seeded generator, so the same score and protocol produce the same interval to the last digit.
+//                          The seed is derived from the protocol hash: nobody picks a seed after seeing a result.
+//   wilson-95              For a single proportion (the completion rate), where there is no group structure to resample.
+//
+// When there are too few groups for resampling to mean anything (`minGroups`, default 10), the interval is `unmeasured` with the
+// reason, never a degenerate zero-width or a made-up range. An undefined statistic in a resample (no positives drawn) is skipped
+// and counted, and a method that skips too many resamples is itself reported `unmeasured`.
+//
+// Pure: no fs, no clock, no Math.random.
+
+import * as crypto from 'node:crypto';
+
+export const INTERVAL_METHODS = Object.freeze({
+  'grouped-bootstrap-95': Object.freeze({
+    level: 0.95,
+    unit: 'target group (grouping.js union-find over pair, upstream, advisory, commit, template)',
+    estimator: 'percentile',
+    replicates: 2000,
+    minGroups: 10,
+    maxUndefinedFraction: 0.1,
+    seed: 'first 32 bits of sha256(protocolHash + "|" + statisticName)',
+  }),
+  'wilson-95': Object.freeze({ level: 0.95, unit: 'case', estimator: 'wilson-score', z: 1.959964 }),
+});
+
+/** mulberry32: a small, well-distributed 32-bit generator. Deterministic for a given seed. */
+export function seededRng(seed) {
+  let a = seed >>> 0;
+  return () => {
+    a = (a + 0x6D2B79F5) >>> 0;
+    let t = a;
+    t = Math.imul(t ^ (t >>> 15), t | 1);
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+/** A 32-bit seed that depends only on the frozen protocol and the statistic's name. */
+export function seedFor(protocolHash, statisticName) {
+  return crypto.createHash('sha256').update(`${protocolHash}|${statisticName}`).digest().readUInt32BE(0);
+}
+
+const quantile = (sorted, q) => {
+  if (!sorted.length) return null;
+  const pos = (sorted.length - 1) * q;
+  const lo = Math.floor(pos); const hi = Math.ceil(pos);
+  return sorted[lo] + (sorted[hi] - sorted[lo]) * (pos - lo);
+};
+
+const ZERO = () => ({ tp: 0, fn: 0, fp: 0, tn: 0 });
+const add = (a, b) => { a.tp += b.tp; a.fn += b.fn; a.fp += b.fp; a.tn += b.tn; };
+
+export const f1Of = (c) => (c.tp + c.fp + c.fn === 0 ? null : (2 * c.tp) / (2 * c.tp + c.fp + c.fn));
+export const precisionOf = (c) => (c.tp + c.fp > 0 ? c.tp / (c.tp + c.fp) : null);
+export const recallOf = (c) => (c.tp + c.fn > 0 ? c.tp / (c.tp + c.fn) : null);
+
+/**
+ * Grouped percentile bootstrap.
+ * @param {object} o
+ * @param {{id: string, language: string, tp: number, fn: number, fp: number, tn: number}[]} o.units  per-target tallies
+ * @param {Object<string,string>} o.groupOf  target id -> group id (a target with no entry is its own group)
+ * @param {(agg: {total: object, byLanguage: Object<string,object>}) => (number|null)} o.statistic
+ * @param {string} o.name        the statistic's name (feeds the seed)
+ * @param {string} o.protocolHash
+ */
+export function groupedBootstrap({ units, groupOf = {}, statistic, name, protocolHash, replicates = INTERVAL_METHODS['grouped-bootstrap-95'].replicates, minGroups = INTERVAL_METHODS['grouped-bootstrap-95'].minGroups, level = 0.95 }) {
+  const method = 'grouped-bootstrap-95';
+  const groups = new Map();
+  for (const u of units || []) {
+    const g = groupOf[u.id] || `solo:${u.id}`;
+    if (!groups.has(g)) groups.set(g, []);
+    groups.get(g).push(u);
+  }
+  const G = [...groups.keys()].sort();
+  const aggOf = (members) => {
+    const total = ZERO(); const byLanguage = {};
+    for (const u of members) {
+      add(total, u);
+      byLanguage[u.language] = byLanguage[u.language] || ZERO();
+      add(byLanguage[u.language], u);
+    }
+    return { total, byLanguage };
+  };
+  const all = aggOf(units || []);
+  const point = statistic(all);
+  const base = { method, level, replicates, groups: G.length, units: (units || []).length, point };
+  if (G.length < minGroups) {
+    return { ...base, status: 'unmeasured', low: null, high: null, reason: `${G.length} independent group${G.length === 1 ? '' : 's'}, fewer than the ${minGroups} needed for a resampling interval to mean anything` };
+  }
+  const rng = seededRng(seedFor(protocolHash, name));
+  const perGroup = G.map((g) => aggOf(groups.get(g)));
+  const draws = [];
+  let undefinedDraws = 0;
+  for (let r = 0; r < replicates; r++) {
+    const total = ZERO(); const byLanguage = {};
+    for (let i = 0; i < G.length; i++) {
+      const pick = perGroup[Math.floor(rng() * G.length)];
+      add(total, pick.total);
+      for (const [lang, c] of Object.entries(pick.byLanguage)) { byLanguage[lang] = byLanguage[lang] || ZERO(); add(byLanguage[lang], c); }
+    }
+    const v = statistic({ total, byLanguage });
+    if (v === null || v === undefined || Number.isNaN(v)) undefinedDraws++; else draws.push(v);
+  }
+  const maxUndef = INTERVAL_METHODS[method].maxUndefinedFraction;
+  if (undefinedDraws / replicates > maxUndef) {
+    return { ...base, status: 'unmeasured', low: null, high: null, undefinedDraws, reason: `the statistic was undefined in ${undefinedDraws} of ${replicates} resamples (more than ${maxUndef * 100}%), so the interval would describe a different population` };
+  }
+  draws.sort((a, b) => a - b);
+  const alpha = (1 - level) / 2;
+  return { ...base, status: 'measured', low: quantile(draws, alpha), high: quantile(draws, 1 - alpha), undefinedDraws, seed: seedFor(protocolHash, name) };
+}
+
+/** Wilson score interval for a proportion. n = 0 is unmeasured. */
+export function wilsonInterval(successes, n, z = INTERVAL_METHODS['wilson-95'].z) {
+  const method = 'wilson-95';
+  if (!Number.isInteger(successes) || !Number.isInteger(n) || n <= 0 || successes < 0 || successes > n) {
+    return { method, level: 0.95, status: 'unmeasured', point: null, low: null, high: null, successes, n, reason: 'no cases' };
+  }
+  const p = successes / n; const z2 = z * z;
+  const denom = 1 + z2 / n;
+  const centre = (p + z2 / (2 * n)) / denom;
+  const half = (z * Math.sqrt((p * (1 - p)) / n + z2 / (4 * n * n))) / denom;
+  return { method, level: 0.95, status: 'measured', point: p, low: Math.max(0, centre - half), high: Math.min(1, centre + half), successes, n };
+}