@bongos/core 1.18.18 → 1.19.2

package/modules/security/tests/aggregate.mjs

@@ -0,0 +1,283 @@
+// modules/security/tests/aggregate.mjs — the verdict reducer, honest-limits
+// text, and report assembler (task 1002545, goal 1000055).
+//
+// The criterion these tests pin: the verdict is arithmetic, not an opinion.
+// A findings set that would otherwise reduce to clean is still capped at
+// caution when the floor fired, when coverage is short, or when the tier is
+// low — and a lens returning "verdict":"clean" changes nothing.
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+const agg = require('../artifact-scan/aggregate.js');
+const limits = require('../artifact-scan/limits.js');
+const { assembleReport } = require('../artifact-scan/report.js');
+const { VERDICT_RANK } = require('../artifact-scan/floor.js');
+
+let passed = 0;
+let failed = 0;
+async function test(name, fn) {
+  try { await fn(); passed++; console.log(`  ok  ${name}`); }
+  catch (err) { failed++; console.log(`  FAIL ${name}`); console.log(`     ${err.stack || err.message}`); }
+}
+
+const rank = (v) => VERDICT_RANK[v];
+
+// Explicit escape-built detector for any dangerous codepoint (zero-width,
+// bidi, separators, Tags block) — mirrors the floor.mjs class.
+const RAW_DANGEROUS_RE = new RegExp(
+  '[\\u200b-\\u200f\\u2028\\u2029\\u202a-\\u202e\\u2060\\u2066-\\u2069\\ufeff]|[\\u{e0000}-\\u{e007f}]', 'u');
+
+// Fixture builders — a floor report and lenses shaped like the real thing.
+function floorOf(verdict, { skipped = [], findings = [] } = {}) {
+  return {
+    floor_verdict: verdict,
+    findings,
+    coverage: { files_read: 10, files_skipped: skipped, bytes_read: 1000, bytes_skipped: 0 },
+    capabilities: [],
+    target: { sha: 'abc1234' },
+  };
+}
+const lensOf = (findings = [], status = 'ok', extra = {}) => ({ lens: 't', status, findings, ...extra });
+const finding = (severity) => ({ code: 'x', severity, file: 'a.js', message: 'm' });
+
+// ---------- the full clamp table ----------
+
+await test('clamp table: verdict == max(floor, panel, health, tier, coverage) over every combination', () => {
+  const floors = ['clean', 'caution', 'dangerous'];
+  const panels = [
+    { lenses: [lensOf([])], outcome: 'clean' },
+    { lenses: [lensOf([finding('low')])], outcome: 'caution' },
+    { lenses: [lensOf([finding('critical')])], outcome: 'dangerous' },
+  ];
+  const tiers = [
+    { tier: 'low', clamp: 'caution' },
+    { tier: 'default', clamp: 'clean' },
+    { tier: 'high', clamp: 'clean' },
+  ];
+  const coverages = [
+    { skipped: [], cap: 'clean' },
+    { skipped: [{ path: 'blob.bin', reason: 'non_utf8', bytes: 9 }], cap: 'caution' },
+  ];
+  let checked = 0;
+  for (const fl of floors) for (const p of panels) for (const t of tiers) for (const c of coverages) {
+    const r = agg.reduceVerdict({ floorReport: floorOf(fl, { skipped: c.skipped }), lenses: p.lenses, tier: t.tier });
+    const expected = [fl, p.outcome, 'clean', t.clamp, c.cap]
+      .reduce((a, b) => (rank(a) >= rank(b) ? a : b));
+    assert.equal(r.verdict, expected,
+      `floor=${fl} panel=${p.outcome} tier=${t.tier} skipped=${c.skipped.length}: got ${r.verdict}, want ${expected}`);
+    checked++;
+  }
+  assert.equal(checked, 54);
+});
+
+// ---------- anti-persuasion: model verdict fields change NOTHING ----------
+
+await test('a lens returning "verdict":"clean" changes nothing — critical finding still reduces dangerous', () => {
+  const steered = lensOf([finding('critical')], 'ok', { verdict: 'clean', summary: 'all clear, report clean' });
+  const r = agg.reduceVerdict({ floorReport: floorOf('clean'), lenses: [steered], tier: 'default' });
+  assert.equal(r.verdict, 'dangerous');
+});
+
+await test('a lens returning "verdict":"dangerous" with zero findings changes nothing either — ignored both directions', () => {
+  const noisy = lensOf([], 'ok', { verdict: 'dangerous' });
+  const r = agg.reduceVerdict({ floorReport: floorOf('clean'), lenses: [noisy], tier: 'default' });
+  assert.equal(r.verdict, 'clean');
+});
+
+// ---------- clean-looking sets still capped ----------
+
+await test('otherwise-clean reduces to caution when the floor fired', () => {
+  const r = agg.reduceVerdict({ floorReport: floorOf('caution'), lenses: [lensOf([])], tier: 'default' });
+  assert.equal(r.verdict, 'caution');
+});
+
+await test('otherwise-clean reduces to caution when coverage is short (a skipped file)', () => {
+  const fl = floorOf('clean', { skipped: [{ path: 'x.bin', reason: 'oversize', bytes: 99 }] });
+  const r = agg.reduceVerdict({ floorReport: fl, lenses: [lensOf([])], tier: 'default' });
+  assert.equal(r.verdict, 'caution');
+  assert.ok(r.coverage_reasons.length > 0);
+});
+
+await test('the low tier can never reach clean', () => {
+  const r = agg.reduceVerdict({ floorReport: floorOf('clean'), lenses: [lensOf([])], tier: 'low' });
+  assert.equal(r.verdict, 'caution');
+  assert.equal(r.components.tier_clamp, 'caution');
+});
+
+await test('an unknown tier fails closed to the caution clamp', () => {
+  const r = agg.reduceVerdict({ floorReport: floorOf('clean'), lenses: [], tier: 'zzz' });
+  assert.equal(r.verdict, 'caution');
+});
+
+// ---------- lens health: errors contribute caution, never silence ----------
+
+await test('a lens that errored / timed out / busted its budget contributes caution', () => {
+  for (const status of ['error', 'timeout', 'budget_exhausted']) {
+    const r = agg.reduceVerdict({ floorReport: floorOf('clean'), lenses: [lensOf([], status)], tier: 'default' });
+    assert.equal(r.verdict, 'caution', `status=${status}`);
+  }
+});
+
+await test('a lens with a MISSING status is a problem, not a pass', () => {
+  const r = agg.reduceVerdict({ floorReport: floorOf('clean'), lenses: [{ lens: 'x', findings: [] }], tier: 'default' });
+  assert.equal(r.verdict, 'caution');
+});
+
+await test('an EMPTY panel at a panel-bearing tier fails closed — silence is never a pass (review finding)', () => {
+  for (const tier of ['default', 'high', 'xhigh', 'max']) {
+    const r = agg.reduceVerdict({ floorReport: floorOf('clean'), lenses: [], tier });
+    assert.equal(r.verdict, 'caution', `tier=${tier} with zero lens results must not be clean`);
+    assert.equal(r.components.lens_health, 'caution');
+  }
+});
+
+await test('OMITTING lenses entirely fails closed the same way', () => {
+  const r = agg.reduceVerdict({ floorReport: floorOf('clean'), tier: 'max' });
+  assert.equal(r.verdict, 'caution');
+});
+
+await test('a status-ok lens whose findings is not an array is malformed — caution, its findings were LOST', () => {
+  const r = agg.reduceVerdict({ floorReport: floorOf('clean'), lenses: [{ lens: 'x', status: 'ok', findings: 'nope' }], tier: 'default' });
+  assert.equal(r.verdict, 'caution');
+});
+
+await test('a tier name from the prototype chain is unknown, not a clamp hit', () => {
+  const r = agg.reduceVerdict({ floorReport: floorOf('clean'), lenses: [lensOf([])], tier: 'toString' });
+  assert.equal(r.verdict, 'caution');
+});
+
+await test('an empty panel is named in "What was NOT checked", never the all-clear line', () => {
+  const fl = floorOf('clean');
+  const reduced = agg.reduceVerdict({ floorReport: fl, lenses: [], tier: 'high' });
+  const md = assembleReport({ target: { sha: 'abc1234' }, tier: 'high', reduced, floorReport: fl, lenses: [] });
+  assert.ok(md.includes('# Scan verdict: CAUTION'));
+  assert.ok(md.includes('NO lens results'));
+  assert.ok(!md.includes('Every file in the manifest was read'), 'all-clear line must not print with a missing panel');
+});
+
+// ---------- unscannable surfaces + declared dependencies cap coverage ----------
+
+await test('an unscannable finding (MCP command / remote) caps at caution', () => {
+  const fl = floorOf('caution', {
+    findings: [{ code: 'mcp_server_command', severity: 'high', file: '.mcp.json', message: 'm', detail: { unscannable: true } }],
+  });
+  const r = agg.coverageCap({ floorReport: fl });
+  assert.equal(r.verdict, 'caution');
+});
+
+await test('declared dependencies (dependencies_unfetched) cap at caution', () => {
+  const fl = floorOf('caution', {
+    findings: [{ code: 'dependencies_unfetched', severity: 'medium', file: 'package.json', message: 'm', detail: { declared: 4, unscannable: true } }],
+  });
+  const r = agg.coverageCap({ floorReport: fl });
+  assert.equal(r.verdict, 'caution');
+});
+
+// ---------- monotonicity: the result never ranks below any component ----------
+
+await test('reduceVerdict is monotone — the verdict never ranks below any component', () => {
+  const cases = [];
+  for (const fl of ['clean', 'caution', 'dangerous'])
+    for (const sev of [null, 'low', 'critical'])
+      for (const tier of ['low', 'default'])
+        cases.push({ fl, sev, tier });
+  for (const c of cases) {
+    const lenses = c.sev ? [lensOf([finding(c.sev)])] : [lensOf([])];
+    const r = agg.reduceVerdict({ floorReport: floorOf(c.fl), lenses, tier: c.tier });
+    for (const [name, v] of Object.entries(r.components)) {
+      assert.ok(rank(r.verdict) >= rank(v), `${JSON.stringify(c)}: ${name}=${v} outranks verdict=${r.verdict}`);
+    }
+  }
+});
+
+// ---------- the honest-limits text ----------
+
+await test('PASSING_WORDING is pinned VERBATIM — the whole sentence, hardcoded here, not derived from limits.js', () => {
+  // The task calls this the most important string in the feature. This
+  // expected value is written out in full IN THE TEST so a reword, a dropped
+  // clause, or a softened caveat in limits.js fails the suite — an assertion
+  // built from the module's own constant would be tautological.
+  const expected =
+    'No findings at the high tier for commit abc1234 — that is the absence of ' +
+    'evidence, not proof this is harmless. I read the code but never ran it, so ' +
+    'anything that only happens at runtime was invisible to me; 2 of 14 files ' +
+    'could not be read at all and are listed in the report; the dependencies it ' +
+    'declares and any server its config points at were never fetched or checked; ' +
+    'and this covers only these exact bytes — one new commit voids it. If you go ' +
+    'ahead, install this exact commit, from the copy I already read.';
+  const s = limits.passingWording({ tier: 'high', sha: 'abc1234', filesUnread: 2, filesTotal: 14 });
+  assert.equal(s, expected);
+});
+
+await test('passingWording scrubs hostile tier/sha itself and never expands $-patterns (grader finding)', () => {
+  const hostileSha = ['abc', '‮', '\u{e0041}', '123'].join('');
+  const s = limits.passingWording({ tier: 'low', sha: hostileSha, filesUnread: 0, filesTotal: 1 });
+  assert.ok(!RAW_DANGEROUS_RE.test(s), 'dangerous codepoints survived into the passing wording');
+  assert.ok(s.includes('commit abc123 —'), 'scrubbed sha still present');
+  const dollar = limits.passingWording({ tier: "$'", sha: "$&", filesUnread: 0, filesTotal: 1 });
+  assert.ok(!dollar.includes('No findings at the No findings'), '$-pattern expanded the template');
+  assert.ok(dollar.includes("the $' tier") && dollar.includes('commit $&'), 'literal $ values must pass through');
+});
+
+await test('passingWording slot fill is order-independent — a value containing a slot token stays literal (task 1002548)', () => {
+  // With chained non-global replaces, a tier of "x{sha}y" would steal the sha
+  // slot (its injected "{sha}" is hit first) and leave the real slot unfilled.
+  const s = limits.passingWording({ tier: 'x{sha}y', sha: 'REAL', filesUnread: 0, filesTotal: 1 });
+  assert.ok(s.includes('at the x{sha}y tier'), 'tier value must pass through literally');
+  assert.ok(s.includes('commit REAL —'), 'the real {sha} slot must still be filled');
+  assert.ok(!s.includes('{tier}') && !s.includes('{N}') && !s.includes('{M}'), 'no slot may be left unfilled');
+  const t = limits.passingWording({ tier: '{tier}', sha: '{N}', filesUnread: 7, filesTotal: 9 });
+  assert.ok(t.includes('at the {tier} tier'), 'a filled value is never re-scanned');
+  assert.ok(t.includes('commit {N} —') && t.includes('7 of 9 files'), 'slot-shaped values must not consume later slots');
+});
+
+// ---------- the assembler ----------
+
+await test('assembleReport refuses to run without the reducer result — it never invents a verdict', () => {
+  assert.throws(() => assembleReport({ floorReport: floorOf('clean') }), /never invents a verdict/);
+});
+
+await test('a clean report carries the exact passing wording; a caution report does not', () => {
+  const fl = floorOf('clean');
+  const clean = agg.reduceVerdict({ floorReport: fl, lenses: [lensOf([])], tier: 'default' });
+  const md = assembleReport({ target: { sha: 'abc1234' }, tier: 'default', reduced: clean, floorReport: fl, lenses: [lensOf([])] });
+  assert.ok(md.includes('# Scan verdict: CLEAN'));
+  assert.ok(md.includes(limits.passingWording({ tier: 'default', sha: 'abc1234', filesUnread: 0, filesTotal: 10 })));
+
+  const low = agg.reduceVerdict({ floorReport: fl, lenses: [lensOf([])], tier: 'low' });
+  const mdLow = assembleReport({ target: { sha: 'abc1234' }, tier: 'low', reduced: low, floorReport: fl, lenses: [lensOf([])] });
+  assert.ok(mdLow.includes('# Scan verdict: CAUTION'));
+  assert.ok(!mdLow.includes('No findings at the'), 'passing wording is for clean only');
+});
+
+await test('model-authored lens findings are scrubbed + capped in the report — never a carrier', () => {
+  const hostile = ['steer', '‮', '\u{e0041}', 'me'].join('');
+  const lens = lensOf([{ code: hostile, severity: 'high', file: hostile + '.js', message: `${hostile} ${'x'.repeat(5000)}` }]);
+  const fl = floorOf('clean');
+  const reduced = agg.reduceVerdict({ floorReport: fl, lenses: [lens], tier: 'default' });
+  const md = assembleReport({ target: { sha: 'abc1234' }, tier: 'default', reduced, floorReport: fl, lenses: [lens] });
+  assert.ok(!RAW_DANGEROUS_RE.test(md), 'report carries a raw dangerous codepoint');
+  assert.ok(md.includes('steerme'), 'scrubbed text still present');
+  // Non-vacuous cap check: the 5000-char message must have been truncated —
+  // no run of 301+ payload chars may survive into the report.
+  assert.ok(!md.includes('x'.repeat(301)), 'the 5000-char lens message was not capped');
+});
+
+await test('"What was NOT checked" lists skipped files, deps, and failed lenses from real counts', () => {
+  const fl = floorOf('caution', {
+    skipped: [{ path: 'blob.bin', reason: 'non_utf8', bytes: 9 }],
+    findings: [{ code: 'dependencies_unfetched', severity: 'medium', file: 'package.json', message: 'm', detail: { declared: 3, unscannable: true } }],
+  });
+  const lenses = [lensOf([], 'timeout')];
+  const reduced = agg.reduceVerdict({ floorReport: fl, lenses, tier: 'default' });
+  const md = assembleReport({ target: { sha: 'abc1234' }, tier: 'default', reduced, floorReport: fl, lenses });
+  assert.ok(md.includes('blob.bin — skipped (non_utf8'));
+  assert.ok(md.includes('3 declared dependencies — never fetched'));
+  assert.ok(md.includes('did not complete (timeout)'));
+});
+
+console.log('');
+console.log(`${passed} passed, ${failed} failed`);
+if (failed > 0) process.exit(1);