@bongos/core 1.19.387 → 1.19.389

package/tests/grader_quality_window_overflow.mjs

@@ -0,0 +1,324 @@
+// tests/grader_quality_window_overflow.mjs
+//
+// task 1003318: the grader's Quality worker ran on haiku 4.5, whose 200k context
+// window overflows on any diff over ~150 KB. The CLI exits before the worker
+// starts (`Prompt is too long: the request is ~251248 tokens (limit 200000)`),
+// the exit was retried once as if transient, the dead axis was forwarded to the
+// server as code_quality: null → Number(null) === 0 → a fabricated
+// "FAIL — code_quality below 6" at (9+0)/2 = 4.5. Deterministic, not flaky:
+// tasks 1003297 and 1003298 each burned three ~$5 panel runs on it.
+//
+// Three things must hold now:
+//   1. the Quality tier is a model whose window covers the diffs the panel sees
+//      (claude-sonnet-5), still behind the ADR 0024 cross-model guard;
+//   2. a window overflow is recorded as its own non-transient worker error
+//      (PROMPT_TOO_LONG) — no retry, stderr tail kept — and the panel treats it
+//      as missing signal (CONCERNS), never a 0/10 axis;
+//   3. the server, seeing an explicit code_quality: null corroborated by an
+//      errored no-finding Quality worker, records the lens as UNAVAILABLE
+//      (json-null axis, score on the axis that ran, cq_lens_unavailable) and
+//      parks the task for manual-confirm — fail-closed, never an auto-pass, and
+//      never the fabricated 4.5. An uncorroborated null stays schema_error.
+//
+// Run:  node tests/grader_quality_window_overflow.mjs
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+const rubric = require('../modules/grading/grader-rubric.json');
+const grader = require('../modules/grading/grader.js');
+const workersMod = require('../modules/grading/grader-workers/index.js');
+const ship = require('../scripts/gds/ship.js');
+
+const { aggregatePanel, validateAndScore, workerModelFor } = grader;
+const { runWorker, TRANSIENT_WORKER_ERRORS } = workersMod;
+const { formatGradeOutcome } = ship;
+
+let passed = 0;
+let failed = 0;
+function test(name, fn) {
+  try { fn(); passed++; console.log(`  ok  ${name}`); }
+  catch (err) { failed++; console.error(`  FAIL ${name}`); console.error(`     ${err.message}`); }
+}
+async function atest(name, fn) {
+  try { await fn(); passed++; console.log(`  ok  ${name}`); }
+  catch (err) { failed++; console.error(`  FAIL ${name}`); console.error(`     ${err.message}`); }
+}
+
+const OVERFLOW_LINE = 'Prompt is too long: the request is ~251248 tokens (limit 200000)';
+
+// The exact error runSubagent throws on a nonzero exit (grader-subagent.js).
+const nonzeroExit = ({ exitCode = 1, stderr = '', stdout = '' } = {}) =>
+  Object.assign(new Error(`subagent exited with code ${exitCode}`), {
+    code: 'SUBAGENT_NONZERO_EXIT', exit_code: exitCode, run_ms: 120, stdout, stderr,
+  });
+
+// A Quality-shaped worker stub: no spawning, the real error short-circuit shape.
+const STUB_QUALITY = {
+  KIND: 'quality', NAME: 'Quality', SPECIALTY: 'stub',
+  buildPrompt: () => 'prompt',
+  extractJson: (s) => JSON.parse(s),
+  computeVerdict: (parsed, meta) => (meta.subprocess_error
+    ? { verdict: 'fail', scores: {}, issues: [], notes_md: '', confidence: 0, error: meta.subprocess_error }
+    : { verdict: 'pass', scores: { code_quality: 9 }, issues: [], notes_md: '', confidence: 0.9, error: null }),
+};
+
+const healthy = (kind, scores) => ({ kind, verdict: 'pass', scores, issues: [], confidence: 0.85, error: null });
+
+// ---------- 1. the tier ----------
+
+test('rubric: the Quality worker tier is claude-sonnet-5, not the 200k-window haiku', () => {
+  assert.equal(rubric.v4.worker_model_tiers.quality, 'claude-sonnet-5');
+  assert.doesNotMatch(String(rubric.v4.worker_model_tiers.quality), /haiku/);
+  assert.match(rubric.v4.worker_model_tiers_note, /1003318/, 'the note records why the tier moved');
+});
+
+test('workerModelFor: the sonnet tier is used for a fable builder, refused for a sonnet builder (ADR 0024 guard)', () => {
+  assert.equal(workerModelFor('quality', 'claude-sonnet-5', 'claude-fable-5'), 'claude-sonnet-5');
+  assert.equal(workerModelFor('quality', 'claude-opus-5', 'claude-sonnet-5'), 'claude-opus-5',
+    'a sonnet builder must not be graded by its own family — falls back to the panel model');
+});
+
+// ---------- 2. the worker ----------
+
+await atest('runWorker: a "Prompt is too long" exit is recorded as PROMPT_TOO_LONG and NOT retried', async () => {
+  let calls = 0;
+  const r = await runWorker(STUB_QUALITY, {
+    task: {}, diff: '', changedFiles: [], model: 'm',
+    runSubagentFn: async () => { calls++; throw nonzeroExit({ exitCode: 1, stderr: OVERFLOW_LINE }); },
+  });
+  assert.equal(calls, 1, 'a window overflow is deterministic — the retry only burns another round');
+  assert.equal(r.retried, false);
+  assert.equal(r.error, 'PROMPT_TOO_LONG');
+  assert.equal(TRANSIENT_WORKER_ERRORS.has('PROMPT_TOO_LONG'), false);
+  assert.deepEqual(r.scores, {}, 'no axis is fabricated for a worker that never ran');
+  assert.equal(r.exit_code, 1);
+  assert.match(r.stderr_tail, /Prompt is too long/, 'the CLI line survives so the grade row names its cause');
+});
+
+await atest('runWorker: the overflow line on STDOUT classifies the same way', async () => {
+  const r = await runWorker(STUB_QUALITY, {
+    task: {}, diff: '', changedFiles: [], model: 'm',
+    runSubagentFn: async () => { throw nonzeroExit({ exitCode: 1, stdout: OVERFLOW_LINE }); },
+  });
+  assert.equal(r.error, 'PROMPT_TOO_LONG');
+  assert.equal(r.retried, false);
+});
+
+await atest('runWorker: an unrelated nonzero exit keeps SUBAGENT_NONZERO_EXIT and its one retry (#465 unchanged)', async () => {
+  let calls = 0;
+  const r = await runWorker(STUB_QUALITY, {
+    task: {}, diff: '', changedFiles: [], model: 'm',
+    runSubagentFn: async () => { calls++; throw nonzeroExit({ exitCode: 1, stderr: 'Invalid API key · Run /login' }); },
+  });
+  assert.equal(calls, 2);
+  assert.equal(r.retried, true);
+  assert.equal(r.error, 'SUBAGENT_NONZERO_EXIT');
+});
+
+const DEAD_QUALITY = {
+  kind: 'quality', verdict: 'fail', error: 'PROMPT_TOO_LONG', scores: {}, issues: [],
+  confidence: 0, exit_code: 1, stderr_tail: OVERFLOW_LINE,
+};
+const PANEL_WITH_DEAD_QUALITY = [
+  healthy('narc', { functional_fidelity: 9 }),
+  DEAD_QUALITY,
+  healthy('hacker', { security_score: 9 }),
+  healthy('efficiency', { efficiency_score: 8 }),
+];
+
+test('aggregatePanel: an overflowed Quality is missing signal → CONCERNS with a null code_quality axis, not a 0', () => {
+  const r = aggregatePanel(PANEL_WITH_DEAD_QUALITY);
+  assert.equal(r.signals.panel_outcome, 'concerns');
+  assert.equal(r.passed, false);
+  assert.equal(r.rubric_scores.code_quality, null);
+  assert.equal(r.signals.error_count, 1);
+  assert.equal(r.signals.fail_count, 0);
+  assert.match(r.notes_md, /quality\s+ERROR.*PROMPT_TOO_LONG/);
+});
+
+// ---------- 3. the server ----------
+
+// What ship-grade-steps.buildGradePayload forwards for the panel above.
+const OVERFLOW_PAYLOAD = {
+  functional_fidelity: 9,
+  code_quality: null,
+  security_score: 9,
+  efficiency_score: 8,
+  notes_md: 'quality overflowed; the rest passed',
+  confidence: 0.3,
+  issues: [],
+};
+const OVERFLOW_META = { workers: PANEL_WITH_DEAD_QUALITY, panel_outcome: 'concerns' };
+
+test('validateAndScore: corroborated null cq → lens UNAVAILABLE, score on the axis that ran, never the fabricated 4.5', () => {
+  const r = validateAndScore(OVERFLOW_PAYLOAD, OVERFLOW_META);
+  assert.equal(r.passed, false, 'a gating lens that did not run can never auto-pass');
+  assert.equal(r.signals.cq_lens_unavailable, true);
+  assert.notEqual(r.signals.schema_error, true);
+  assert.equal(r.rubric_scores.code_quality, null, 'json-null axis — the DB buckets it as unavailable, not a 0 fail');
+  assert.equal(r.rubric_scores.code_quality_effective, null);
+  assert.equal(r.rubric_scores.functional_fidelity, 9);
+  assert.equal(r.score, 9, 'the axis that ran — not (9+0)/2');
+  assert.match(r.notes_md, /lens unavailable/);
+  assert.match(r.notes_md, /manual-confirm/);
+  assert.doesNotMatch(r.notes_md, /code_quality below/);
+  assert.doesNotMatch(r.notes_md, /cq=0\//);
+});
+
+test('validateAndScore: null cq with a HEALTHY Quality in signals → fail-closed schema_error (injection shape)', () => {
+  const r = validateAndScore(OVERFLOW_PAYLOAD, { workers: [
+    healthy('narc', { functional_fidelity: 9 }),
+    { ...healthy('quality', { code_quality: null }) },
+    healthy('hacker', { security_score: 9 }),
+  ] });
+  assert.equal(r.passed, false);
+  assert.equal(r.signals.schema_error, true);
+  assert.notEqual(r.signals.cq_lens_unavailable, true);
+  assert.match(r.notes_md, /without a corroborated Quality outage/);
+});
+
+test('validateAndScore: null cq with an errored Quality that still raised a major issue → fail-closed (a finding wins)', () => {
+  const r = validateAndScore(OVERFLOW_PAYLOAD, { workers: [
+    healthy('narc', { functional_fidelity: 9 }),
+    { ...DEAD_QUALITY, issues: [{ severity: 'major', file: 'x.js', summary: 'real finding' }] },
+    healthy('hacker', { security_score: 9 }),
+  ] });
+  assert.equal(r.passed, false);
+  assert.equal(r.signals.schema_error, true);
+  assert.notEqual(r.signals.cq_lens_unavailable, true);
+});
+
+test('validateAndScore: null cq with NO workers signal at all → fail-closed schema_error', () => {
+  const r = validateAndScore(OVERFLOW_PAYLOAD, {});
+  assert.equal(r.passed, false);
+  assert.equal(r.signals.schema_error, true);
+});
+
+test('validateAndScore: a JUDGED cq of 0 beside an errored Quality is still a real FAIL (only corroborated null is a lens outage)', () => {
+  const r = validateAndScore({ ...OVERFLOW_PAYLOAD, code_quality: 0, confidence: 0.9 }, OVERFLOW_META);
+  assert.equal(r.passed, false);
+  assert.notEqual(r.signals.cq_lens_unavailable, true);
+  assert.equal(r.rubric_scores.code_quality, 0);
+  assert.match(r.notes_md, /code_quality below/);
+});
+
+test('validateAndScore: both lenses out (null ff + null cq, both corroborated) → unavailable, score 0, no pass', () => {
+  const r = validateAndScore({ ...OVERFLOW_PAYLOAD, functional_fidelity: null }, { workers: [
+    { kind: 'narc', verdict: 'fail', error: 'SUBAGENT_TIMEOUT', scores: {}, issues: [] },
+    DEAD_QUALITY,
+    healthy('hacker', { security_score: 9 }),
+  ] });
+  assert.equal(r.passed, false);
+  assert.equal(r.signals.ff_lens_unavailable, true);
+  assert.equal(r.signals.cq_lens_unavailable, true);
+  assert.equal(r.score, 0);
+  assert.equal(r.rubric_scores.functional_fidelity, null);
+  assert.equal(r.rubric_scores.code_quality, null);
+});
+
+test('validateAndScore: the healthy path is byte-for-byte unchanged (both axes judged → PASS at (9+8)/2)', () => {
+  const r = validateAndScore({ functional_fidelity: 9, code_quality: 8, confidence: 0.9, issues: [], notes_md: '' }, {
+    workers: [healthy('narc', { functional_fidelity: 9 }), healthy('quality', { code_quality: 8 }), healthy('hacker', { security_score: 9 })],
+  });
+  assert.equal(r.passed, true);
+  assert.equal(r.score, 8.5);
+  assert.notEqual(r.signals.cq_lens_unavailable, true);
+});
+
+// ---------- the builder-facing card ----------
+
+test('formatGradeOutcome: a Quality-lens outage reads as UNAVAILABLE + manual-confirm, never "FAIL 4.5 — fix the issues"', () => {
+  const g = validateAndScore(OVERFLOW_PAYLOAD, OVERFLOW_META);
+  const out = formatGradeOutcome(g, 1003318);
+  assert.match(out, /QUALITY LENS UNAVAILABLE/i);
+  assert.match(out, /not a quality rejection/i);
+  assert.match(out, /Prompt is too long/, 'the CLI line explains the outage');
+  assert.match(out, /--regrade/);
+  assert.match(out, /\/confirm/, 'manual-confirm is the release valve');
+  assert.doesNotMatch(out, /Fix the issues above/);
+  assert.doesNotMatch(out, /blended 4\.5/);
+  assert.doesNotMatch(out, /code_quality:\s+0\/10/);
+});
+
+// ---------- the review's adversarial shapes (a dark lens never softens the gate) ----------
+
+await atest('runWorker: a SUBAGENT_TIMEOUT whose partial stdout quotes the overflow sentence keeps its timeout tag', async () => {
+  const r = await runWorker(STUB_QUALITY, {
+    task: {}, diff: '', changedFiles: [], model: 'm',
+    runSubagentFn: async () => {
+      throw Object.assign(new Error('timeout'), { code: 'SUBAGENT_TIMEOUT', run_ms: 360000, stdout: `quoting: ${OVERFLOW_LINE}`, stderr: '' });
+    },
+  });
+  assert.equal(r.error, 'SUBAGENT_TIMEOUT', 'the #467 budget signal is never relabelled');
+});
+
+await atest('runWorker: stdout is consulted only when stderr is silent (the model may quote a diff that mentions the phrase)', async () => {
+  const r = await runWorker(STUB_QUALITY, {
+    task: {}, diff: '', changedFiles: [], model: 'm',
+    runSubagentFn: async () => { throw nonzeroExit({ exitCode: 1, stderr: 'Invalid API key · Run /login', stdout: `diff says: ${OVERFLOW_LINE}` }); },
+  });
+  assert.equal(r.error, 'SUBAGENT_NONZERO_EXIT');
+  assert.equal(r.retried, true);
+});
+
+const BLOCKER = { severity: 'blocker', file: 'src/gds/routes/x.js', summary: 'ungated write route' };
+const HACKER_FAIL_META = { workers: [
+  healthy('narc', { functional_fidelity: 9 }),
+  DEAD_QUALITY,
+  { kind: 'hacker', verdict: 'fail', scores: { security_score: 2 }, issues: [BLOCKER], confidence: 0.9, error: null },
+], panel_outcome: 'fail' };
+
+test('validateAndScore: a Hacker BLOCKER beside a dead Quality is a FAIL with a missing axis, never an outage', () => {
+  const r = validateAndScore({ ...OVERFLOW_PAYLOAD, security_score: 2, confidence: 0, issues: [BLOCKER] }, HACKER_FAIL_META);
+  assert.equal(r.passed, false);
+  assert.equal(r.signals.cq_lens_unavailable, true, 'the axis really is unavailable');
+  assert.equal(r.rubric_scores.code_quality, null);
+  assert.doesNotMatch(r.notes_md, /not a quality rejection/);
+  assert.doesNotMatch(r.notes_md, /manual-confirm available/);
+  assert.match(r.notes_md, /FAIL/);
+});
+
+test('formatGradeOutcome: with a blocker from another lens the card prints the findings, not the outage invitation', () => {
+  const g = validateAndScore({ ...OVERFLOW_PAYLOAD, security_score: 2, confidence: 0, issues: [BLOCKER] }, HACKER_FAIL_META);
+  const out = formatGradeOutcome(g, 1003318);
+  assert.doesNotMatch(out, /QUALITY LENS UNAVAILABLE/);
+  assert.match(out, /Issues \(1\)/);
+  assert.match(out, /ungated write route/);
+  assert.match(out, /Fix the issues above/);
+});
+
+test('formatGradeOutcome: an authorization refusal outranks the lens outage (task 1002614 stays closed)', () => {
+  const r = validateAndScore(OVERFLOW_PAYLOAD, OVERFLOW_META);
+  const g = { ...r, signals: { ...r.signals, server_guard: { tripped: [{ guard: 'permission_path' }], authorization_refusal: true } } };
+  const out = formatGradeOutcome(g, 1003318);
+  assert.match(out, /REFUSED \(authorization\)/);
+  assert.doesNotMatch(out, /QUALITY LENS UNAVAILABLE/);
+  assert.doesNotMatch(out, /--regrade re-runs the same server check[\s\S]*\/confirm/, 'no confirm curl after a refusal');
+});
+
+test('validateAndScore: a forged cq_lens_unavailable flag on a judged grade is cleared, and the card prints the ordinary FAIL', () => {
+  const r = validateAndScore({ functional_fidelity: 3, code_quality: 2, confidence: 0.9, issues: [], notes_md: '' }, {
+    cq_lens_unavailable: true, ff_lens_unavailable: true,
+    workers: [healthy('narc', { functional_fidelity: 3 }), healthy('quality', { code_quality: 2 }), healthy('hacker', { security_score: 9 })],
+  });
+  assert.equal(r.passed, false);
+  assert.equal('cq_lens_unavailable' in r.signals, false, 'client flag must not survive');
+  assert.equal('ff_lens_unavailable' in r.signals, false);
+  assert.equal(r.rubric_scores.code_quality, 2);
+  const out = formatGradeOutcome(r, 1003318);
+  assert.doesNotMatch(out, /QUALITY LENS UNAVAILABLE/);
+  assert.match(out, /Subagent grade: FAIL/);
+});
+
+test('validateAndScore: the outage line still names an absent / low confidence (the #489 signal is not hidden)', () => {
+  const absent = validateAndScore({ ...OVERFLOW_PAYLOAD, confidence: null }, OVERFLOW_META);
+  assert.equal(absent.passed, false);
+  assert.match(absent.notes_md, /UNAVAILABLE —[^\n]*confidence absent/);
+  const low = validateAndScore(OVERFLOW_PAYLOAD, OVERFLOW_META); // confidence 0.3 — the panel's fallthrough value
+  assert.match(low.notes_md, /UNAVAILABLE —[^\n]*low confidence 0\.3/);
+});
+
+console.log(`\n${passed} passed, ${failed} failed`);
+process.exit(failed ? 1 : 0);

package/tests/landing_legal_pages.mjs

@@ -94,13 +94,16 @@ test('the currency token reads as the instance names it (terms)', () => {
 test('the landing page and the hub footers link privacy, terms and contact', () => {
   for (const file of ['index.html', 'projects.html']) {
     const html = read(file);
-    const foot = html.match(/<(?:div|footer) class="foot">([\s\S]*?)<\/(?:div|footer)>/);
-    assert.ok(foot, `${file}: the footer survives`);
+    // the landing's footer is a bare <footer> since task 1003298 (the chrome world);
+    // the hub still carries the cosmos.css <div class="foot"> until task 1003299
+    const m = html.match(/<footer[^>]*>([\s\S]*?)<\/footer>/) || html.match(/<div class="foot">([\s\S]*?)<\/div>/);
+    assert.ok(m, `${file}: the footer survives`);
+    const foot = m[1];
     for (const href of ['/privacy', '/terms', '/contact']) {
-      assert.match(foot[1], new RegExp(`<a href="${href}">`), `${file}: footer links ${href}`);
+      assert.match(foot, new RegExp(`<a href="${href}">`), `${file}: footer links ${href}`);
     }
-    assert.match(foot[1], /Open source \(AGPL-3\.0\)/, `${file}: the licence link stays`);
-    assert.match(foot[1], /Built by agents/, `${file}: the attribution half stays`);
+    assert.match(foot, /Open source \(AGPL-3\.0\)/, `${file}: the licence link stays`);
+    assert.match(foot, /Built by agents/, `${file}: the attribution half stays`);
   }
 });