@bongos/core 1.19.783 → 1.19.784

package/tests/session_accessibility.mjs

@@ -0,0 +1,243 @@
+// tests/session_accessibility.mjs
+//
+// Unit tests for the ACCESSIBILITY dimension of the session evaluator
+// (task 1002516; clause 4 of criterion `accessibility-fluency-profile`) —
+// modules/sessions/accessibility.js, which is PURE.
+//
+// These pin the four contracts the module's own header names, because those are
+// what make the metric safe rather than harmful. In particular the task's "watch
+// out" is that a metric which punishes an expert's terseness is ACTIVELY harmful,
+// so the terseness and waiver tests below are the point of the file, not trim.
+//
+// Run:  node tests/session_accessibility.mjs
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+const acc = require('../modules/sessions/accessibility.js');
+const interactionPrefs = require('../modules/builder-settings/interaction-prefs.js');
+
+let passed = 0;
+let failed = 0;
+function test(name, fn) {
+  try {
+    fn();
+    passed++;
+    console.log(`  ok  ${name}`);
+  } catch (err) {
+    failed++;
+    console.log(`  FAIL ${name}`);
+    console.log(`     ${err.message}`);
+  }
+}
+
+// ---- fixtures -------------------------------------------------------------
+
+// Plain prose carrying no jargon, no definition marker, no command handoff and
+// no framing term — so a fixture's length can be tuned without changing what any
+// detector sees.
+const FILLER = 'the quick brown fox jumps over the lazy dog again and again ';
+const padTo = (text, chars) => {
+  let s = text;
+  while (s.length < chars) s += FILLER;
+  return s.slice(0, Math.max(text.length, chars));
+};
+const say = (text, chars) => ({ role: 'assistant', snippet: chars ? padTo(text, chars) : text });
+
+// ---- the interaction-prefs tripwire ---------------------------------------
+//
+// accessibility.js may not require builder-settings (ADR 0083 forbids a module
+// importing a sibling), so it keys on literal profile values. This test is the
+// coupling: a rename in interaction-prefs must fail HERE rather than silently
+// stop waiving an engineer's session.
+
+test('tripwire: every profile value accessibility.js keys on still exists in interaction-prefs', () => {
+  for (const [field, values] of Object.entries(acc.PROFILE_VALUES)) {
+    assert.ok(interactionPrefs.FIELD_KEYS.includes(field), `interaction-prefs still has field ${field}`);
+    for (const v of values) {
+      assert.ok(interactionPrefs.isValidValue(field, v), `interaction-prefs still has ${field}=${v}`);
+    }
+  }
+});
+
+test('tripwire: the defaults accessibility.js falls back to are interaction-prefs own defaults', () => {
+  for (const [field, value] of Object.entries(acc.PROFILE_DEFAULTS)) {
+    assert.equal(interactionPrefs.defaultValue(field), value, `${field} default`);
+  }
+});
+
+// ---- contract A: profile-relative, never absolute --------------------------
+
+test('default profile scores the jargon signal (it is not waived)', () => {
+  const w = acc.waiversFor({});
+  assert.ok(!w.unexplained_jargon, 'jargon is scored for the default non-technical profile');
+  assert.ok(!w.terminal_where_ui_exists, 'terminal-first is scored for a toolchain-new profile');
+});
+
+test('engineer waives jargon + framing; comfortable waives terminal-first', () => {
+  const eng = acc.waiversFor({ technical_level: 'engineer' });
+  assert.ok(eng.unexplained_jargon, 'engineer waives jargon');
+  assert.ok(eng.implementation_over_outcome, 'engineer waives implementation framing');
+  assert.ok(!eng.terminal_where_ui_exists, 'engineer alone does not waive terminal-first');
+
+  const comf = acc.waiversFor({ toolchain_familiarity: 'comfortable' });
+  assert.ok(comf.terminal_where_ui_exists, 'comfortable waives terminal-first');
+  assert.ok(!comf.unexplained_jargon, 'comfortable alone does not waive jargon');
+});
+
+test('a waived signal carries the reason that waived it, not a silent absence', () => {
+  const r = acc.scoreAccessibility([say('the webhook is idempotent', 400)], {
+    profile: { technical_level: 'engineer' },
+  });
+  const jargon = r.signals.find((s) => s.type === 'unexplained_jargon');
+  assert.equal(jargon.waived, true);
+  assert.match(jargon.reason, /technical_level: engineer/);
+});
+
+test('every-signal-waived scores null, never a fabricated 10', () => {
+  const dense = say('the async closure memoize the webhook regex upsert', 300);
+  const r = acc.scoreAccessibility([dense], {
+    profile: { technical_level: 'engineer', toolchain_familiarity: 'comfortable' },
+  });
+  assert.equal(r.score, null, 'no scored signal → no score');
+  assert.equal(r.band, 'n/a');
+  assert.ok(r.signals.every((s) => s.waived), 'all three signals waived');
+});
+
+test('an expert profile can never score WORSE than the same transcript under the default', () => {
+  const detail = [say('the middleware closure will deserialize the jsonb schema', 500)];
+  const def = acc.scoreAccessibility(detail, { profile: {} });
+  const eng = acc.scoreAccessibility(detail, {
+    profile: { technical_level: 'engineer', toolchain_familiarity: 'comfortable' },
+  });
+  assert.ok(def.score != null && def.score < 10, 'the default profile does score this down');
+  assert.equal(eng.score, null, 'the expert profile is not scored down at all');
+});
+
+test('mixed technical level relaxes the jargon allowance rather than waiving it', () => {
+  // One jargon hit in ~500 sampled chars = a rate of 2.0 per 1000. The default
+  // free rate is 1.0 (a penalty); `mixed` doubles it to 2.0 (no penalty).
+  const detail = [say('the webhook fires on save', 500)];
+  assert.equal(acc.leniencyFor({ technical_level: 'mixed' }, 'unexplained_jargon'), 2);
+  const def = acc.scoreAccessibility(detail, { profile: {} });
+  const mixed = acc.scoreAccessibility(detail, { profile: { technical_level: 'mixed' } });
+  assert.ok(def.score < 10, 'default profile penalises the hit');
+  assert.equal(mixed.score, 10, 'mixed absorbs it inside the wider allowance');
+});
+
+// ---- contract B: terseness is never a penalty ------------------------------
+
+test('the score is a DENSITY: the same rate scores the same at any reply length', () => {
+  const short = [say('the webhook the closure the regex', 1000)];
+  const long = [say('the webhook the closure the regex', 1000), say('the webhook the closure the regex', 1000)];
+  const a = acc.scoreAccessibility(short, { profile: {} });
+  const b = acc.scoreAccessibility(long, { profile: {} });
+  assert.equal(a.sampled_chars * 2, b.sampled_chars, 'the fixture really is twice as long');
+  assert.equal(a.score, b.score, 'twice the prose at the same density scores identically');
+});
+
+test('verbosity is not an input — a concise profile is scored exactly like a detailed one', () => {
+  const detail = [say('the webhook the closure the regex', 900)];
+  const concise = acc.scoreAccessibility(detail, { profile: { verbosity: 'concise' } });
+  const detailed = acc.scoreAccessibility(detail, { profile: { verbosity: 'detailed' } });
+  assert.equal(concise.score, detailed.score);
+  assert.ok(!('verbosity' in concise.profile), 'verbosity is not even carried onto the result');
+});
+
+// ---- the detectors ---------------------------------------------------------
+
+test('jargon: a term with a nearby definition marker is explained, not counted', () => {
+  assert.equal(acc.countUnexplainedJargon(['the webhook fires on save']).count, 1);
+  assert.equal(acc.countUnexplainedJargon(['a webhook (a call another site makes to us) fires']).count, 0);
+  assert.equal(acc.countUnexplainedJargon(['a webhook — a call another site makes to us']).count, 0);
+  assert.equal(acc.countUnexplainedJargon(['a webhook, which means another site calls us']).count, 0);
+});
+
+test('jargon: matching is word-bounded, so ordinary English is never a hit', () => {
+  // "ORM" inside "format", "CORS" inside "scores", "seam" inside "seams" — the
+  // substring bug this guards is the difference between a metric and a nuisance.
+  assert.equal(acc.countUnexplainedJargon(['the format of the scores is fine']).count, 0);
+  assert.equal(acc.countUnexplainedJargon(['performance is normal and the form works']).count, 0);
+});
+
+test('jargon: multi-token and punctuated terms still anchor', () => {
+  assert.equal(acc.countUnexplainedJargon(['this is a race condition']).count, 1);
+  assert.equal(acc.countUnexplainedJargon(['a classic N+1 here']).count, 1);
+});
+
+test('terminal-first: only counted when the subject HAS a click-path', () => {
+  const withUi = acc.countTerminalWhereUiExists(['run `node scripts/gds/api-key.js` to set your api key']);
+  assert.equal(withUi.count, 1);
+  assert.match(withUi.evidence, /Settings/, 'the evidence names where the button is');
+
+  // A command for something with no UI at all is the only way to do the job.
+  assert.equal(acc.countTerminalWhereUiExists(['run `npm test` to check the suite']).count, 0);
+  // Naming a UI-backed subject without handing over a command is not a finding.
+  assert.equal(acc.countTerminalWhereUiExists(['your api key is missing']).count, 0);
+});
+
+test('framing: implementation taxonomy counts only when no outcome framing rides with it', () => {
+  assert.equal(acc.countImplementationOverOutcome(['this changes the module boundary and the data model']).count, 1);
+  assert.equal(
+    acc.countImplementationOverOutcome(['this changes the data model, so it costs about 20 minutes']).count,
+    0,
+    'naming the architecture AND what it costs is exactly right'
+  );
+});
+
+// ---- scoring shape ---------------------------------------------------------
+
+test('score is monotonic in the finding rate and clamped to [0,10]', () => {
+  const clean = acc.scoreAccessibility([say('we changed the button label', 1000)], { profile: {} });
+  const some = acc.scoreAccessibility([say('the webhook the closure the regex', 1000)], { profile: {} });
+  const awful = acc.scoreAccessibility(
+    [say('the webhook closure regex upsert jsonb middleware singleton polyfill debounce mutex', 200)],
+    { profile: {} }
+  );
+  assert.equal(clean.score, 10);
+  assert.ok(some.score < clean.score, 'more findings score lower');
+  assert.ok(awful.score < some.score, 'many more findings score lower still');
+  assert.ok(awful.score >= 0, 'never below 0');
+});
+
+test('no assistant prose → null score and an honest confidence, not a default pass', () => {
+  const r = acc.scoreAccessibility([], { profile: {} });
+  assert.equal(r.score, null);
+  assert.equal(r.confidence, 'none');
+  assert.equal(r.sampled_turns, 0);
+});
+
+test('user turns are not scored — the metric judges the agent, not the builder', () => {
+  const detail = [
+    { role: 'user', snippet: 'the webhook closure regex upsert jsonb middleware' },
+    say('we changed the button label', 1000),
+  ];
+  const r = acc.scoreAccessibility(detail, { profile: {} });
+  assert.equal(r.sampled_turns, 1, 'only the assistant turn was sampled');
+  assert.equal(r.score, 10, 'the builder\'s own jargon does not count against the agent');
+});
+
+test('a thin sample says so rather than pretending to a normal reading', () => {
+  const r = acc.scoreAccessibility([say('we changed the button label')], { profile: {} });
+  assert.ok(r.sampled_chars < acc.LOW_CONFIDENCE_CHARS);
+  assert.equal(r.confidence, 'low');
+});
+
+test('an unknown or partial profile falls back to the accessible default, never unscored', () => {
+  const r = acc.scoreAccessibility([say('the webhook fires', 1000)], {
+    profile: { technical_level: 'wizard', toolchain_familiarity: null },
+  });
+  assert.equal(r.profile.technical_level, 'non_technical');
+  assert.equal(r.profile.toolchain_familiarity, 'new');
+  assert.ok(r.score != null, 'a junk profile is still scored, at the default posture');
+});
+
+test('the signal spec can never penalise more than a 10 in total', () => {
+  const total = Object.values(acc.SIGNAL_SPEC).reduce((n, s) => n + s.max, 0);
+  assert.equal(total, 10, 'the three signals sum to exactly the full scale');
+});
+
+console.log('');
+console.log(`Session accessibility (clause 4) tests: ${passed} passed, ${failed} failed`);
+if (failed > 0) process.exit(1);

package/tests/session_eval.mjs

@@ -331,6 +331,57 @@ test('evaluateSession: safe on empty/malformed', () => {
   assert.equal(ev.evaluateSession(null).verdict, 'efficient');
 });
 
+// ---- accessibility dimension (task 1002516, clause 4) ----
+//
+// The dimension itself is tested in tests/session_accessibility.mjs; these pin
+// the WIRING — that the evaluator prefers the score the upload route already
+// computed, re-scores a row that predates it, and only raises a footgun when the
+// finding is both real and profile-sanctioned-free.
+
+test('accessibilityFor prefers the stored score over re-scoring', () => {
+  const stored = { score: 4.2, band: 'dense', signals: [], confidence: 'normal' };
+  const r = ev.accessibilityFor({ metrics: { accessibility: stored }, detail: [] });
+  assert.equal(r, stored, 'the route\'s score — it had the real profile — is used as-is');
+  assert.ok(!r.rescored);
+});
+
+test('accessibilityFor re-scores a row that predates the dimension, and says so', () => {
+  const r = ev.accessibilityFor({ metrics: {}, detail: [{ role: 'assistant', snippet: 'we changed the button label' }] });
+  assert.equal(r.rescored, true, 'a re-derived score is marked, not passed off as the original');
+  assert.ok('score' in r);
+});
+
+test('detectFootguns: unplain_speech fires below the 6.0 threshold', () => {
+  const acc = { score: 3.5, band: 'opaque', confidence: 'normal', signals: [{ type: 'unexplained_jargon', waived: false, penalty: 3, evidence: '"webhook" undefined' }] };
+  const out = ev.detectFootguns([], {}, { accessibility: acc });
+  const f = out.find((x) => x.type === 'unplain_speech');
+  assert.ok(f, 'the footgun fires');
+  assert.match(f.evidence, /unexplained_jargon/, 'the evidence names the worst signal');
+});
+
+test('detectFootguns: unplain_speech never fires on a waived (null) score or a thin sample', () => {
+  // An engineer's session waives every signal and scores null — it must be
+  // impossible for it to reach this finding at all.
+  const waived = { score: null, band: 'n/a', confidence: 'normal', signals: [] };
+  assert.equal(ev.detectFootguns([], {}, { accessibility: waived }).some((x) => x.type === 'unplain_speech'), false);
+  const thin = { score: 2, band: 'opaque', confidence: 'low', signals: [] };
+  assert.equal(ev.detectFootguns([], {}, { accessibility: thin }).some((x) => x.type === 'unplain_speech'), false);
+  const fine = { score: 8, band: 'clear', confidence: 'normal', signals: [] };
+  assert.equal(ev.detectFootguns([], {}, { accessibility: fine }).some((x) => x.type === 'unplain_speech'), false);
+  assert.equal(ev.detectFootguns([], {}, {}).some((x) => x.type === 'unplain_speech'), false);
+});
+
+test('evaluateSession surfaces the accessibility dimension per session', () => {
+  const r = ev.evaluateSession({
+    session_id: 's1',
+    github_login: 'alice',
+    metrics: { accessibility: { score: 7.5, band: 'workable', confidence: 'normal', signals: [] } },
+    detail: [],
+  });
+  assert.equal(r.accessibility.score, 7.5);
+  assert.equal(r.builder, 'alice', 'the score is attributable to a builder');
+});
+
 test('detectFootguns: a null/non-object turn does not throw (regression)', () => {
   // computeOverhead guards each turn; detectFootguns must too, or one malformed
   // turn crashes the whole evaluation.
@@ -376,7 +427,7 @@ test('findingsByBuilder groups flagged sessions; excludes footgun-free builders'
 });
 
 test('FINDING_TEMPLATES cover every footgun class', () => {
-  for (const t of ['guess_and_retry_api', 'heavyweight_pipeline_on_trivial_change', 'stale_test_red_herring', 'repeated_tool_failures', 'empty_diff_grade', 'claim_consumed_on_fail', 'wrong_path_worktree', 'off_task_wandering']) {
+  for (const t of ['guess_and_retry_api', 'heavyweight_pipeline_on_trivial_change', 'stale_test_red_herring', 'repeated_tool_failures', 'empty_diff_grade', 'claim_consumed_on_fail', 'wrong_path_worktree', 'off_task_wandering', 'unplain_speech']) {
     assert.ok(ev.FINDING_TEMPLATES[t] && ev.FINDING_TEMPLATES[t].title, `template for ${t}`);
   }
 });