@hyperfixi/testing-framework 2.7.2 → 2.8.0

package/src/multilingual/foreign-canonical-validity.ts

@@ -0,0 +1,158 @@
+/**
+ * Foreign→English canonical-validity gate
+ * ---------------------------------------
+ * The sibling en-render gate (canonical-validity.ts) renders every corpus English
+ * reference and parses it on the real `hyperscript.org` engine. But the PRODUCTION
+ * path is foreign→English: an authored non-English source is parsed and rendered to
+ * English (`preprocessToEnglish` → `render`). A parse can be role-faithful (the
+ * fidelity ratchet scores ~1.0) yet still render English the canonical parser rejects.
+ * (The build-time `@hyperscript-tools/i18n` transpiler now parse-gates its ENGLISH
+ * side with the same loader recipe; faithful foreign-output gating there still awaits
+ * the v2 semantic-engine transpiler, since its GrammarTransformer is lossy in reverse.)
+ *
+ * This gate closes that blind spot for the multilingual path: for every language, it
+ * renders each authored `pattern_translation` to English and parses the result on
+ * the canonical engine, failing on any invalid (pattern, language) pair that is not
+ * in the committed allowlist. The allowlist is keyed by pattern id → the languages
+ * that currently fail, so a fix that clears a family across languages shrinks (or
+ * removes) its entry — the list only ever ratchets down.
+ *
+ * Denominator: only (pattern, language) pairs whose EN reference the canonical parser
+ * already accepts are scored, so a handful of inherently non-canonical corpus rows
+ * never distort the signal (same fairness rule as the en gate).
+ *
+ * DB dependency: reads authored translations from `pattern_translations`, which only
+ * exist after `npm run populate`. Generate the baseline and run the gate against a
+ * freshly populated DB (CI's multilingual-validation job populates; see the
+ * provenance-stamp discipline in packages/patterns-reference/CLAUDE.md).
+ */
+
+import { getAllPatterns, getTranslationsByLanguage } from '@hyperfixi/patterns-reference';
+import { parseSemantic, render } from '@lokascript/semantic';
+import { loadCanonicalParser, type CanonicalValidate } from './canonical-validity';
+
+/**
+ * The 23 non-English priority languages (the `browser-priority` corpus set). English
+ * is the reference, scored by the sibling en gate.
+ */
+export const FOREIGN_LANGUAGES = [
+  'es',
+  'fr',
+  'pt',
+  'it',
+  'id',
+  'ms',
+  'sw',
+  'zh',
+  'vi',
+  'tl',
+  'ja',
+  'ko',
+  'tr',
+  'qu',
+  'hi',
+  'bn',
+  'ar',
+  'de',
+  'ru',
+  'uk',
+  'pl',
+  'th',
+  'he',
+] as const;
+
+export interface ForeignValidityFailure {
+  /** Corpus pattern id (`code_example` id the translation belongs to). */
+  id: string;
+  language: string;
+  /** The authored foreign source that was rendered. */
+  foreign: string;
+  /** The English the renderer produced from the foreign parse. */
+  rendered: string;
+  error: string;
+}
+
+export interface ForeignValidityResult {
+  /** (pattern, language) pairs whose EN reference the canonical parser accepts. */
+  checked: number;
+  valid: number;
+  failures: ForeignValidityFailure[];
+}
+
+/**
+ * Render every authored foreign `pattern_translation` to English and parse it on the
+ * canonical engine. Only pairs whose EN reference is itself canonical-valid are scored.
+ */
+export async function checkForeignRenderValidity(opts?: {
+  validate?: CanonicalValidate;
+  languages?: readonly string[];
+  /** Translations fetched per language (defaults comfortably above the corpus size). */
+  perLanguageLimit?: number;
+}): Promise<ForeignValidityResult> {
+  const validate = opts?.validate ?? (await loadCanonicalParser());
+  const languages = opts?.languages ?? FOREIGN_LANGUAGES;
+  const limit = opts?.perLanguageLimit ?? 500;
+
+  const patterns = await getAllPatterns();
+  const enAccepts = new Map(patterns.map(p => [p.id, validate(p.rawCode).length === 0]));
+
+  const failures: ForeignValidityFailure[] = [];
+  let checked = 0;
+  let valid = 0;
+
+  for (const language of languages) {
+    const translations = await getTranslationsByLanguage(language, limit);
+    for (const t of translations) {
+      if (!enAccepts.get(t.codeExampleId)) continue; // fair denominator: EN accepts the reference
+      checked++;
+
+      let rendered: string;
+      let errors: string[];
+      try {
+        const node = parseSemantic(t.hyperscript, language).node;
+        rendered = node ? render(node, 'en') : '(no node)';
+        errors = validate(rendered);
+      } catch (e) {
+        // parseSemantic/render only — validate never throws (its tokenizer-level
+        // throws fold into the returned array), so an invalid render is reported
+        // WITH the render that caused it, not discarded as '(threw)'.
+        rendered = '(threw)';
+        errors = ['threw: ' + (e as Error).message.split('\n')[0]];
+      }
+
+      if (errors.length === 0) {
+        valid++;
+      } else {
+        failures.push({
+          id: t.codeExampleId,
+          language,
+          foreign: t.hyperscript,
+          rendered,
+          error: errors[0] ?? 'unknown error',
+        });
+      }
+    }
+  }
+
+  return { checked, valid, failures };
+}
+
+/**
+ * Group failures into the committed allowlist shape: `{ patternId: [langs…] }`
+ * (languages sorted for a stable diff). Used by the baseline generator and by the
+ * gate's stale-entry check.
+ */
+export function groupFailuresByPattern(
+  failures: readonly ForeignValidityFailure[]
+): Record<string, string[]> {
+  const byPattern = new Map<string, Set<string>>();
+  for (const f of failures) {
+    if (!byPattern.has(f.id)) byPattern.set(f.id, new Set());
+    byPattern.get(f.id)!.add(f.language);
+  }
+  const out: Record<string, string[]> = {};
+  for (const id of [...byPattern.keys()].sort()) {
+    out[id] = [...byPattern.get(id)!].sort();
+  }
+  return out;
+}

package/src/multilingual/orchestrator.ts

@@ -20,7 +20,13 @@ import type {
   Reporter,
   BundleInfo,
 } from './types';
-import { computeFidelity, computePrecision, FIDELITY_THRESHOLD } from './fidelity';
+import {
+  computeFidelity,
+  computePrecision,
+  computeMultisetRecall,
+  spuriousActions,
+  FIDELITY_THRESHOLD,
+} from './fidelity';
 
 const execAsync = promisify(exec);
 
@@ -258,6 +264,8 @@ export class TestOrchestrator {
     const multisetReference = new Map<string, string[]>();
     // R1: codeExampleId -> English role signature (action.role:valueType set).
     const roleReference = new Map<string, string[]>();
+    // R3: codeExampleId -> English role-VALUE signature (invariant values, multiset).
+    const roleValueReference = new Map<string, string[]>();
     for (const r of en.parseResults) {
       if (r.success && r.actionSignature && r.actionSignature.length > 0) {
         reference.set(r.pattern.codeExampleId, r.actionSignature);
@@ -268,6 +276,9 @@ export class TestOrchestrator {
       if (r.success && r.roleSignature && r.roleSignature.length > 0) {
         roleReference.set(r.pattern.codeExampleId, r.roleSignature);
       }
+      if (r.success && r.roleValueSignature && r.roleValueSignature.length > 0) {
+        roleValueReference.set(r.pattern.codeExampleId, r.roleValueSignature);
+      }
     }
 
     for (const lang of languageResults) {
@@ -277,7 +288,9 @@ export class TestOrchestrator {
       const lossy: string[] = [];
       const scores: number[] = [];
       const precisionScores: number[] = [];
+      const multisetRecallScores: number[] = [];
       const roleScores: number[] = [];
+      const valueRecallScores: number[] = [];
 
       for (const result of lang.parseResults) {
         if (!result.success || !result.actionSignature) continue;
@@ -294,6 +307,9 @@ export class TestOrchestrator {
 
         // R0-precision — fraction of THIS parse's actions justified by the en
         // multiset reference (catches phantom/spurious commands recall misses).
+        // R0-recall-multiset — the mirror: fraction of the en reference's actions,
+        // counting duplicates, present here (catches a DROPPED repeated command,
+        // which the Set-based fidelity/roleFidelity above cannot see).
         const multisetRef = multisetReference.get(result.pattern.codeExampleId);
         if (multisetRef && result.actionMultisetSignature) {
           const precision = computePrecision(multisetRef, result.actionMultisetSignature);
@@ -301,6 +317,11 @@ export class TestOrchestrator {
             result.precision = precision;
             precisionScores.push(precision);
           }
+          const multisetRecall = computeMultisetRecall(multisetRef, result.actionMultisetSignature);
+          if (multisetRecall !== undefined) {
+            result.multisetRecall = multisetRecall;
+            multisetRecallScores.push(multisetRecall);
+          }
         }
 
         // R1 — role recall vs the en role signature (role name + value type).
@@ -312,6 +333,23 @@ export class TestOrchestrator {
             roleScores.push(roleFidelity);
           }
         }
+
+        // R3 — invariant role-VALUE recall vs the en value signature (multiset).
+        // Values are compared verbatim: the filtered subset (selectors, sigil
+        // refs, time literals, colon-qualified events, URLs) is code, not prose.
+        const valueRef = roleValueReference.get(result.pattern.codeExampleId);
+        if (valueRef && result.roleValueSignature) {
+          const valueRecall = computeMultisetRecall(valueRef, result.roleValueSignature);
+          if (valueRecall !== undefined) {
+            result.valueRecall = valueRecall;
+            valueRecallScores.push(valueRecall);
+            if (valueRecall < 1) {
+              // Missing = reference entries not covered here (multiset diff;
+              // spuriousActions with the arguments swapped).
+              result.valueRecallMissing = spuriousActions(result.roleValueSignature, valueRef);
+            }
+          }
+        }
       }
 
       lang.avgFidelity =
@@ -320,10 +358,18 @@ export class TestOrchestrator {
         precisionScores.length > 0
           ? precisionScores.reduce((a, b) => a + b, 0) / precisionScores.length
           : undefined;
+      lang.avgMultisetRecall =
+        multisetRecallScores.length > 0
+          ? multisetRecallScores.reduce((a, b) => a + b, 0) / multisetRecallScores.length
+          : undefined;
       lang.avgRoleFidelity =
         roleScores.length > 0
           ? roleScores.reduce((a, b) => a + b, 0) / roleScores.length
           : undefined;
+      lang.avgValueRecall =
+        valueRecallScores.length > 0
+          ? valueRecallScores.reduce((a, b) => a + b, 0) / valueRecallScores.length
+          : undefined;
       lang.degeneratePasses = degenerate.sort();
       lang.lossyPasses = lossy.sort();
     }

package/src/multilingual/reporters/console-reporter.ts

@@ -158,11 +158,62 @@ export class ConsoleReporter implements Reporter {
     }
 
     this.reportDegeneratePasses(results);
+    this.reportValueRecall(results);
     this.reportExecutionFailures(results);
 
     this.log('');
   }
 
+  /**
+   * R3 — surface parses that lost/corrupted a language-invariant role VALUE
+   * (selector, sigil ref, time literal, colon-qualified event name, URL) vs
+   * the en reference. These score 1.0 on every action/type-based signal yet
+   * behave differently at runtime, so they're reported separately. Report-only:
+   * the ratchet gate lives in the --regression path.
+   */
+  private reportValueRecall(results: TestResults): void {
+    const scored = results.languageResults.filter(l => l.avgValueRecall !== undefined);
+    if (scored.length === 0) return;
+
+    const avg = scored.reduce((sum, l) => sum + (l.avgValueRecall ?? 0), 0) / scored.length;
+    this.log('');
+    this.log(
+      this.bright(`Role values (R3): avgValueRecall ${avg.toFixed(4)} over invariant values`)
+    );
+
+    // pattern id -> "lang: missing entries" rows
+    const byPattern = new Map<string, string[]>();
+    let instances = 0;
+    for (const lang of scored) {
+      for (const r of lang.parseResults) {
+        if (r.valueRecall === undefined || r.valueRecall >= 1) continue;
+        instances++;
+        let rows = byPattern.get(r.pattern.codeExampleId);
+        if (!rows) {
+          rows = [];
+          byPattern.set(r.pattern.codeExampleId, rows);
+        }
+        rows.push(`${lang.language}: missing ${(r.valueRecallMissing ?? []).join(', ')}`);
+      }
+    }
+    if (byPattern.size === 0) return;
+
+    this.log(
+      this.yellow(
+        `⚠ Invariant value loss: ${instances} instance(s) across ${byPattern.size} pattern(s)`
+      )
+    );
+    this.log(
+      this.dim(
+        '  (an action.role=value present in the en reference is absent — not parse failures)'
+      )
+    );
+    for (const [id, rows] of [...byPattern.entries()].sort((a, b) => b[1].length - a[1].length)) {
+      this.log(`  ${this.dim('-')} ${id}`);
+      for (const row of rows.sort()) this.log(`      ${this.dim(row)}`);
+    }
+  }
+
   /**
    * R2 — surface curated-subset patterns whose jsdom execution diverged from
    * the en reference's DOM effects. These can score 1.0 on parse fidelity yet

package/src/multilingual/reporters/regression-reporter.ts

@@ -118,12 +118,25 @@ export class RegressionReporter implements Reporter {
         langResult.avgPrecision !== undefined && baselineLang.avgPrecision !== undefined
           ? langResult.avgPrecision - baselineLang.avgPrecision
           : 0;
+      // R0-recall-multiset — same both-sides guard. Negative = a repeated command
+      // is now being dropped, which avgFidelity (a Set) cannot see.
+      const avgMultisetRecallDelta =
+        langResult.avgMultisetRecall !== undefined && baselineLang.avgMultisetRecall !== undefined
+          ? langResult.avgMultisetRecall - baselineLang.avgMultisetRecall
+          : 0;
       // R1 — only meaningful when BOTH sides carry role data; an un-regenerated
       // baseline (no avgRoleFidelity yet) must never retro-flag.
       const avgRoleFidelityDelta =
         langResult.avgRoleFidelity !== undefined && baselineLang.avgRoleFidelity !== undefined
           ? langResult.avgRoleFidelity - baselineLang.avgRoleFidelity
           : 0;
+      // R3 — same both-sides guard. Negative = a language-invariant role VALUE
+      // (selector/sigil/time/colon-event/URL) is now lost or corrupted, which
+      // every action/type-based signal scores as perfect.
+      const avgValueRecallDelta =
+        langResult.avgValueRecall !== undefined && baselineLang.avgValueRecall !== undefined
+          ? langResult.avgValueRecall - baselineLang.avgValueRecall
+          : 0;
       // R2 — same both-sides guard: an un-regenerated baseline (no execution
       // data yet) must never retro-flag.
       const avgExecutionFidelityDelta =
@@ -163,7 +176,9 @@ export class RegressionReporter implements Reporter {
         avgConfidenceDelta,
         avgFidelityDelta,
         avgPrecisionDelta,
+        avgMultisetRecallDelta,
         avgRoleFidelityDelta,
+        avgValueRecallDelta,
         avgExecutionFidelityDelta,
         bundleSizeDelta: bundleSizeDelta !== undefined ? bundleSizeDelta : undefined,
         newFailures,
@@ -364,9 +379,17 @@ export class RegressionReporter implements Reporter {
         // R0-precision — fraction of each parse's actions justified by the en
         // reference (phantom-command signal recall can't see). Recorded + ratcheted.
         avgPrecision: langResult.avgPrecision ?? undefined,
+        // R0-recall-multiset — the mirror of precision: a DROPPED repeated command
+        // (`[bind, bind]` parsed as `[bind]`) is invisible to the Set-based
+        // avgFidelity and avgRoleFidelity. Recorded + ratcheted.
+        avgMultisetRecall: langResult.avgMultisetRecall ?? undefined,
         // R1 — role fidelity (role name + value type vs the en reference).
         // Recorded + ratcheted; burn-down is NOT part of the parsing-track goal.
         avgRoleFidelity: langResult.avgRoleFidelity ?? undefined,
+        // R3 — invariant role-VALUE recall (verbatim value comparison over the
+        // code-shaped subset: selectors, sigil refs, time literals,
+        // colon-qualified event names, URLs). Recorded + ratcheted.
+        avgValueRecall: langResult.avgValueRecall ?? undefined,
         // R2 — execution fidelity over the curated subset (DOM effects vs the
         // en reference in jsdom). Recorded + ratcheted; burn-down deferred.
         avgExecutionFidelity: langResult.avgExecutionFidelity ?? undefined,

package/src/multilingual/tools/diagnose-coverage.ts

@@ -0,0 +1,118 @@
+/**
+ * diagnose-coverage.ts
+ *
+ * Read-only sweep that counts how often the semantic parser matches a pattern
+ * covering only PART of its input — the `unconsumed-input` diagnostic.
+ *
+ * Why this exists: confidence is computed from role coverage (how many of the
+ * pattern's own roles were filled), never from input coverage. A short pattern
+ * can fill every role, score 1.0, and silently drop a trailing clause. Turning
+ * that into a scoring penalty would move the multilingual fidelity baseline
+ * across all 24 languages, so the firing rate has to be MEASURED first — on the
+ * real corpus, per language, with the dropped spans visible.
+ *
+ * This tool never writes a baseline and never gates. It only reads.
+ */
+import { parseSemantic } from '@lokascript/semantic';
+
+import { loadPatterns } from '../pattern-loader';
+import type { TestConfig } from '../types';
+
+interface LanguageTally {
+  language: string;
+  total: number;
+  fired: number;
+  /** A few representative dropped spans, for eyeballing false positives. */
+  samples: Array<{ source: string; message: string; confidence: number }>;
+}
+
+const MAX_SAMPLES_PER_LANGUAGE = 3;
+const UNCONSUMED_CODE = 'unconsumed-input';
+
+function unconsumedMessages(node: unknown): string[] {
+  const diagnostics = (node as { diagnostics?: Array<{ code?: string; message: string }> } | null)
+    ?.diagnostics;
+  if (!diagnostics?.length) return [];
+  return diagnostics.filter(d => d.code === UNCONSUMED_CODE).map(d => d.message);
+}
+
+/**
+ * Sweep the corpus and report the `unconsumed-input` firing rate per language.
+ *
+ * @returns the total number of firings (0 when the signal never trips)
+ */
+export async function diagnoseCoverage(config: TestConfig): Promise<number> {
+  const patterns = await loadPatterns(config);
+  const byLanguage = new Map<string, LanguageTally>();
+
+  for (const pattern of patterns) {
+    const tally = byLanguage.get(pattern.language) ?? {
+      language: pattern.language,
+      total: 0,
+      fired: 0,
+      samples: [],
+    };
+    tally.total++;
+
+    let messages: string[] = [];
+    let confidence = 0;
+    try {
+      const result = parseSemantic(pattern.hyperscript, pattern.language);
+      confidence = result.confidence;
+      messages = unconsumedMessages(result.node);
+    } catch {
+      // A parse failure is not an input-coverage problem — it is already visible
+      // to the parse-rate gate. Skip it rather than double-counting.
+    }
+
+    const firstMessage = messages[0];
+    if (firstMessage !== undefined) {
+      tally.fired++;
+      if (tally.samples.length < MAX_SAMPLES_PER_LANGUAGE) {
+        tally.samples.push({ source: pattern.hyperscript, message: firstMessage, confidence });
+      }
+    }
+    byLanguage.set(pattern.language, tally);
+  }
+
+  report([...byLanguage.values()].sort((a, b) => b.fired / b.total - a.fired / a.total));
+  return [...byLanguage.values()].reduce((sum, t) => sum + t.fired, 0);
+}
+
+function report(tallies: LanguageTally[]): void {
+  const total = tallies.reduce((s, t) => s + t.total, 0);
+  const fired = tallies.reduce((s, t) => s + t.fired, 0);
+
+  console.log('\nInput-coverage diagnostic — `unconsumed-input` firing rate');
+  console.log('(a fired row parsed successfully but ignored part of its source)\n');
+  console.log('  lang    fired /  total    rate');
+  console.log('  ─────────────────────────────────');
+  for (const t of tallies) {
+    const rate = t.total ? (t.fired / t.total) * 100 : 0;
+    console.log(
+      `  ${t.language.padEnd(6)} ${String(t.fired).padStart(5)} / ${String(t.total).padStart(6)}  ${rate.toFixed(1).padStart(6)}%`
+    );
+  }
+  console.log('  ─────────────────────────────────');
+  const overall = total ? (fired / total) * 100 : 0;
+  console.log(
+    `  ALL    ${String(fired).padStart(5)} / ${String(total).padStart(6)}  ${overall.toFixed(1).padStart(6)}%\n`
+  );
+
+  const withSamples = tallies.filter(t => t.samples.length > 0);
+  if (withSamples.length === 0) {
+    console.log('No unconsumed input anywhere in the corpus.\n');
+    return;
+  }
+
+  console.log('Sample dropped spans (check these for false positives — trailing');
+  console.log('particles and legitimately-optional tokens are expected residue):\n');
+  for (const t of withSamples) {
+    console.log(`  [${t.language}]`);
+    for (const s of t.samples) {
+      console.log(`    source (conf ${s.confidence.toFixed(2)}): ${s.source}`);
+      console.log(`    ${s.message}`);
+    }
+    console.log('');
+  }
+}

package/src/multilingual/tools/triage-r1.ts

@@ -0,0 +1,149 @@
+/**
+ * triage-r1.ts
+ *
+ * Read-only sweep that itemizes R1 role-fidelity misses per language: for every
+ * corpus pattern whose roleFidelity < 1, the `action.role:type` entries present
+ * in the en reference parse but absent from the translation's parse — clustered
+ * by entry so the dominant failure families are visible, with the entries the
+ * translation captured INSTEAD for the same action (the mistype pairing).
+ *
+ * Why this exists: avgRoleFidelity summarizes the SOV-six gap (qu ~0.95, the
+ * rest ~0.97) to one number per language, and the regression gate only reports
+ * DROPS. Planning an improvement arc needs the misses themselves — which roles,
+ * on which actions, in which patterns — measured with exactly the pipeline the
+ * gate scores (ParseValidator → fillSchemaDefaults → collectRoleSignature →
+ * Set recall vs the en reference), so a "fix" that moves the probe also moves
+ * the signal.
+ *
+ * This tool never writes a baseline and never gates. It only reads.
+ */
+import { computeFidelity } from '../fidelity';
+import { loadPatterns } from '../pattern-loader';
+import { ParseValidator } from '../validators/parse-validator';
+import type { ParseResult, TestConfig } from '../types';
+
+interface MissCluster {
+  /** The en-reference `action.role:type` entry the language failed to recall. */
+  entry: string;
+  /** Pattern ids where this entry was missed. */
+  patterns: string[];
+  /**
+   * What the language's parse captured for the SAME action in those patterns
+   * (entries absent from the en reference) — usually the mistype the miss
+   * paired with (`trigger.patient:literal` opposite a missing
+   * `trigger.event:literal`). Empty when the whole command dropped.
+   */
+  haveInstead: Set<string>;
+}
+
+const MAX_PATTERN_IDS_SHOWN = 6;
+
+/**
+ * Sweep the corpus and itemize R1 misses for the requested languages
+ * (default: every non-en language in the run).
+ */
+export async function triageR1(config: TestConfig): Promise<void> {
+  // The en rows ARE the reference — force-load them even when the caller
+  // filtered --languages to the triage targets.
+  const requested = config.languages?.filter(l => l !== 'en');
+  const loadConfig: TestConfig =
+    requested && requested.length > 0
+      ? { ...config, languages: [...requested, 'en' as const] }
+      : config;
+
+  const patterns = await loadPatterns(loadConfig);
+  const validator = new ParseValidator();
+  const results = await validator.validate(patterns);
+
+  const reference = new Map<string, string[]>();
+  for (const r of results) {
+    if (r.pattern.language !== 'en') continue;
+    if (r.success && r.roleSignature && r.roleSignature.length > 0) {
+      reference.set(r.pattern.codeExampleId, r.roleSignature);
+    }
+  }
+
+  const byLanguage = new Map<string, ParseResult[]>();
+  for (const r of results) {
+    if (r.pattern.language === 'en') continue;
+    if (requested && requested.length > 0 && !requested.includes(r.pattern.language)) continue;
+    const list = byLanguage.get(r.pattern.language) ?? [];
+    list.push(r);
+    byLanguage.set(r.pattern.language, list);
+  }
+
+  console.log('\nR1 role-fidelity triage (action.role:type recall vs the en reference)');
+  console.log('━'.repeat(72));
+
+  for (const [language, langResults] of [...byLanguage.entries()].sort()) {
+    const clusters = new Map<string, MissCluster>();
+    const scores: number[] = [];
+    let patternsWithMisses = 0;
+
+    for (const result of langResults) {
+      if (!result.success || !result.roleSignature) continue;
+      const ref = reference.get(result.pattern.codeExampleId);
+      if (!ref) continue;
+
+      const fidelity = computeFidelity(ref, result.roleSignature);
+      if (fidelity === undefined) continue;
+      scores.push(fidelity);
+      if (fidelity >= 1) continue;
+
+      patternsWithMisses++;
+      const candidate = new Set(result.roleSignature);
+      const refSet = new Set(ref);
+      const missing = ref.filter(e => !candidate.has(e));
+      // Entries this parse has that the reference doesn't — candidates for the
+      // "captured instead" pairing, keyed by action.
+      const extrasByAction = new Map<string, string[]>();
+      for (const e of result.roleSignature) {
+        if (refSet.has(e)) continue;
+        const action = e.slice(0, e.indexOf('.'));
+        const list = extrasByAction.get(action) ?? [];
+        list.push(e);
+        extrasByAction.set(action, list);
+      }
+
+      for (const entry of missing) {
+        const cluster = clusters.get(entry) ?? {
+          entry,
+          patterns: [],
+          haveInstead: new Set<string>(),
+        };
+        cluster.patterns.push(result.pattern.codeExampleId);
+        const action = entry.slice(0, entry.indexOf('.'));
+        for (const e of extrasByAction.get(action) ?? []) cluster.haveInstead.add(e);
+        clusters.set(entry, cluster);
+      }
+    }
+
+    const avg = scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : undefined;
+    const totalMisses = [...clusters.values()].reduce((a, c) => a + c.patterns.length, 0);
+
+    console.log(
+      `\n[${language}] avgRoleFidelity ${avg?.toFixed(4) ?? 'n/a'} — ` +
+        `${totalMisses} miss(es) across ${patternsWithMisses} pattern(s), ` +
+        `${clusters.size} distinct entry(ies)`
+    );
+
+    const sorted = [...clusters.values()].sort(
+      (a, b) => b.patterns.length - a.patterns.length || a.entry.localeCompare(b.entry)
+    );
+    for (const cluster of sorted) {
+      const ids =
+        cluster.patterns.length > MAX_PATTERN_IDS_SHOWN
+          ? `${cluster.patterns.slice(0, MAX_PATTERN_IDS_SHOWN).join(', ')}, +${
+              cluster.patterns.length - MAX_PATTERN_IDS_SHOWN
+            } more`
+          : cluster.patterns.join(', ');
+      console.log(`  ×${cluster.patterns.length}  missing ${cluster.entry}`);
+      console.log(`        in: ${ids}`);
+      if (cluster.haveInstead.size > 0) {
+        console.log(`        captured instead: ${[...cluster.haveInstead].sort().join(', ')}`);
+      }
+    }
+  }
+
+  console.log();
+}