@hyperfixi/testing-framework 2.7.2 → 2.8.0

package/src/multilingual/cli.ts

@@ -10,6 +10,8 @@ import * as path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { checkDbStamp, getDefaultDbPath } from '@hyperfixi/patterns-reference';
 import { TestOrchestrator } from './orchestrator';
+import { diagnoseCoverage } from './tools/diagnose-coverage';
+import { triageR1 } from './tools/triage-r1';
 import type { TestConfig, LanguageCode } from './types';
 
 /**
@@ -172,6 +174,17 @@ function parseArgs(): TestConfig {
         config.saveBaseline = true;
         break;
 
+      case '--diagnose-coverage':
+        // Read-only measurement pass; short-circuits main() before any gate.
+        config.diagnoseCoverage = true;
+        break;
+
+      case '--triage-r1':
+        // Read-only measurement pass; short-circuits main() before any gate.
+        // Combine with --languages to scope (en is force-loaded as reference).
+        config.triageR1 = true;
+        break;
+
       default:
         if (arg && arg.startsWith('-')) {
           console.error(`Unknown option: ${arg}`);
@@ -227,6 +240,15 @@ OPTIONS:
       --categories <cats>      Filter by categories (comma-separated)
       --limit <n>              Patterns per language in quick mode (default: 10)
       --save-baseline          Save current results as new baseline
+      --triage-r1              Itemize R1 role-fidelity misses per language
+                               (missing action.role:type entries vs the en
+                               reference, clustered; scope with --languages)
+      --diagnose-coverage      Report how often the semantic parser matched a
+                               pattern that ignored part of its input, per
+                               language, with sample dropped spans. Read-only:
+                               never gates and never writes a baseline. Run it
+                               before considering an input-coverage penalty in
+                               the confidence model.
   -h, --help                   Show this help message
 
 EXAMPLES:
@@ -257,6 +279,39 @@ async function main(): Promise<void> {
     // --save-baseline needs the regression reporter wired so it can persist.
     if (config.saveBaseline) config.regression = true;
 
+    // --diagnose-coverage is a measurement pass, not a gate: it reads the corpus,
+    // parses each row, and reports the `unconsumed-input` firing rate. It compares
+    // nothing against the baseline and writes nothing, so it runs before (and
+    // instead of) the regression machinery. It DOES execute dist/, so a stale
+    // build would mis-measure — warn rather than refuse, since no baseline is at
+    // risk.
+    if (config.diagnoseCoverage) {
+      const staleDists = findStaleDists();
+      if (staleDists.length > 0) {
+        console.warn(
+          `⚠ Stale dist/ in: ${staleDists.join(', ')} — these numbers describe the built\n` +
+            '  output, not your checkout. Rebuild first: npm run test:multilingual:build-deps\n'
+        );
+      }
+      await diagnoseCoverage(config);
+      process.exit(0);
+    }
+
+    // --triage-r1 is the same kind of measurement pass for R1 role fidelity:
+    // itemizes each language's missing `action.role:type` entries vs the en
+    // reference, clustered by entry (with the mistype pairing). Read-only.
+    if (config.triageR1) {
+      const staleDists = findStaleDists();
+      if (staleDists.length > 0) {
+        console.warn(
+          `⚠ Stale dist/ in: ${staleDists.join(', ')} — these numbers describe the built\n` +
+            '  output, not your checkout. Rebuild first: npm run test:multilingual:build-deps\n'
+        );
+      }
+      await triageR1(config);
+      process.exit(0);
+    }
+
     // DB freshness guard: refuse to run a regression/baseline compare against a
     // patterns.db generated from *different* source than is currently checked out
     // (the cross-branch "phantom regression" footgun). The committed baseline only
@@ -387,6 +442,19 @@ async function main(): Promise<void> {
           r => r.avgPrecisionDelta < -AVG_PRECISION_DROP_TOLERANCE
         );
 
+        // R0-recall-multiset ratchet: the mirror of the precision ratchet. Every
+        // signal above is computed on a deduped SET, so a parse that drops a
+        // REPEATED command scores a perfect 1.0 — reference `[bind, bind]`
+        // collapses to `{bind}`, which `[bind]` satisfies in full. That is exactly
+        // how `bind-two-way` recorded fidelity 1.0 in all 24 languages while every
+        // one of them parsed only the first of its two binds. Counting duplicates
+        // makes the drop visible. Deltas are 0 unless the baseline carries
+        // avgMultisetRecall, so an un-regenerated baseline never retro-flags.
+        const AVG_MULTISET_RECALL_DROP_TOLERANCE = 0.02;
+        const multisetRecallDrops = allResults.filter(
+          r => r.avgMultisetRecallDelta < -AVG_MULTISET_RECALL_DROP_TOLERANCE
+        );
+
         // R1 — role-fidelity ratchet (§8): same semantics as the avgFidelity
         // ratchet, on the role-recall signal (action.role:valueType vs the en
         // reference). Deltas are 0 unless the baseline carries avgRoleFidelity,
@@ -398,6 +466,20 @@ async function main(): Promise<void> {
           r => r.avgRoleFidelityDelta < -AVG_ROLE_FIDELITY_DROP_TOLERANCE
         );
 
+        // R3 — role-VALUE ratchet: same semantics as the role-fidelity ratchet,
+        // on the invariant-value signal (`action.role=value` multiset over the
+        // code-shaped subset: selectors, sigil refs, time literals,
+        // colon-qualified event names, URLs — compared VERBATIM vs the en
+        // reference). A drop means a translation started losing/corrupting an
+        // invariant value (`draggable` captured for `draggable:start`, the #633
+        // class) — invisible to every action/type-based signal above. Deltas
+        // are 0 unless the baseline carries avgValueRecall, so an
+        // un-regenerated baseline never retro-flags.
+        const AVG_VALUE_RECALL_DROP_TOLERANCE = 0.02;
+        const valueRecallDrops = allResults.filter(
+          r => r.avgValueRecallDelta < -AVG_VALUE_RECALL_DROP_TOLERANCE
+        );
+
         // R2 — execution ratchet (§8): curated-subset patterns whose jsdom DOM
         // effects matched the en reference in the baseline but diverge now.
         // Tolerance 0: execution is binary and the harness is deterministic
@@ -408,6 +490,55 @@ async function main(): Promise<void> {
           r.newExecutionFailures.map(id => `${r.language}/${id}`)
         );
 
+        // R4 — canonical-validity ratchet: render every authored foreign
+        // translation to English and parse the result on the real
+        // hyperscript.org engine, diffing the invalid (pattern, language)
+        // pairs against the committed allowlist
+        // (baselines/foreign-canonical-validity.json). Both directions fail:
+        // a NEW invalid pair is a validity regression; a stale entry (an
+        // allowlisted pair that now renders valid) must be pruned in the same
+        // change that cleared it (tools/regen-foreign-baseline.ts). Tolerance
+        // 0 — the render+parse is deterministic against a fresh DB, and the
+        // DB/dist freshness guards above already refused stale inputs. Reuses
+        // checkForeignRenderValidity + the allowlist VERBATIM, so this and the
+        // vitest gate (foreign-canonical-validity.test.ts) cannot disagree.
+        // Full-mode only (quick mode keeps its speed contract) and skipped
+        // with a warning when the allowlist is absent, so a checkout without
+        // the baseline never retro-flags.
+        let validityNewInvalid: string[] = [];
+        let validityStale: string[] = [];
+        let validityChecked = false;
+        if (config.mode !== 'quick') {
+          const validityBaselinePath = path.resolve(
+            path.dirname(fileURLToPath(import.meta.url)),
+            '../../baselines/foreign-canonical-validity.json'
+          );
+          if (!fs.existsSync(validityBaselinePath)) {
+            console.warn(`⚠ R4 validity ratchet skipped: no allowlist at ${validityBaselinePath}.`);
+          } else {
+            const { checkForeignRenderValidity, groupFailuresByPattern } =
+              await import('./foreign-canonical-validity');
+            const allowlist = JSON.parse(fs.readFileSync(validityBaselinePath, 'utf8')) as {
+              allowedInvalid: Record<string, string[]>;
+            };
+            const pairKey = (id: string, lang: string) => `${id}/${lang}`;
+            const allowed = new Set(
+              Object.entries(allowlist.allowedInvalid).flatMap(([id, langs]) =>
+                langs.map(l => pairKey(id, l))
+              )
+            );
+            const validity = await checkForeignRenderValidity();
+            const current = new Set(
+              Object.entries(groupFailuresByPattern(validity.failures)).flatMap(([id, langs]) =>
+                langs.map(l => pairKey(id, l))
+              )
+            );
+            validityNewInvalid = [...current].filter(p => !allowed.has(p)).sort();
+            validityStale = [...allowed].filter(p => !current.has(p)).sort();
+            validityChecked = true;
+          }
+        }
+
         let failed = false;
 
         if (regressed.length > 0) {
@@ -485,6 +616,21 @@ async function main(): Promise<void> {
           failed = true;
         }
 
+        if (multisetRecallDrops.length > 0) {
+          console.error(
+            `\n✗ avgMultisetRecall dropped > ${AVG_MULTISET_RECALL_DROP_TOLERANCE} in ` +
+              `${multisetRecallDrops.length} language(s) — a parse started dropping a ` +
+              `REPEATED command (invisible to the Set-based fidelity/roleFidelity):`
+          );
+          for (const r of multisetRecallDrops) {
+            console.error(
+              `   ${r.language}: ΔavgMultisetRecall ${r.avgMultisetRecallDelta.toFixed(4)}`
+            );
+          }
+          console.error(`   (if intentional, regenerate the baseline with --save-baseline)`);
+          failed = true;
+        }
+
         if (roleFidelityDrops.length > 0) {
           console.error(
             `\n✗ avgRoleFidelity dropped > ${AVG_ROLE_FIDELITY_DROP_TOLERANCE} in ` +
@@ -499,6 +645,19 @@ async function main(): Promise<void> {
           failed = true;
         }
 
+        if (valueRecallDrops.length > 0) {
+          console.error(
+            `\n✗ avgValueRecall dropped > ${AVG_VALUE_RECALL_DROP_TOLERANCE} in ` +
+              `${valueRecallDrops.length} language(s) — a parse started losing/corrupting ` +
+              `a language-invariant role VALUE (invisible to the action/type-based signals):`
+          );
+          for (const r of valueRecallDrops) {
+            console.error(`   ${r.language}: ΔavgValueRecall ${r.avgValueRecallDelta.toFixed(4)}`);
+          }
+          console.error(`   (if intentional, regenerate the baseline with --save-baseline)`);
+          failed = true;
+        }
+
         if (executionRegressions.length > 0) {
           console.error(
             `\n✗ Execution regression vs baseline (R2): ${executionRegressions.length} ` +
@@ -512,12 +671,37 @@ async function main(): Promise<void> {
           failed = true;
         }
 
+        if (validityNewInvalid.length > 0) {
+          console.error(
+            `\n✗ Canonical-validity regression (R4): ${validityNewInvalid.length} foreign ` +
+              `translation(s) now render English the hyperscript.org parser rejects:`
+          );
+          for (const p of validityNewInvalid) console.error(`   ${p}`);
+          console.error(
+            `   (triage with tools/triage-foreign-residual.ts; if the invalidity is ` +
+              `expected, allowlist it via tools/regen-foreign-baseline.ts)`
+          );
+          failed = true;
+        }
+
+        if (validityStale.length > 0) {
+          console.error(
+            `\n✗ Stale validity allowlist (R4): ${validityStale.length} allowlisted ` +
+              `pair(s) now render VALID — prune with tools/regen-foreign-baseline.ts:`
+          );
+          for (const p of validityStale) console.error(`   ${p}`);
+          failed = true;
+        }
+
         if (failed) {
           exitCode = 1;
         } else {
           console.log(
             `\n✓ No regression vs baseline ` +
-              `(parse-rate ${REGRESSION_TOLERANCE_PTS}pts, fidelity + correctness + execution ratchets).`
+              `(parse-rate ${REGRESSION_TOLERANCE_PTS}pts, fidelity + correctness + ` +
+              `precision + multiset-recall + role + value + execution ratchets` +
+              (validityChecked ? ` + canonical validity (R4)` : '') +
+              `).`
           );
           exitCode = 0;
         }

package/src/multilingual/fidelity.test.ts

@@ -2,7 +2,9 @@ import { describe, it, expect } from 'vitest';
 import {
   collectActions,
   collectActionsMultiset,
+  collectRoleValueSignature,
   computeFidelity,
+  computeMultisetRecall,
   computePrecision,
   spuriousActions,
   FIDELITY_THRESHOLD,
@@ -119,6 +121,196 @@ describe('computePrecision', () => {
   });
 });
 
+describe('computeMultisetRecall', () => {
+  it('catches the dropped duplicate that Set-based recall is blind to', () => {
+    // The `bind-two-way` shape. EN `bind $n to #a bind $n to #b` is [bind, bind];
+    // a parse that truncates to the first command yields [bind]. Every Set-based
+    // signal reads this as perfect — which is why the row recorded fidelity 1.0
+    // across all 24 languages while every one of them dropped half its body.
+    const en = ['bind', 'bind'];
+    const truncated = ['bind'];
+
+    expect(computeFidelity(collectSet(en), collectSet(truncated))).toBe(1); // recall is fooled
+    expect(computePrecision(en, truncated)).toBe(1); // precision is fooled too
+    expect(computeMultisetRecall(en, truncated)).toBe(0.5); // this is not
+  });
+
+  it('scores a faithful (reordered) parse 1.0', () => {
+    expect(computeMultisetRecall(['bind', 'bind'], ['bind', 'bind'])).toBe(1);
+    expect(computeMultisetRecall(['add', 'on', 'remove'], ['on', 'remove', 'add'])).toBe(1);
+  });
+
+  it('is not fooled by an added duplicate (that is precision’s job)', () => {
+    // A candidate with a phantom extra still has full recall; precision catches it.
+    expect(computeMultisetRecall(['bind'], ['bind', 'bind'])).toBe(1);
+    expect(computePrecision(['bind'], ['bind', 'bind'])).toBe(0.5);
+  });
+
+  it('returns undefined when there is no reference to score against', () => {
+    expect(computeMultisetRecall([], ['bind'])).toBeUndefined();
+  });
+});
+
+/** The deduped Set signature `collectActions` produces, from a multiset. */
+function collectSet(actions: readonly string[]): string[] {
+  return [...new Set(actions)].sort();
+}
+
+describe('collectRoleValueSignature', () => {
+  it('emits every invariant-shaped value class, from a roles Map', () => {
+    const node = {
+      kind: 'event-handler',
+      action: 'on',
+      roles: new Map<string, unknown>([
+        ['event', { type: 'expression', raw: 'draggable:start' }], // colon-qualified (the #633 class)
+      ]),
+      body: [
+        {
+          kind: 'command',
+          action: 'toggle',
+          roles: new Map<string, unknown>([
+            ['patient', { type: 'selector', value: '.active', selectorKind: 'class' }],
+          ]),
+        },
+        {
+          kind: 'command',
+          action: 'set',
+          roles: new Map<string, unknown>([
+            ['destination', { type: 'expression', raw: ':x' }], // sigil ref
+          ]),
+        },
+        {
+          kind: 'command',
+          action: 'wait',
+          roles: new Map<string, unknown>([
+            ['duration', { type: 'literal', value: '200ms', dataType: 'duration' }],
+          ]),
+        },
+        {
+          kind: 'command',
+          action: 'fetch',
+          roles: new Map<string, unknown>([['source', { type: 'literal', value: '/api/data' }]]),
+        },
+      ],
+    };
+    expect(collectRoleValueSignature(node)).toEqual([
+      'fetch.source=/api/data',
+      'on.event=draggable:start',
+      'set.destination=:x',
+      'toggle.patient=.active',
+      'wait.duration=200ms',
+    ]);
+  });
+
+  it('excludes non-invariant values: references, prose, bare words, mixed raws, flags, property-paths', () => {
+    const node = {
+      kind: 'command',
+      action: 'put',
+      roles: new Map<string, unknown>([
+        ['destination', { type: 'reference', value: 'me' }], // fillSchemaDefaults noise
+        ['patient', { type: 'literal', value: 'Hello world', dataType: 'string' }], // legitimately translated
+        ['source', { type: 'expression', raw: 'startX' }], // bare identifier — v1 exclusion
+        ['modifier', { type: 'expression', raw: '次 .item' }], // native words + code mixed
+        ['condition', { type: 'expression', raw: '#modal exists' }], // selector-prefixed prose (if-exists class)
+        ['url', { type: 'literal', value: '/api/search?q=${my' }], // truncated template interpolation
+        ['flagRole', { type: 'flag', name: 'async', enabled: true }],
+        [
+          'pathRole',
+          { type: 'property-path', object: { type: 'reference', value: 'me' }, property: 'value' },
+        ],
+      ]),
+    };
+    expect(collectRoleValueSignature(node)).toEqual([]);
+  });
+
+  it('is a multiset — a dropped duplicate value is visible via computeMultisetRecall', () => {
+    // The bind-two-way shape: two `bind`s to two different sigil refs; a
+    // truncating parse keeps only the first. Both entries share NO key with a
+    // Set — but if both bound the SAME ref, dedup would hide the drop:
+    const twoBinds = {
+      action: 'compound',
+      statements: [
+        {
+          action: 'bind',
+          roles: new Map<string, unknown>([['source', { type: 'expression', raw: '$name' }]]),
+        },
+        {
+          action: 'bind',
+          roles: new Map<string, unknown>([['source', { type: 'expression', raw: '$name' }]]),
+        },
+      ],
+    };
+    const oneBind = {
+      action: 'bind',
+      roles: new Map<string, unknown>([['source', { type: 'expression', raw: '$name' }]]),
+    };
+    const ref = collectRoleValueSignature(twoBinds);
+    const cand = collectRoleValueSignature(oneBind);
+    expect(ref).toEqual(['bind.source=$name', 'bind.source=$name']);
+    expect(cand).toEqual(['bind.source=$name']);
+    expect(computeMultisetRecall(ref, cand)).toBe(0.5);
+  });
+
+  it('recurses into behavior-shaped nodes (eventHandlers + initBlock)', () => {
+    // No other walker test exercises these two CHILD_FIELDS; ad-hoc walkers
+    // that omit them see behavior bodies as empty.
+    const behavior = {
+      kind: 'behavior',
+      action: 'behavior',
+      roles: new Map<string, unknown>([['name', { type: 'expression', raw: 'Draggable' }]]),
+      eventHandlers: [
+        {
+          kind: 'event-handler',
+          action: 'on',
+          roles: new Map<string, unknown>([
+            ['event', { type: 'literal', value: 'draggable:start' }],
+          ]),
+          body: [
+            {
+              kind: 'command',
+              action: 'add',
+              roles: new Map<string, unknown>([
+                ['patient', { type: 'selector', value: '.dragging' }],
+              ]),
+            },
+          ],
+        },
+      ],
+      initBlock: [
+        {
+          kind: 'command',
+          action: 'set',
+          roles: new Map<string, unknown>([['destination', { type: 'expression', raw: '*width' }]]),
+        },
+      ],
+    };
+    expect(collectRoleValueSignature(behavior)).toEqual([
+      'add.patient=.dragging',
+      'on.event=draggable:start',
+      'set.destination=*width',
+    ]);
+  });
+
+  it('reads plain-object roles (synthetic/JSON-shaped nodes) too', () => {
+    const node = {
+      action: 'toggle',
+      roles: { patient: { type: 'selector', value: '#count' } },
+    };
+    expect(collectRoleValueSignature(node)).toEqual(['toggle.patient=#count']);
+  });
+
+  it('skips the structural compound wrapper and handles non-object input', () => {
+    const node = {
+      action: 'compound',
+      roles: new Map<string, unknown>([['patient', { type: 'selector', value: '.x' }]]),
+      statements: [{ action: 'toggle', roles: { patient: { type: 'selector', value: '.x' } } }],
+    };
+    expect(collectRoleValueSignature(node)).toEqual(['toggle.patient=.x']);
+    expect(collectRoleValueSignature(null)).toEqual([]);
+    expect(collectRoleValueSignature(undefined)).toEqual([]);
+  });
+});
+
 describe('spuriousActions', () => {
   it('lists the hallucinated commands a render/parse introduced', () => {
     expect(spuriousActions(['add', 'on', 'remove'], ['add', 'on', 'remove', 'toggle'])).toEqual([

package/src/multilingual/fidelity.ts

@@ -121,6 +121,39 @@ export function computeFidelity(
   return hits / reference.length;
 }
 
+/**
+ * R0-recall on the **multiset** in [0, 1]: the fraction of the reference's
+ * actions — counting duplicates — also present in the candidate.
+ *
+ * {@link computeFidelity} scores the deduped Set signature, so a candidate that
+ * drops a REPEATED command scores 1.0: reference `[bind, bind]` collapses to
+ * `{bind}`, which `[bind]` satisfies in full. That is how `bind-two-way` sat at
+ * fidelity 1.0 across all 24 languages while every one of them parsed only the
+ * first of its two `bind`s. R1 (role signatures) is a Set too, and is equally
+ * blind. {@link computePrecision} catches the mirror case — a candidate that ADDS
+ * a duplicate — so before this signal existed the ratchet saw spurious commands
+ * but never dropped ones.
+ *
+ * Pass multisets (see {@link collectActionsMultiset}) on both sides.
+ * Returns `undefined` when the reference has no actions to compare against.
+ */
+export function computeMultisetRecall(
+  reference: readonly string[],
+  candidate: readonly string[]
+): number | undefined {
+  if (reference.length === 0) return undefined;
+  const cand = actionCounts(candidate);
+  let matched = 0;
+  for (const a of reference) {
+    const remaining = cand.get(a) ?? 0;
+    if (remaining > 0) {
+      matched++;
+      cand.set(a, remaining - 1);
+    }
+  }
+  return matched / reference.length;
+}
+
 /**
  * Structural **precision** in [0, 1]: the fraction of the *candidate's* actions
  * that are justified by the reference (multiset-aware). The complement of
@@ -222,3 +255,123 @@ function walkRoles(node: unknown, acc: Set<string>, depth: number): void {
     }
   }
 }
+
+/**
+ * Role-value kinds never emitted by the R3 walker. `reference` values (`me`,
+ * `it`, …) are mostly `fillSchemaDefaults` injections present identically on
+ * both sides — noise. `flag` names and `property-path` properties are bare
+ * identifiers, excluded by v1 (see {@link collectRoleValueSignature}).
+ */
+const VALUE_KIND_EXCLUSIONS = new Set(['reference', 'flag', 'property-path']);
+
+/**
+ * A role value is compared cross-language only when its WHOLE surface form is
+ * code-shaped — language-invariant by construction, never legitimately
+ * translated. Conservative v1 whitelist; when a sweep firing turns out to be a
+ * legit translation difference, tighten here and document the exclusion.
+ */
+const INVARIANT_VALUE_PATTERNS: readonly RegExp[] = [
+  /^[#.[<@*]/, // selectors: #id  .class  [attr]  <tag/>  @attr  *style
+  /^[:$^][A-Za-z_]\w*$/, // sigil refs: :local  $global  ^element
+  /^\d+(\.\d+)?(ms|s|m|h)?$/, // numbers and time literals: 2  1.5  200ms
+  /^[A-Za-z_][\w-]*:[\w-]+$/, // colon-qualified event names: draggable:start
+  /^(\.{0,2}\/|https?:)/, // URLs / paths: /api/data  ./x  ../y  https://…
+];
+
+function isInvariantSurface(surface: string): boolean {
+  // Whole-surface rule, sweep-validated exclusions:
+  // - whitespace ⇒ mixed content. `if #modal exists` captures its condition as
+  //   `#modal exists` — starts selector-shaped, but `exists` is prose that every
+  //   language legitimately translates (16-language false firing without this).
+  // - `${` ⇒ template interpolation. `/api/search?q=${my value}` is an
+  //   expression, and tokenizers split it at different points per language, so
+  //   the captured surface isn't comparable verbatim.
+  if (/\s/.test(surface) || surface.includes('${')) return false;
+  return INVARIANT_VALUE_PATTERNS.some(re => re.test(surface));
+}
+
+/**
+ * The comparable surface form of a role value, or `undefined` when the value
+ * carries none: `.value` for `literal`/`selector` (string/number/boolean
+ * coerced), `.raw` for `expression`. Kinds in {@link VALUE_KIND_EXCLUSIONS}
+ * and values without a string `type` discriminator yield `undefined`.
+ */
+function roleValueSurface(value: unknown): string | undefined {
+  if (value === null || typeof value !== 'object') return undefined;
+  const rec = value as { type?: unknown; value?: unknown; raw?: unknown };
+  if (typeof rec.type !== 'string' || VALUE_KIND_EXCLUSIONS.has(rec.type)) return undefined;
+  if (rec.type === 'expression') {
+    return typeof rec.raw === 'string' ? rec.raw : undefined;
+  }
+  const v = rec.value;
+  return typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean'
+    ? String(v)
+    : undefined;
+}
+
+/**
+ * R3 — role-VALUE signature (invariant values only, multiset).
+ *
+ * R0/R1 compare actions and role *types*; values are never compared because
+ * they are legitimately translated. That leaves a live defect class with zero
+ * signal: right action counts, right role types, wrong role VALUE — the #633
+ * class, where ms captured `trigger` events named `draggable` instead of
+ * `draggable:start` (correct multiset, correct `trigger.event:literal`
+ * signature, silently wrong runtime behavior), and 18 other languages carried
+ * the same corruption with no side-effect at all.
+ *
+ * The subset of values compared here is language-invariant by construction —
+ * code, not prose (see {@link INVARIANT_VALUE_PATTERNS}). Emits a **multiset**
+ * of `` `action.role=value` `` entries (duplicates preserved, sorted); score
+ * with {@link computeMultisetRecall} so a dropped duplicate value is visible.
+ *
+ * Deliberately EXCLUDED in v1: bare-word identifiers (`startX` — usually
+ * invariant, but property/variable names occasionally get localized in seeds),
+ * string literals (message strings are legitimately translated), expression
+ * raws mixing native words + code (`次 .item`), and `reference` values
+ * (`me`/`it` — mostly `fillSchemaDefaults` injections on both sides).
+ *
+ * Blind spot: recall fires when a *translation* loses/corrupts an invariant
+ * value. If the **en reference itself** corrupts a value, every language flags
+ * at once — a 24-language R3 firestorm on one pattern means "suspect the en
+ * parse first" (unlike R0, where en corruption moves nothing).
+ *
+ * The roles container is a ReadonlyMap on live nodes (serializes to {} in
+ * results.json), so collect at validation time, never from the JSON.
+ */
+export function collectRoleValueSignature(node: unknown): string[] {
+  const acc: string[] = [];
+  walkRoleValues(node, acc, 0);
+  return acc.sort();
+}
+
+function walkRoleValues(node: unknown, acc: string[], depth: number): void {
+  if (depth > 64 || node === null || typeof node !== 'object') return;
+
+  const rec = node as Record<string, unknown>;
+  const action = rec.action;
+  if (typeof action === 'string' && !STRUCTURAL_ACTIONS.has(action)) {
+    const roles = rec.roles;
+    const entries: Array<[unknown, unknown]> =
+      roles instanceof Map
+        ? [...roles.entries()]
+        : roles && typeof roles === 'object'
+          ? Object.entries(roles)
+          : [];
+    for (const [role, value] of entries) {
+      if (value === undefined || value === null) continue;
+      const surface = roleValueSurface(value);
+      if (surface === undefined || !isInvariantSurface(surface)) continue;
+      acc.push(`${action}.${String(role)}=${surface}`);
+    }
+  }
+
+  for (const field of CHILD_FIELDS) {
+    const child = rec[field];
+    if (Array.isArray(child)) {
+      for (const c of child) walkRoleValues(c, acc, depth + 1);
+    } else if (child && typeof child === 'object') {
+      walkRoleValues(child, acc, depth + 1);
+    }
+  }
+}