@hyperfixi/testing-framework 2.5.1 → 2.7.0

package/src/multilingual/fidelity.test.ts

@@ -0,0 +1,136 @@
+import { describe, it, expect } from 'vitest';
+import {
+  collectActions,
+  collectActionsMultiset,
+  computeFidelity,
+  computePrecision,
+  spuriousActions,
+  FIDELITY_THRESHOLD,
+} from './fidelity';
+
+describe('collectActions', () => {
+  it('collects distinct actions across a nested event-handler tree', () => {
+    // Shape of the English `focus-trap` parse: on { if ; focus ; halt }.
+    const node = {
+      kind: 'event-handler',
+      action: 'on',
+      body: [
+        {
+          kind: 'compound',
+          action: 'compound',
+          statements: [
+            { kind: 'command', action: 'if' },
+            { kind: 'command', action: 'focus' },
+            { kind: 'command', action: 'halt' },
+          ],
+        },
+      ],
+    };
+    expect(collectActions(node)).toEqual(['focus', 'halt', 'if', 'on']);
+  });
+
+  it('excludes the structural `compound` wrapper', () => {
+    const node = { action: 'compound', statements: [{ action: 'toggle' }] };
+    expect(collectActions(node)).toEqual(['toggle']);
+  });
+
+  it('handles a flat command and non-object input', () => {
+    expect(collectActions({ kind: 'command', action: 'toggle' })).toEqual(['toggle']);
+    expect(collectActions(null)).toEqual([]);
+    expect(collectActions(undefined)).toEqual([]);
+  });
+});
+
+describe('computeFidelity', () => {
+  const en = ['focus', 'halt', 'if', 'on'];
+
+  it('scores a faithful (reordered) parse 1.0', () => {
+    // SOV/VSO reorder keeps the same command set → full fidelity.
+    expect(computeFidelity(en, ['on', 'if', 'halt', 'focus'])).toBe(1);
+  });
+
+  it('scores a degenerate parse low (focus-trap dropping commands)', () => {
+    // ja `focus-trap` parsed as `on { from }` — only `on` survives of 4.
+    const score = computeFidelity(en, ['on', 'from']);
+    expect(score).toBeCloseTo(0.25);
+    expect(score! < FIDELITY_THRESHOLD).toBe(true);
+  });
+
+  it('is recall, not precision — extra candidate actions do not lower it', () => {
+    expect(computeFidelity(['toggle'], ['toggle', 'add', 'remove'])).toBe(1);
+  });
+
+  it('returns undefined when there is no reference to score against', () => {
+    expect(computeFidelity([], ['on'])).toBeUndefined();
+  });
+});
+
+describe('collectActionsMultiset', () => {
+  it('preserves duplicate actions the Set-based collector drops', () => {
+    // The ja/ko/tr round-trip of `on click toggle .open then put "hi" into #out`
+    // renders a phantom `toggle` ahead of the real one: [on, toggle, toggle, put].
+    const node = {
+      kind: 'event-handler',
+      action: 'on',
+      body: [
+        { kind: 'command', action: 'toggle' }, // phantom, injected by the renderer
+        { kind: 'command', action: 'toggle' }, // the real one
+        { kind: 'command', action: 'put' },
+      ],
+    };
+    expect(collectActionsMultiset(node)).toEqual(['on', 'put', 'toggle', 'toggle']);
+    // The existing Set-based collector cannot see the duplicate:
+    expect(collectActions(node)).toEqual(['on', 'put', 'toggle']);
+  });
+
+  it('still excludes the structural `compound` wrapper', () => {
+    const node = { action: 'compound', statements: [{ action: 'toggle' }, { action: 'toggle' }] };
+    expect(collectActionsMultiset(node)).toEqual(['toggle', 'toggle']);
+  });
+});
+
+describe('computePrecision', () => {
+  it('scores a faithful (reordered) parse 1.0', () => {
+    const en = ['focus', 'halt', 'if', 'on'];
+    expect(computePrecision(en, ['on', 'if', 'halt', 'focus'])).toBe(1);
+  });
+
+  it('catches the phantom `toggle` recall is blind to (the renderer bug)', () => {
+    // Real shape: EN `on click add .x then remove .y` round-tripped through ja
+    // renders as [on, toggle, add, remove] — recall stays 1.0, precision drops.
+    const en = ['add', 'on', 'remove'];
+    const ja = ['add', 'on', 'remove', 'toggle'];
+    expect(computeFidelity(en, ja)).toBe(1); // recall is fooled
+    expect(computePrecision(en, ja)).toBeCloseTo(3 / 4); // precision is not
+  });
+
+  it('penalizes a duplicated spurious action (multiset)', () => {
+    // [toggle, toggle, put] vs reference [put, toggle]: one toggle is spurious.
+    expect(computePrecision(['put', 'toggle'], ['put', 'toggle', 'toggle'])).toBeCloseTo(2 / 3);
+  });
+
+  it('scores 0 when every candidate action is spurious (ar/ru substitutive case)', () => {
+    // ar `on click if … end` collapsed to just a phantom [toggle].
+    expect(computePrecision(['if', 'on'], ['toggle'])).toBe(0);
+  });
+
+  it('returns undefined when there is no candidate to score', () => {
+    expect(computePrecision(['on'], [])).toBeUndefined();
+  });
+});
+
+describe('spuriousActions', () => {
+  it('lists the hallucinated commands a render/parse introduced', () => {
+    expect(spuriousActions(['add', 'on', 'remove'], ['add', 'on', 'remove', 'toggle'])).toEqual([
+      'toggle',
+    ]);
+  });
+
+  it('counts duplicates as spurious beyond the reference multiset', () => {
+    expect(spuriousActions(['put', 'toggle'], ['put', 'toggle', 'toggle'])).toEqual(['toggle']);
+  });
+
+  it('is empty for a faithful subset/reorder', () => {
+    expect(spuriousActions(['focus', 'halt', 'if', 'on'], ['on', 'if', 'focus'])).toEqual([]);
+  });
+});

package/src/multilingual/fidelity.ts

@@ -0,0 +1,224 @@
+/**
+ * Structural fidelity for multilingual parses.
+ *
+ * The parse-validator's success metric is "the parser returned a non-null node".
+ * That conflates a faithful parse with a *degenerate* one — a translated pattern
+ * can parse non-null while silently dropping most of the source's commands (e.g.
+ * `focus-trap` parses as a bare `if` or a stray `from` in several languages, with
+ * the `focus`/`halt`/condition lost). This module derives a lightweight
+ * structural signature from a parsed node and scores a translation's parse against
+ * the English reference parse, so those degenerate passes are visible.
+ *
+ * The signature is intentionally word-order agnostic: it is the *set* of command
+ * actions in the node tree, so a faithful SOV/VSO reorder scores 1.0 while a parse
+ * that loses commands scores low.
+ */
+
+/** Passes scoring below this are flagged as degenerate (lost >half the structure). */
+export const FIDELITY_THRESHOLD = 0.5;
+
+/** The structural `compound` wrapper is not a command; never counts as an action. */
+const STRUCTURAL_ACTIONS = new Set(['compound']);
+
+/** Node-array fields the walk recurses into (event/loop/conditional/behavior bodies). */
+const CHILD_FIELDS = [
+  'body',
+  'statements',
+  'thenBranch',
+  'elseBranch',
+  'branches',
+  'eventHandlers',
+  'initBlock',
+] as const;
+
+/**
+ * Collect the distinct command actions anywhere in a parsed semantic node tree
+ * (top-level action + nested body/statements/branches), excluding the structural
+ * `compound` wrapper. Returns a sorted array for stable comparison/serialization.
+ */
+export function collectActions(node: unknown): string[] {
+  const acc = new Set<string>();
+  walk(node, acc, 0);
+  return [...acc].sort();
+}
+
+function walk(node: unknown, acc: Set<string>, depth: number): void {
+  // Guard against pathological/cyclic structures.
+  if (depth > 64 || node === null || typeof node !== 'object') return;
+
+  const rec = node as Record<string, unknown>;
+  const action = rec.action;
+  if (typeof action === 'string' && !STRUCTURAL_ACTIONS.has(action)) {
+    acc.add(action);
+  }
+
+  for (const field of CHILD_FIELDS) {
+    const child = rec[field];
+    if (Array.isArray(child)) {
+      for (const c of child) walk(c, acc, depth + 1);
+    } else if (child && typeof child === 'object') {
+      walk(child, acc, depth + 1);
+    }
+  }
+}
+
+/**
+ * Like {@link collectActions} but **multiset**: duplicates are preserved (sorted,
+ * not deduped). Needed for precision — a *duplicate* spurious command (e.g. a
+ * renderer that injects a phantom `toggle` ahead of a real `toggle`, yielding
+ * `[toggle, toggle, put]`) is invisible to the Set-based `collectActions`.
+ *
+ * Kept as a parallel walk rather than refactoring `collectActions`: the latter
+ * feeds the committed regression baseline, so its traversal stays byte-identical.
+ */
+export function collectActionsMultiset(node: unknown): string[] {
+  const acc: string[] = [];
+  walkMultiset(node, acc, 0);
+  return acc.sort();
+}
+
+function walkMultiset(node: unknown, acc: string[], depth: number): void {
+  if (depth > 64 || node === null || typeof node !== 'object') return;
+
+  const rec = node as Record<string, unknown>;
+  const action = rec.action;
+  if (typeof action === 'string' && !STRUCTURAL_ACTIONS.has(action)) {
+    acc.push(action);
+  }
+
+  for (const field of CHILD_FIELDS) {
+    const child = rec[field];
+    if (Array.isArray(child)) {
+      for (const c of child) walkMultiset(c, acc, depth + 1);
+    } else if (child && typeof child === 'object') {
+      walkMultiset(child, acc, depth + 1);
+    }
+  }
+}
+
+/** Build a multiset count map from an action list. */
+function actionCounts(items: readonly string[]): Map<string, number> {
+  const m = new Map<string, number>();
+  for (const it of items) m.set(it, (m.get(it) ?? 0) + 1);
+  return m;
+}
+
+/**
+ * Structural fidelity in [0, 1]: the fraction of the reference (English) actions
+ * also present in the candidate (recall). Returns `undefined` when the reference
+ * has no actions to compare against.
+ */
+export function computeFidelity(
+  reference: readonly string[],
+  candidate: readonly string[]
+): number | undefined {
+  if (reference.length === 0) return undefined;
+  const cand = new Set(candidate);
+  let hits = 0;
+  for (const a of reference) {
+    if (cand.has(a)) hits++;
+  }
+  return hits / reference.length;
+}
+
+/**
+ * Structural **precision** in [0, 1]: the fraction of the *candidate's* actions
+ * that are justified by the reference (multiset-aware). The complement of
+ * {@link computeFidelity}'s recall — it falls below 1.0 when a parse/render adds
+ * commands the source never had.
+ *
+ * This is the signal recall + role-fidelity (R1) cannot see: a renderer that
+ * injects a phantom `toggle` (ja/ko/tr/ar/ru event-handler rendering) keeps recall
+ * 1.0 while precision drops. Pass multisets (see {@link collectActionsMultiset})
+ * so a *duplicated* spurious action is counted, not absorbed.
+ *
+ * Returns `undefined` when the candidate has no actions to score.
+ */
+export function computePrecision(
+  reference: readonly string[],
+  candidate: readonly string[]
+): number | undefined {
+  if (candidate.length === 0) return undefined;
+  const ref = actionCounts(reference);
+  let matched = 0;
+  for (const a of candidate) {
+    const remaining = ref.get(a) ?? 0;
+    if (remaining > 0) {
+      matched++;
+      ref.set(a, remaining - 1);
+    }
+  }
+  return matched / candidate.length;
+}
+
+/**
+ * The candidate actions **not** justified by the reference (multiset difference,
+ * sorted) — i.e. the spurious/hallucinated commands a parse or render introduced.
+ * Empty when the candidate is a structural subset of the reference. Use for
+ * diagnostics ("which command was hallucinated?"), complementing the
+ * {@link computePrecision} score.
+ */
+export function spuriousActions(
+  reference: readonly string[],
+  candidate: readonly string[]
+): string[] {
+  const ref = actionCounts(reference);
+  const extras: string[] = [];
+  for (const a of candidate) {
+    const remaining = ref.get(a) ?? 0;
+    if (remaining > 0) ref.set(a, remaining - 1);
+    else extras.push(a);
+  }
+  return extras.sort();
+}
+
+/**
+ * R1 — role-fidelity signature.
+ *
+ * Action-set fidelity cannot see a parse that finds the right commands with the
+ * WRONG roles: a swapped patient/destination executes wrongly while scoring 1.0.
+ * This signature captures, for every command node in the tree, which roles were
+ * filled and with what value *type* (`add.patient:selector`,
+ * `put.destination:reference`). Cross-language comparison is by role name +
+ * value type — never by value string, which is legitimately translated.
+ * The roles container is a ReadonlyMap on live nodes (serializes to {} in JSON),
+ * so the signature must be collected at validation time, not from results.json.
+ */
+export function collectRoleSignature(node: unknown): string[] {
+  const acc = new Set<string>();
+  walkRoles(node, acc, 0);
+  return [...acc].sort();
+}
+
+function walkRoles(node: unknown, acc: Set<string>, depth: number): void {
+  if (depth > 64 || node === null || typeof node !== 'object') return;
+
+  const rec = node as Record<string, unknown>;
+  const action = rec.action;
+  if (typeof action === 'string' && !STRUCTURAL_ACTIONS.has(action)) {
+    const roles = rec.roles;
+    const entries: Array<[unknown, unknown]> =
+      roles instanceof Map
+        ? [...roles.entries()]
+        : roles && typeof roles === 'object'
+          ? Object.entries(roles)
+          : [];
+    for (const [role, value] of entries) {
+      if (value === undefined || value === null) continue;
+      const kind =
+        typeof value === 'object' && typeof (value as { type?: unknown }).type === 'string'
+          ? (value as { type: string }).type
+          : typeof value;
+      acc.add(`${action}.${String(role)}:${kind}`);
+    }
+  }
+
+  for (const field of CHILD_FIELDS) {
+    const child = rec[field];
+    if (Array.isArray(child)) {
+      for (const c of child) walkRoles(c, acc, depth + 1);
+    } else if (child && typeof child === 'object') {
+      walkRoles(child, acc, depth + 1);
+    }
+  }
+}

package/src/multilingual/html-pattern.test.ts

@@ -0,0 +1,39 @@
+/**
+ * Pattern Loader Tests
+ *
+ * Covers isHtmlMarkupPattern: the guard that keeps HTML-markup patterns out of
+ * the semantic-parse denominator (they're validated by the DOM/Playwright
+ * suites, not the hyperscript text parser).
+ */
+import { describe, it, expect } from 'vitest';
+import { isHtmlMarkupPattern } from './html-pattern';
+
+describe('isHtmlMarkupPattern', () => {
+  it('flags HTML-markup patterns (excluded from semantic parsing)', () => {
+    const htmlPatterns = [
+      '<div hx-live="put $count into me"></div>',
+      '<div sse-connect="/events" sse-swap="tick" hx-target="#feed"></div>',
+      '<div ws-connect="wss://example/api"><form ws-send></form></div>',
+      '<script type="text/hyperscript-template" component="hello-world"><span>Hi</span></script>',
+      '  <button _="on click increment ^count">+</button>', // leading whitespace
+    ];
+    for (const code of htmlPatterns) {
+      expect(isHtmlMarkupPattern(code), code).toBe(true);
+    }
+  });
+
+  it('does not flag real hyperscript source', () => {
+    const hyperscript = [
+      'toggle .active on #button',
+      'on click increment $count',
+      'put :x into me',
+      'bind $greeting to #name-input',
+      'live put `Count: ${$count}` into me end',
+      'fetch /api/data as json then put it into #out',
+      '#count を 増加', // SOV, non-Latin
+    ];
+    for (const code of hyperscript) {
+      expect(isHtmlMarkupPattern(code), code).toBe(false);
+    }
+  });
+});

package/src/multilingual/html-pattern.ts

@@ -0,0 +1,24 @@
+/**
+ * HTML-markup pattern detection.
+ *
+ * A handful of DB patterns (`hx-live`, `sse-connect`/`ws-connect`, and the
+ * `<script type="text/hyperscript-template">` component patterns) are authored
+ * as HTML markup — the hyperscript they exercise lives inside attributes
+ * (`hx-live="…"`, `_="…"`) and is processed at runtime by the htmx-compat /
+ * component layer, which is covered by the DOM + Playwright suites.
+ *
+ * The multilingual harness validates patterns with the hyperscript *text*
+ * parser, which structurally cannot parse HTML — grading these is a category
+ * error that understates the real parse rate, so they are excluded from the
+ * semantic-parse denominator.
+ *
+ * Top-level hyperscript never begins with `<` (commands start with a
+ * verb/event keyword or a `.`/`#`/`:`/`$` token), so a leading `<` is a
+ * reliable, self-maintaining signal — new HTML patterns are excluded
+ * automatically without a manual allow/deny list.
+ *
+ * Kept dependency-free so it is unit-testable without the patterns DB.
+ */
+export function isHtmlMarkupPattern(code: string): boolean {
+  return /^\s*</.test(code);
+}

package/src/multilingual/orchestrator.ts

@@ -7,11 +7,20 @@ import { promisify } from 'node:util';
 import { loadPatterns } from './pattern-loader';
 import { selectBundle, getBundleInfo } from './bundle-builder';
 import { ParseValidator } from './validators/parse-validator';
+import { ExecutionValidator, loadExecutionSubset } from './validators/execution-validator';
 import { SizeValidator } from './validators/size-validator';
 import { ConsoleReporter } from './reporters/console-reporter';
 import { JSONReporter } from './reporters/json-reporter';
 import { RegressionReporter } from './reporters/regression-reporter';
-import type { TestConfig, TestResults, LanguageResults, LanguageCode, Reporter } from './types';
+import type {
+  TestConfig,
+  TestResults,
+  LanguageResults,
+  LanguageCode,
+  Reporter,
+  BundleInfo,
+} from './types';
+import { computeFidelity, computePrecision, FIDELITY_THRESHOLD } from './fidelity';
 
 const execAsync = promisify(exec);
 
@@ -48,9 +57,12 @@ export class TestOrchestrator {
     // Add JSON reporter for structured output
     this.reporters.push(new JSONReporter('./test-results/results.json'));
 
-    // Add regression reporter if requested
+    // Add regression reporter if requested. Default to the COMMITTED baseline
+    // (test-results/ is gitignored), overridable via --baseline.
     if (this.config.regression) {
-      this.reporters.push(new RegressionReporter('./test-results/baseline.json'));
+      this.reporters.push(
+        new RegressionReporter(this.config.baselinePath ?? './baselines/multilingual-priority.json')
+      );
     }
   }
 
@@ -88,6 +100,15 @@ export class TestOrchestrator {
         languageResults.push(result);
       }
 
+      // Score structural fidelity against the English reference parse. This
+      // surfaces *degenerate* passes — patterns that parse non-null but drop most
+      // of the source's commands — which the parse-rate metric alone can't see.
+      this.scoreFidelity(languageResults);
+
+      // R2 — execution smoke: run the curated subset in jsdom and compare DOM
+      // effects against the en reference's (see execution-validator.ts).
+      await this.scoreExecution(languageResults);
+
       // Collect bundle information
       const bundles: TestResults['bundles'] = {};
       for (const langResult of languageResults) {
@@ -158,13 +179,25 @@ export class TestOrchestrator {
   private async testLanguage(language: LanguageCode, patterns: any[]): Promise<LanguageResults> {
     const startTime = performance.now();
 
-    // Select bundle for this language
-    const bundle = this.config.bundle
+    // Select bundle for this language. The bundle is consumed only for size
+    // reporting — parse validation below runs in-process via parseSemantic — so a
+    // missing display bundle is a warning, not a fatal error that aborts the sweep.
+    const selected = this.config.bundle
       ? await getBundleInfo(this.config.bundle)
       : await selectBundle([language], this.config.build || false);
 
-    if (!bundle || !bundle.exists) {
-      throw new Error(`Bundle not found for language: ${language}`);
+    const bundle: BundleInfo = selected ?? {
+      name: `browser-${language}`,
+      path: '',
+      languages: [language],
+      size: 0,
+      exists: false,
+    };
+
+    if (!bundle.exists) {
+      console.warn(
+        `⚠ No display bundle for '${language}' (${bundle.name}); reporting size 0 and continuing with in-process parse.`
+      );
     }
 
     // Notify reporters
@@ -209,6 +242,143 @@ export class TestOrchestrator {
     return result;
   }
 
+  /**
+   * Score structural fidelity of every parse against the English reference parse
+   * of the same pattern, in-place. Fills `ParseResult.fidelity` plus per-language
+   * `avgFidelity` / `degeneratePasses`. English is the reference, so its own
+   * results are left unscored. No-op when English wasn't part of the run.
+   */
+  private scoreFidelity(languageResults: LanguageResults[]): void {
+    const en = languageResults.find(r => r.language === 'en');
+    if (!en) return;
+
+    // codeExampleId -> English action signature (only successful en parses).
+    const reference = new Map<string, string[]>();
+    // R0-precision: codeExampleId -> English action MULTISET (duplicates kept).
+    const multisetReference = new Map<string, string[]>();
+    // R1: codeExampleId -> English role signature (action.role:valueType set).
+    const roleReference = new Map<string, string[]>();
+    for (const r of en.parseResults) {
+      if (r.success && r.actionSignature && r.actionSignature.length > 0) {
+        reference.set(r.pattern.codeExampleId, r.actionSignature);
+      }
+      if (r.success && r.actionMultisetSignature && r.actionMultisetSignature.length > 0) {
+        multisetReference.set(r.pattern.codeExampleId, r.actionMultisetSignature);
+      }
+      if (r.success && r.roleSignature && r.roleSignature.length > 0) {
+        roleReference.set(r.pattern.codeExampleId, r.roleSignature);
+      }
+    }
+
+    for (const lang of languageResults) {
+      if (lang.language === 'en') continue;
+
+      const degenerate: string[] = [];
+      const lossy: string[] = [];
+      const scores: number[] = [];
+      const precisionScores: number[] = [];
+      const roleScores: number[] = [];
+
+      for (const result of lang.parseResults) {
+        if (!result.success || !result.actionSignature) continue;
+        const ref = reference.get(result.pattern.codeExampleId);
+        if (!ref) continue;
+
+        const fidelity = computeFidelity(ref, result.actionSignature);
+        if (fidelity === undefined) continue;
+
+        result.fidelity = fidelity;
+        scores.push(fidelity);
+        if (fidelity < FIDELITY_THRESHOLD) degenerate.push(result.pattern.codeExampleId);
+        else if (fidelity < 1) lossy.push(result.pattern.codeExampleId);
+
+        // R0-precision — fraction of THIS parse's actions justified by the en
+        // multiset reference (catches phantom/spurious commands recall misses).
+        const multisetRef = multisetReference.get(result.pattern.codeExampleId);
+        if (multisetRef && result.actionMultisetSignature) {
+          const precision = computePrecision(multisetRef, result.actionMultisetSignature);
+          if (precision !== undefined) {
+            result.precision = precision;
+            precisionScores.push(precision);
+          }
+        }
+
+        // R1 — role recall vs the en role signature (role name + value type).
+        const roleRef = roleReference.get(result.pattern.codeExampleId);
+        if (roleRef && result.roleSignature) {
+          const roleFidelity = computeFidelity(roleRef, result.roleSignature);
+          if (roleFidelity !== undefined) {
+            result.roleFidelity = roleFidelity;
+            roleScores.push(roleFidelity);
+          }
+        }
+      }
+
+      lang.avgFidelity =
+        scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : undefined;
+      lang.avgPrecision =
+        precisionScores.length > 0
+          ? precisionScores.reduce((a, b) => a + b, 0) / precisionScores.length
+          : undefined;
+      lang.avgRoleFidelity =
+        roleScores.length > 0
+          ? roleScores.reduce((a, b) => a + b, 0) / roleScores.length
+          : undefined;
+      lang.degeneratePasses = degenerate.sort();
+      lang.lossyPasses = lossy.sort();
+    }
+  }
+
+  /**
+   * R2 — execute the curated execution subset for every language in the run
+   * and score each translation's DOM-effect signature against the en
+   * reference's, in-place (`avgExecutionFidelity` / `executionFailures`).
+   * English is the reference and is left unscored; patterns whose en
+   * reference errors or produces no effects are excluded (no usable
+   * reference). No-op when English wasn't part of the run.
+   */
+  private async scoreExecution(languageResults: LanguageResults[]): Promise<void> {
+    const en = languageResults.find(r => r.language === 'en');
+    if (!en) return;
+
+    const sources = await loadExecutionSubset(languageResults.map(r => r.language));
+    const validator = new ExecutionValidator();
+    await validator.initialize();
+
+    // codeExampleId -> en effect signature (only clean, effectful references).
+    const reference = new Map<string, string[]>();
+    for (const [id, code] of sources.get('en') ?? []) {
+      const res = await validator.execute(id, code, 'en');
+      if (!res.error && res.effects.length > 0) reference.set(id, res.effects);
+    }
+    if (reference.size === 0) return;
+
+    for (const lang of languageResults) {
+      if (lang.language === 'en') continue;
+      const langSources = sources.get(lang.language);
+      if (!langSources) continue;
+
+      const failures: string[] = [];
+      let scored = 0;
+      let matched = 0;
+      for (const [id, refEffects] of reference) {
+        const code = langSources.get(id);
+        if (!code) continue; // no translation — excluded, not failed
+        const res = await validator.execute(id, code, lang.language);
+        // Match on DOM effects ONLY. Trapped runtime errors are diagnostic:
+        // their attribution rides on unhandled-rejection timing (racy), while
+        // the effect snapshot is synchronous and deterministic — and an AST
+        // mis-build that damages behavior shows up as differing effects anyway.
+        const match = JSON.stringify(res.effects) === JSON.stringify(refEffects);
+        scored++;
+        if (match) matched++;
+        else failures.push(id);
+      }
+      lang.avgExecutionFidelity = scored > 0 ? matched / scored : undefined;
+      lang.executionFailures = failures.sort();
+    }
+  }
+
   /**
    * Group patterns by language
    */

package/src/multilingual/pattern-loader.ts

@@ -12,6 +12,9 @@ import {
   type Translation,
 } from '@hyperfixi/patterns-reference';
 import type { LanguageCode, PatternTranslation, TestConfig, SamplingStrategy } from './types';
+import { isHtmlMarkupPattern } from './html-pattern';
+
+export { isHtmlMarkupPattern };
 
 /**
  * Load patterns for testing based on configuration
@@ -25,13 +28,19 @@ export async function loadPatterns(config: TestConfig): Promise<PatternTranslati
     results.push(...translations);
   }
 
+  // Exclude HTML-markup patterns from the semantic-parse set (see
+  // isHtmlMarkupPattern): the hyperscript text parser can't grade HTML, so
+  // counting them would understate the real parse rate. They are validated by
+  // the DOM/Playwright + i18n-htmx suites instead.
+  const semanticPatterns = results.filter(p => !isHtmlMarkupPattern(p.hyperscript));
+
   // Apply sampling if in quick mode
   if (config.mode === 'quick') {
     const limit = config.quickModeLimit || 10;
-    return samplePatterns(results, { type: 'stratified', perCategory: limit });
+    return samplePatterns(semanticPatterns, { type: 'stratified', perCategory: limit });
   }
 
-  return results;
+  return semanticPatterns;
 }
 
 /**