@hyperfixi/testing-framework 2.7.2 → 2.8.0

package/src/multilingual/types.ts

@@ -100,6 +100,19 @@ export interface TestConfig {
 
   /** Number of patterns per language in quick mode */
   quickModeLimit?: number;
+
+  /**
+   * Report the semantic parser's `unconsumed-input` firing rate per language,
+   * then exit. Read-only: never gates, never writes a baseline.
+   */
+  diagnoseCoverage?: boolean;
+
+  /**
+   * Itemize R1 role-fidelity misses (missing `action.role:type` entries vs the
+   * en reference, clustered) per language, then exit. Read-only: never gates,
+   * never writes a baseline. Scope with `languages`.
+   */
+  triageR1?: boolean;
 }
 
 /**
@@ -138,6 +151,17 @@ export interface ParseResult {
    */
   roleSignature?: string[];
 
+  /**
+   * R3 — role-VALUE signature: `action.role=value` multiset, filtered to
+   * language-invariant surface forms (selectors, sigil refs, numbers/time
+   * literals, colon-qualified event names, URLs — code, not prose). The one
+   * value class that CAN be compared verbatim across languages; catches
+   * right-action/right-type/wrong-VALUE corruption (`draggable` captured for
+   * `draggable:start`) that every type-based signal misses. Set by the
+   * validator (roles are a ReadonlyMap — serialize to {} in results.json).
+   */
+  roleValueSignature?: string[];
+
   /**
    * Structural fidelity vs the English reference parse, in [0, 1]: the fraction
    * of the English parse's distinct actions also present in this language's
@@ -157,6 +181,15 @@ export interface ParseResult {
    */
   precision?: number;
 
+  /**
+   * R0-recall on the MULTISET vs the English reference, in [0, 1]. `fidelity`
+   * above scores the deduped Set, so a parse that drops a REPEATED command
+   * (`[bind, bind]` → `[bind]`) still scores 1.0. This counts duplicates, and is
+   * the mirror of `precision`: together they see both a dropped and an added
+   * copy of a command. `undefined` when there is no usable English reference.
+   */
+  multisetRecall?: number;
+
   /**
    * R1 — role fidelity vs the English reference, in [0, 1]: the fraction of the
    * English parse's role-signature entries also present here. Catches a parse
@@ -164,6 +197,23 @@ export interface ParseResult {
    * patient/destination executes wrongly while action-fidelity scores 1.0).
    */
   roleFidelity?: number;
+
+  /**
+   * R3 — invariant role-VALUE recall vs the English reference, in [0, 1]: the
+   * fraction of the English parse's `roleValueSignature` entries (multiset —
+   * duplicates counted) also present here. Falls below 1.0 when a translation
+   * loses or corrupts a language-invariant value (selector, sigil ref, time
+   * literal, colon-qualified event name, URL) while actions and role types
+   * still match. `undefined` when the en reference has no invariant values.
+   */
+  valueRecall?: number;
+
+  /**
+   * R3 — the en reference's `action.role=value` entries missing from this
+   * parse (multiset difference). Populated only when `valueRecall` < 1, for
+   * diagnostics ("which value was lost/corrupted?").
+   */
+  valueRecallMissing?: string[];
 }
 
 /**
@@ -205,6 +255,14 @@ export interface LanguageResults {
    */
   avgPrecision?: number | undefined;
 
+  /**
+   * R0-recall-multiset — mean `multisetRecall` over successful parses with an
+   * English reference. Recorded + ratcheted alongside avgFidelity: a drop means a
+   * parser change started dropping a REPEATED command, which the Set-based
+   * avgFidelity and avgRoleFidelity cannot see.
+   */
+  avgMultisetRecall?: number | undefined;
+
   /**
    * R1 — mean `roleFidelity` over successful parses with an English reference.
    * Recorded + ratcheted in the baseline; the burn-down is deliberately NOT part
@@ -212,6 +270,14 @@ export interface LanguageResults {
    */
   avgRoleFidelity?: number | undefined;
 
+  /**
+   * R3 — mean `valueRecall` over successful parses with an English reference.
+   * Recorded + ratcheted alongside avgRoleFidelity: a drop means a translation
+   * started losing/corrupting a language-invariant role VALUE (the #633 class),
+   * which every action/type-based signal above scores as perfect.
+   */
+  avgValueRecall?: number | undefined;
+
   /**
    * R2 — mean executionFidelity over the curated execution subset (see
    * validators/execution-validator.ts): 1 per pattern whose jsdom DOM-effect
@@ -298,8 +364,12 @@ export interface Baseline {
         avgFidelity?: number | undefined;
         /** R0-precision — mean precision vs the English reference (see ParseResult.precision). */
         avgPrecision?: number | undefined;
+        /** R0-recall-multiset — mean multisetRecall vs the English reference (see ParseResult.multisetRecall). */
+        avgMultisetRecall?: number | undefined;
         /** R1 — mean role fidelity vs the English reference (see ParseResult.roleFidelity). */
         avgRoleFidelity?: number | undefined;
+        /** R3 — mean invariant role-VALUE recall vs the English reference (see ParseResult.valueRecall). */
+        avgValueRecall?: number | undefined;
         /** R2 — mean execution fidelity over the curated execution subset (see LanguageResults.avgExecutionFidelity). */
         avgExecutionFidelity?: number | undefined;
         /** R2 — curated-subset pattern IDs whose execution diverged from the en reference. */
@@ -335,8 +405,12 @@ export interface RegressionResult {
   avgFidelityDelta: number;
   /** R0-precision — absolute change in avgPrecision (current − baseline). 0 when either side lacks data. Negative = phantom commands introduced. */
   avgPrecisionDelta: number;
+  /** R0-recall-multiset — absolute change in avgMultisetRecall (current − baseline). 0 when either side lacks data. Negative = a repeated command is being dropped. */
+  avgMultisetRecallDelta: number;
   /** R1 — absolute change in avgRoleFidelity (current − baseline). 0 when either side lacks data. */
   avgRoleFidelityDelta: number;
+  /** R3 — absolute change in avgValueRecall (current − baseline). 0 when either side lacks data. Negative = an invariant role VALUE is being lost/corrupted. */
+  avgValueRecallDelta: number;
   /** R2 — absolute change in avgExecutionFidelity (current − baseline). 0 when either side lacks data. */
   avgExecutionFidelityDelta: number;
   /**

package/src/multilingual/validators/parse-validator.ts

@@ -6,7 +6,12 @@ import { MultilingualHyperscript } from '@hyperfixi/core/multilingual';
 import type { SemanticNode } from '@lokascript/semantic';
 import { fillSchemaDefaults } from '@lokascript/semantic';
 import type { PatternTranslation, ParseResult, Validator } from '../types';
-import { collectActions, collectActionsMultiset, collectRoleSignature } from '../fidelity';
+import {
+  collectActions,
+  collectActionsMultiset,
+  collectRoleSignature,
+  collectRoleValueSignature,
+} from '../fidelity';
 
 /**
  * Parse Validator
@@ -106,6 +111,9 @@ export class ParseValidator implements Validator<ParseResult[]> {
         // R1 role signature (role name + value type per command) — collected here
         // because live nodes carry roles as a ReadonlyMap that serializes to {}.
         roleSignature: collectRoleSignature(semanticNode),
+        // R3 role-VALUE signature (invariant-shaped values only, multiset) —
+        // catches right-action/right-type/wrong-VALUE corruption (the #633 class).
+        roleValueSignature: collectRoleValueSignature(semanticNode),
         duration: performance.now() - startTime,
       };
     } catch (error) {

package/src/runner.test.ts

@@ -438,7 +438,11 @@ describe('CoreTestRunner', () => {
 
       const result = await runner.runTest(test, context);
 
-      expect(result.duration).toBeGreaterThanOrEqual(50);
+      // Tolerance below the 50ms sleep: setTimeout clamping can fire the
+      // timer ~1-2ms early on CI runners (observed: 49ms in pre-publish run
+      // 29800757714), and the assertion is about duration RECORDING, not
+      // timer precision.
+      expect(result.duration).toBeGreaterThanOrEqual(45);
     });
   });
 });
@@ -686,7 +690,8 @@ describe('measurePerformance', () => {
 
     const metrics = await measurePerformance(mockPage, testFn);
 
-    expect(metrics.loadTime).toBeGreaterThanOrEqual(50);
+    // Same timer-clamping tolerance as the duration test above.
+    expect(metrics.loadTime).toBeGreaterThanOrEqual(45);
   });
 });
 

package/src/vocab/batch3-roundtrip.test.ts

@@ -0,0 +1,184 @@
+/**
+ * Batch 3 red→green proofs — one representative render→parse round-trip per
+ * V1 dict-fix class (docs-internal/MULTILINGUAL_NEXT_STEPS.md § "V1 probe
+ * conclusion (Batch 3)"). Each of these was a live render→parse break before
+ * the Batch 3 dictionary fixes:
+ *
+ * - select class: the dict word doubled as the pick keyword — the bare select
+ *   render parsed null (de `auswählen #note`).
+ * - wrong-verb class: the dict word was another command's verb — the render
+ *   parsed as THAT action (bn clone → copy, id close → hide, sw copy → clone,
+ *   vi prepend → add, qu change-event → toggle).
+ * - reset class: the dict event word captured on.event as an expression (a
+ *   broken listener) or a wrong event; the profile verb round-trips.
+ * - submit class: the dict event word doubled as the send verb — corpus
+ *   on-submit rows captured event "send" (es/pl/tr/vi live).
+ * - blur class (it): the noun form dropped blur.patient in command position.
+ * - empty class (ko/qu profile-alternatives): the dict adjective renders the
+ *   empty COMMAND but only the profile verb parsed. (ja is waived: bare 空
+ *   phantoms the hot `is empty` rows if registered.)
+ *
+ * Assertions are on captured ACTION and role VALUES, never "it parses".
+ */
+
+import { describe, expect, it } from 'vitest';
+import { parseSemantic } from '@lokascript/semantic';
+import { GrammarTransformer } from '@lokascript/i18n';
+
+/** Verbatim (action, role, value) triples from a parse tree. */
+function triples(node: unknown, acc: string[] = [], depth = 0): string[] {
+  if (depth > 64 || node === null || typeof node !== 'object') return acc;
+  const rec = node as Record<string, unknown>;
+  if (typeof rec.action === 'string') {
+    const roles = rec.roles;
+    const entries: Array<[unknown, unknown]> =
+      roles instanceof Map
+        ? [...roles.entries()]
+        : roles && typeof roles === 'object'
+          ? Object.entries(roles)
+          : [];
+    for (const [role, v] of entries) {
+      if (v === null || v === undefined) continue;
+      const val = (v as { value?: unknown }).value ?? (v as { raw?: unknown }).raw;
+      acc.push(`${rec.action}.${String(role)}=${String(val)}`);
+    }
+  }
+  for (const f of ['body', 'commands', 'children', 'thenBranch', 'elseBranch']) {
+    const c = rec[f];
+    if (Array.isArray(c)) for (const x of c) triples(x, acc, depth + 1);
+    else if (c && typeof c === 'object') triples(c, acc, depth + 1);
+  }
+  return acc;
+}
+
+function renderAndParse(en: string, lang: string): { render: string; triples: string[] } {
+  const render = new GrammarTransformer('en', lang).transform(en);
+  const result = parseSemantic(render, lang);
+  return { render, triples: result.node ? triples(result.node) : [] };
+}
+
+describe('Batch 3 — select class (dict word was the pick keyword)', () => {
+  it('de: `select #note` renders markieren and parses back as select', () => {
+    const { render, triples: t } = renderAndParse('select #note', 'de');
+    expect(render).toContain('markieren');
+    expect(t).toContain('select.patient=#note');
+  });
+
+  it('ar: `select #note` renders ظلل and parses back as select', () => {
+    const { render, triples: t } = renderAndParse('select #note', 'ar');
+    expect(render).toContain('ظلل');
+    expect(t).toContain('select.patient=#note');
+  });
+});
+
+describe('Batch 3 — wrong-verb class (dict word was another command)', () => {
+  it('bn: `clone #card` no longer parses as copy', () => {
+    const { render, triples: t } = renderAndParse('clone #card', 'bn');
+    expect(render).toContain('ক্লোন');
+    expect(t).toContain('clone.patient=#card');
+  });
+
+  it('id: `close #modal` no longer parses as hide', () => {
+    const { render, triples: t } = renderAndParse('close #modal', 'id');
+    expect(render).toContain('tutupkan');
+    expect(t).toContain('close.patient=#modal');
+  });
+
+  it('sw: `copy #text` no longer parses as clone', () => {
+    const { render, triples: t } = renderAndParse('copy #text', 'sw');
+    expect(render).toContain('nakala');
+    expect(t).toContain('copy.patient=#text');
+  });
+
+  it('vi: `prepend "x" to #list` no longer parses as add', () => {
+    const { render, triples: t } = renderAndParse('prepend "x" to #list', 'vi');
+    expect(render).toContain('thêm vào đầu');
+    expect(t.some(x => x.startsWith('prepend.'))).toBe(true);
+  });
+});
+
+describe('Batch 3 — reset class (broken/wrong-event listener)', () => {
+  it.each([
+    ['it', 'reimpostare'],
+    ['ko', '재설정'],
+    ['pl', 'zresetuj'],
+    ['pt', 'redefinir'],
+    ['ru', 'сбросить'],
+    ['uk', 'скинути'],
+    ['qu', 'musuqchay'],
+  ])('%s: on-reset render captures on.event="reset" (canonical)', (langCode, word) => {
+    const { render, triples: t } = renderAndParse('on reset log "done"', langCode);
+    expect(render).toContain(word);
+    expect(t).toContain('on.event=reset');
+  });
+});
+
+describe('Batch 3 — submit class (dict word was the send verb)', () => {
+  it.each([
+    ['es', 'envío'],
+    ['pl', 'wysłaniu'],
+    ['tr', 'gönderme'],
+    ['vi', 'nộp'],
+    ['qu', 'apaykachay'],
+  ])(
+    '%s: corpus-shaped on-submit render captures on.event="submit", not "send"',
+    (langCode, word) => {
+      const { render, triples: t } = renderAndParse(
+        'on submit add @disabled to <button/> in me put "Submitting..." into <button/> in me',
+        langCode
+      );
+      expect(render).toContain(word);
+      expect(t).toContain('on.event=submit');
+      expect(t).not.toContain('on.event=send');
+    }
+  );
+});
+
+describe('Batch 3 — qu change (dict word was the toggle verb)', () => {
+  it('qu: on-change render captures on.event="change", not "toggle"', () => {
+    const { render, triples: t } = renderAndParse('on change log "x"', 'qu');
+    expect(render).toContain('kambiay');
+    expect(t).toContain('on.event=change');
+  });
+});
+
+describe('Batch 3 — it blur (noun form dropped the command patient)', () => {
+  it('it: blur command render captures blur.patient', () => {
+    const { render, triples: t } = renderAndParse('on keydown[key=="Escape"] blur me', 'it');
+    expect(render).toContain('sfuocare');
+    expect(t).toContain('blur.patient=me');
+  });
+
+  it('it: blur event position still captures on.event="blur"', () => {
+    const { triples: t } = renderAndParse('on blur log "x"', 'it');
+    expect(t).toContain('on.event=blur');
+  });
+});
+
+describe('Batch 3 — empty class (ko/qu profile alternatives)', () => {
+  it('ko: the dict adjective now parses the empty command', () => {
+    const t = triples(parseSemantic('#list 를 비어있는', 'ko').node);
+    expect(t).toContain('empty.patient=#list');
+  });
+
+  it('qu: apostrophe-less chusaq now parses the empty command', () => {
+    const t = triples(parseSemantic('#list ta chusaq', 'qu').node);
+    expect(t).toContain('empty.patient=#list');
+  });
+
+  it('ko/qu: the hot `is empty` expression rows keep parsing without a phantom empty action', () => {
+    for (const [text, langCode] of [
+      [
+        '블러 할 때 만약 내 값 이다 비어있는 .error 를 추가 나 에 아니면 .error 를 제거 나 에서 끝',
+        'ko',
+      ],
+      [
+        'paqariy pi sichus noqaq chanin kanqa chusaq .error ta noqa man yapay manachus .error ta noqa manta qichuy tukuy',
+        'qu',
+      ],
+    ] as const) {
+      const t = triples(parseSemantic(text, langCode).node);
+      expect(t.some(x => x.startsWith('empty.'))).toBe(false);
+    }
+  });
+});