@hyperfixi/testing-framework 2.9.2 → 2.9.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hyperfixi/testing-framework",
-  "version": "2.9.2",
+  "version": "2.9.4",
   "description": "Cross-platform behavior testing suite for LokaScript applications",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
@@ -57,14 +57,14 @@
   "author": "LokaScript Contributors",
   "license": "MIT",
   "dependencies": {
-    "@hyperfixi/core": "^2.9.2",
-    "@hyperfixi/patterns-reference": "^2.9.2",
-    "@lokascript/i18n": "^2.9.2",
-    "@lokascript/semantic": "^2.9.2",
+    "@hyperfixi/core": "^2.9.4",
+    "@hyperfixi/patterns-reference": "^2.9.4",
+    "@lokascript/i18n": "^2.9.4",
+    "@lokascript/semantic": "^2.9.4",
     "diff": "^8.0.3",
     "esbuild": "^0.28.0",
     "happy-dom": "^20.10.6",
-    "jsdom": "^29.0.2",
+    "jsdom": "^30.0.0",
     "playwright": "^1.40.0",
     "puppeteer": "^24.26.1",
     "tsx": "^4.23.1",
@@ -73,7 +73,7 @@
   },
   "devDependencies": {
     "@types/diff": "^5.0.0",
-    "@types/node": "^20.0.0",
+    "@types/node": "^26.1.2",
     "@vitest/coverage-v8": "^4.0.0",
     "hyperscript.org": "0.9.93",
     "tsup": "^8.0.0",

package/src/multilingual/cli.ts

@@ -478,6 +478,28 @@ async function main(): Promise<void> {
           r => r.avgRoleFidelityDelta < -AVG_ROLE_FIDELITY_DROP_TOLERANCE
         );
 
+        // R1 — role-set flip ratchet: a pattern that was ROLE-FAITHFUL in the
+        // baseline (its action.role:valueType set matched the en reference
+        // exactly) and is now role-incomplete. Tolerance 0 for the same reason
+        // the lossy flip and R5 sit at 0: a flip is binary and structural, with
+        // no float/collation noise to absorb — the averages' tolerances are
+        // cross-machine headroom, not signal cushions.
+        // Not redundant with the avg ratchet above: one pattern losing one role
+        // moves avgRoleFidelity by ~0.0013 in a ~154-pattern language, 15× under
+        // its 0.02 tolerance. This was MEASURED as the gate's blind spot, not
+        // hypothesized: a `required: false` duration role on toggleSchema cost
+        // es/pl/vi the second toggle's positional destination on
+        // `toggle-aria-expanded` (`toggle.destination:expression` →
+        // `:reference`, i.e. `next .panel` silently became `me`) and
+        // --regression stayed GREEN; tools/triage-r1.ts found it by hand.
+        // Guarded by the baseline carrying `roleLossyPatterns` (the reporter
+        // returns [] without it), so an un-regenerated baseline never
+        // retro-flags. Pre-existing role-lossy patterns are the R1 burn-down
+        // list, not regressions.
+        const roleSetRegressions = allResults.flatMap(r =>
+          r.newRoleLossyPatterns.map(id => `${r.language}/${id}`)
+        );
+
         // R3 — role-VALUE ratchet: same semantics as the role-fidelity ratchet,
         // on the invariant-value signal (`action.role=value` multiset over the
         // code-shaped subset: selectors, sigil refs, time literals,
@@ -693,6 +715,24 @@ async function main(): Promise<void> {
           failed = true;
         }
 
+        if (roleSetRegressions.length > 0) {
+          console.error(
+            `\n✗ Role-set regression vs baseline (R1): ${roleSetRegressions.length} ` +
+              `role-faithful pattern(s) now have an incomplete role set:`
+          );
+          for (const p of roleSetRegressions.slice(0, 20)) console.error(`   ${p}`);
+          if (roleSetRegressions.length > 20) {
+            console.error(`   … and ${roleSetRegressions.length - 20} more`);
+          }
+          console.error(
+            `   (still parses, but lost/mistyped a role vs the en reference — one flip ` +
+              `moves avgRoleFidelity ~0.0013, far under its tolerance, so this is the ` +
+              `only signal that sees it. Itemize with tools/triage-r1.ts; if ` +
+              `intentional, regenerate the baseline with --save-baseline)`
+          );
+          failed = true;
+        }
+
         if (valueRecallDrops.length > 0) {
           console.error(
             `\n✗ avgValueRecall dropped > ${AVG_VALUE_RECALL_DROP_TOLERANCE} in ` +

package/src/multilingual/orchestrator.ts

@@ -155,8 +155,7 @@ export class TestOrchestrator {
       // Report regression if enabled
       if (this.config.regression) {
         const regressionReporter = this.reporters.find(r => r instanceof RegressionReporter) as
-          | RegressionReporter
-          | undefined;
+          RegressionReporter | undefined;
 
         if (regressionReporter?.hasBaseline()) {
           const regressionResults = regressionReporter.getRegressionResults();
@@ -286,6 +285,7 @@ export class TestOrchestrator {
 
       const degenerate: string[] = [];
       const lossy: string[] = [];
+      const roleLossy: string[] = [];
       const scores: number[] = [];
       const precisionScores: number[] = [];
       const multisetRecallScores: number[] = [];
@@ -331,6 +331,11 @@ export class TestOrchestrator {
           if (roleFidelity !== undefined) {
             result.roleFidelity = roleFidelity;
             roleScores.push(roleFidelity);
+            // Per-pattern R1: the average above cannot see a single pattern
+            // losing a role (~0.0013 in a ~154-pattern language, vs the 0.02
+            // gate tolerance). Recording WHICH patterns are role-incomplete
+            // makes the flip binary and ratchetable at tolerance 0.
+            if (roleFidelity < 1) roleLossy.push(result.pattern.codeExampleId);
           }
         }
 
@@ -372,6 +377,7 @@ export class TestOrchestrator {
           : undefined;
       lang.degeneratePasses = degenerate.sort();
       lang.lossyPasses = lossy.sort();
+      lang.roleLossyPatterns = roleLossy.sort();
     }
   }
 
@@ -471,8 +477,7 @@ export class TestOrchestrator {
    */
   getRegressionReporter(): RegressionReporter | undefined {
     return this.reporters.find(r => r instanceof RegressionReporter) as
-      | RegressionReporter
-      | undefined;
+      RegressionReporter | undefined;
   }
 }
 

package/src/multilingual/reporters/regression-reporter.test.ts

@@ -482,3 +482,106 @@ describe('RegressionReporter per-pattern parse ratchet — R5', () => {
     expect(r.newFailures).toHaveLength(25);
   });
 });
+
+describe('RegressionReporter role-set flip ratchet — R1 per-pattern', () => {
+  // LOCK: the avgRoleFidelity delta cannot see a single pattern losing a role —
+  // ~0.0013 in a ~154-pattern language, 15× under its 0.02 tolerance. That is
+  // not hypothetical: a `required: false` duration role on toggleSchema cost
+  // es/pl/vi the second toggle's positional destination on `toggle-aria-expanded`
+  // (`next .panel` silently became `me`) and `--regression` stayed green. This
+  // ratchet is the binary per-pattern signal that sees the flip, at tolerance 0.
+  let dir: string;
+  afterEach(() => {
+    if (dir) rmSync(dir, { recursive: true, force: true });
+  });
+
+  function reporterWith(baseline: Baseline): RegressionReporter {
+    dir = mkdtempSync(join(tmpdir(), 'roleflip-'));
+    const path = join(dir, 'baseline.json');
+    writeFileSync(path, JSON.stringify(baseline));
+    return new RegressionReporter(path);
+  }
+
+  function langWithRoleLossy(roleLossyPatterns: string[]): LanguageResults {
+    return {
+      ...lang([pass('was-role-faithful'), pass('was-role-lossy'), pass('stable')], []),
+      roleLossyPatterns,
+    };
+  }
+
+  const baseline: Baseline = {
+    timestamp: '',
+    commit: 'base',
+    languages: {
+      ja: {
+        parseSuccess: 3,
+        parseFailure: 0,
+        parseRate: 1,
+        avgConfidence: 1,
+        avgRoleFidelity: 0.99,
+        roleLossyPatterns: ['was-role-lossy'],
+        bundleSize: undefined,
+        patterns: {
+          'was-role-faithful': { success: true, confidence: 1 },
+          'was-role-lossy': { success: true, confidence: 1 },
+          'was-failing': { success: false, confidence: 0 },
+          stable: { success: true, confidence: 1 },
+        },
+      },
+    },
+    bundles: {},
+  };
+
+  it('flags a role-faithful baseline pattern that is now role-incomplete', () => {
+    const reporter = reporterWith(baseline);
+    reporter.reportComplete(results(langWithRoleLossy(['was-role-faithful', 'was-role-lossy'])));
+    const r = reporter.getRegressionResults().find(x => x.language === 'ja')!;
+    expect(r.newRoleLossyPatterns).toEqual(['was-role-faithful']);
+    expect(r.newRoleLossyPatterns).not.toContain('was-role-lossy'); // burn-down, not regression
+    expect(r.status).toBe('regressed');
+  });
+
+  it('does not flag when the role-lossy set is unchanged', () => {
+    const reporter = reporterWith(baseline);
+    reporter.reportComplete(results(langWithRoleLossy(['was-role-lossy'])));
+    const r = reporter.getRegressionResults().find(x => x.language === 'ja')!;
+    expect(r.newRoleLossyPatterns).toEqual([]);
+    expect(r.status).not.toBe('regressed');
+  });
+
+  it('treats a role-lossy pattern that became faithful as improvement-neutral', () => {
+    const reporter = reporterWith(baseline);
+    reporter.reportComplete(results(langWithRoleLossy([])));
+    const r = reporter.getRegressionResults().find(x => x.language === 'ja')!;
+    expect(r.newRoleLossyPatterns).toEqual([]);
+  });
+
+  it('does not flag a pattern whose PARSE failed in the baseline (improvement path)', () => {
+    // `was-failing` did not parse in the baseline; parsing role-incompletely now
+    // is strictly better, not a regression.
+    const reporter = reporterWith(baseline);
+    reporter.reportComplete(results(langWithRoleLossy(['was-failing'])));
+    const r = reporter.getRegressionResults().find(x => x.language === 'ja')!;
+    expect(r.newRoleLossyPatterns).toEqual([]);
+  });
+
+  it('never retro-flags when the baseline has no roleLossyPatterns data', () => {
+    const noRoleData: Baseline = structuredClone(baseline);
+    delete noRoleData.languages.ja!.roleLossyPatterns;
+    const reporter = reporterWith(noRoleData);
+    reporter.reportComplete(
+      results(langWithRoleLossy(['was-role-faithful', 'was-role-lossy', 'stable']))
+    );
+    const r = reporter.getRegressionResults().find(x => x.language === 'ja')!;
+    expect(r.newRoleLossyPatterns).toEqual([]);
+  });
+
+  it('persists roleLossyPatterns when saving a baseline', () => {
+    const reporter = reporterWith(baseline);
+    const res = results(langWithRoleLossy(['was-role-lossy', 'stable']));
+    reporter.reportComplete(res);
+    reporter.saveAsBaseline(res);
+    const saved = JSON.parse(readFileSync(join(dir, 'baseline.json'), 'utf-8')) as Baseline;
+    expect(saved.languages.ja!.roleLossyPatterns).toEqual(['was-role-lossy', 'stable']);
+  });
+});

package/src/multilingual/reporters/regression-reporter.ts

@@ -153,6 +153,8 @@ export class RegressionReporter implements Reporter {
       const newDegeneratePasses = this.findNewDegeneratePasses(langResult, baselineLang);
       // Correctness ratchet (R0): faithful pass in baseline → lossy pass now.
       const newLossyPasses = this.findNewLossyPasses(langResult, baselineLang);
+      // Role-set ratchet (R1): role-faithful in baseline → role-incomplete now.
+      const newRoleLossyPatterns = this.findNewRoleLossyPatterns(langResult, baselineLang);
       // Execution ratchet (R2): faithful execution in baseline → divergent now.
       const newExecutionFailures = this.findNewExecutionFailures(langResult, baselineLang);
 
@@ -163,6 +165,7 @@ export class RegressionReporter implements Reporter {
         newFailures.length > 0 ||
         newDegeneratePasses.length > 0 ||
         newLossyPasses.length > 0 ||
+        newRoleLossyPatterns.length > 0 ||
         newExecutionFailures.length > 0
       ) {
         status = 'regressed';
@@ -185,6 +188,7 @@ export class RegressionReporter implements Reporter {
         newSuccesses,
         newDegeneratePasses,
         newLossyPasses,
+        newRoleLossyPatterns,
         newExecutionFailures,
         status,
       });
@@ -321,6 +325,39 @@ export class RegressionReporter implements Reporter {
     return regressed.sort();
   }
 
+  /**
+   * Role-set ratchet (R1): patterns that were *role-faithful* (roleFidelity 1.0,
+   * i.e. absent from the baseline's `roleLossyPatterns`) but now have an
+   * incomplete role set vs the en reference. The binary counterpart of the
+   * avgRoleFidelity delta, which cannot see a single pattern losing a role
+   * (~0.0013 in a ~154-pattern language, vs the 0.02 tolerance) — measured
+   * doing exactly that (es/pl/vi silently lost `toggle-aria-expanded`'s
+   * positional destination, `next .panel` → `me`, and the gate stayed green).
+   *
+   * Returns [] when the baseline carries no `roleLossyPatterns` data yet, so
+   * adopting the signal never retro-flags — same guard as the degenerate/lossy
+   * ratchets. A pattern already role-lossy in the baseline is the burn-down
+   * list, not a regression; one whose PARSE failed in the baseline and now
+   * parses role-incompletely is an improvement and is excluded by `wasPass`.
+   */
+  private findNewRoleLossyPatterns(
+    current: LanguageResults,
+    baseline: {
+      patterns: Record<string, { success: boolean; confidence: number | undefined }> | undefined;
+      roleLossyPatterns?: string[] | undefined;
+    }
+  ): string[] {
+    if (!baseline.roleLossyPatterns || !baseline.patterns) return [];
+    const baselineRoleLossy = new Set(baseline.roleLossyPatterns);
+
+    const regressed: string[] = [];
+    for (const id of current.roleLossyPatterns ?? []) {
+      const wasPass = baseline.patterns[id]?.success === true;
+      if (wasPass && !baselineRoleLossy.has(id)) regressed.push(id);
+    }
+    return regressed.sort();
+  }
+
   /**
    * Execution ratchet (R2): curated-subset patterns that executed faithfully in
    * the baseline (scored, and not in its `executionFailures`) but diverge from
@@ -400,6 +437,9 @@ export class RegressionReporter implements Reporter {
         executionFailures: langResult.executionFailures ?? undefined,
         degeneratePasses: langResult.degeneratePasses ?? undefined,
         lossyPasses: langResult.lossyPasses ?? undefined,
+        // R1 per-pattern — which patterns are role-incomplete vs the en
+        // reference. The role-set flip ratchet's baseline side.
+        roleLossyPatterns: langResult.roleLossyPatterns ?? undefined,
         bundleSize: langResult.bundleSize ?? undefined,
         patterns,
       };

package/src/multilingual/types.ts

@@ -308,6 +308,18 @@ export interface LanguageResults {
    */
   lossyPasses?: string[] | undefined;
 
+  /**
+   * R1 — pattern IDs that parse with roleFidelity < 1 (the `action.role:valueType`
+   * set vs the English reference is incomplete). The per-pattern counterpart of
+   * `avgRoleFidelity`: a single pattern losing a role moves the average by
+   * ~0.0013 in a ~154-pattern language, which the 0.02 avg tolerance swallows —
+   * measured doing exactly that (es/pl/vi lost `toggle-aria-expanded`'s
+   * positional destination to a schema change and the gate stayed green). The
+   * role-set flip ratchet compares this list against the baseline's at
+   * tolerance 0.
+   */
+  roleLossyPatterns?: string[] | undefined;
+
   /** Duration in ms */
   duration: number;
 
@@ -378,6 +390,8 @@ export interface Baseline {
         degeneratePasses?: string[] | undefined;
         /** Pattern IDs that pass *lossily* (fidelity in [0.5, 1.0) — drop ≥1 command). */
         lossyPasses?: string[] | undefined;
+        /** R1 — pattern IDs whose role set is incomplete vs the en reference (roleFidelity < 1). */
+        roleLossyPatterns?: string[] | undefined;
         bundleSize: number | undefined;
         /** Pattern-level results for detailed tracking */
         patterns: Record<string, { success: boolean; confidence: number | undefined }> | undefined;
@@ -434,6 +448,14 @@ export interface RegressionResult {
    * misses. Empty when the baseline has no lossy data yet (never retro-flags).
    */
   newLossyPasses: string[];
+  /**
+   * R1 — patterns that were *role-faithful* (roleFidelity 1.0) in the baseline
+   * but now have an incomplete role set (roleFidelity < 1). The binary
+   * per-pattern signal the avgRoleFidelity tolerance cannot provide: one flip
+   * moves the average by ~0.0013, far under 0.02. Empty when the baseline has
+   * no `roleLossyPatterns` data yet (never retro-flags).
+   */
+  newRoleLossyPatterns: string[];
   status: 'improved' | 'regressed' | 'unchanged';
 }
 

package/src/multilingual/validators/execution-validator.test.ts

@@ -19,7 +19,7 @@ import { describe, it, expect, beforeAll } from 'vitest';
 import { ExecutionValidator, EXECUTION_SUBSET, loadExecutionSubset } from './execution-validator';
 
 describe('R2 execution subset (lock)', () => {
-  it('contains exactly the 47 curated patterns', () => {
+  it('contains exactly the 48 curated patterns', () => {
     // Changing this list recalibrates avgExecutionFidelity for every language.
     // If you expand the subset, regenerate the baseline (--save-baseline) in
     // the SAME PR and update this lock.
@@ -119,6 +119,7 @@ describe('R2 execution subset (lock)', () => {
         'chained-access-possessive-dot',
         'hide-with-transition',
         'show-with-transition',
+        'swap-content',
         // Wave 10 (SOV literal-role-extraction arc, PRs #560/#561): the two R2
         // blockers — the fronted append content literal (bogus `event` role in
         // the body-clause marker lookup) and the trailing bare increment amount

package/src/multilingual/validators/execution-validator.ts

@@ -203,6 +203,16 @@ export const EXECUTION_SUBSET: readonly string[] = [
   //    Fixture adds `<div id="score">0</div>` (appended last).
   'append-content',
   'increment-by-amount',
+  // Expansion wave 11: `swap-content` — `swap #a with #b`. R2 stayed at 1.0
+  // through the ENTIRE swap breakage (the descriptor emitted modifiers
+  // SwapCommand never reads, so every language including English threw
+  // `could not parse arguments`), and then through eight languages binding the
+  // operands BACKWARDS once #845 made it execute. Both were invisible here for
+  // one reason: swap-content was not in this list. It is the missing gate for
+  // R3 family 6, and it turns the operand fix from "trust me" into a row.
+  // Fixture adds `#a`/`#b` with distinguishable content (appended last, so
+  // existing snapshot indices are preserved).
+  'swap-content',
 ];
 
 /**
@@ -229,6 +239,8 @@ const FIXTURE_HTML = `<!DOCTYPE html><html><body>
   <div id="sr-announce"></div>
   <ul id="list"></ul>
   <div id="score">0</div>
+  <div id="a">AAA</div>
+  <div id="b">BBB</div>
 </body></html>`;
 
 /** Per-pattern fixture preconditions (applied identically for every language). */