@hyperfixi/testing-framework 3.2.0 → 4.0.0

package/src/multilingual/value-matrix.ts

@@ -42,17 +42,29 @@
  * Each cell runs its English source on the real `hyperscript.org` engine: that
  * result is the ORACLE. Then, on the same fixture:
  *
- *   - `en`     hyperfixi's English path (core's parser and runtime);
  *   - `en-rt`  semantic's English parse rendered back to English, on upstream:
  *              where it fails, every translation inherits the loss;
- *   - `<L>`    each of 23 languages on hyperfixi's direct path —
- *              `render(parse_en(src), L)`, compiled with `{ language: L }`;
- *   - `<L>/up` the same translation through `@lokascript/hyperscript-adapter`
- *              (`preprocess`, back to English) on upstream: the multilingual
- *              product for original _hyperscript users.
+ *   - `<L>/up` each of 23 languages — `render(parse_en(src), L)` — through
+ *              `@lokascript/hyperscript-adapter` (`preprocess`, back to
+ *              English) on upstream: the multilingual product for original
+ *              _hyperscript users;
+ *   - `eng`    the English source on `@hyperfixi/engine`, the engine that
+ *              replaces core's;
+ *   - `<L>/eng` the adapter's English (the string the `/up` lane runs) on
+ *              that engine: where it differs from `<L>/up`, the two ENGINES
+ *              differ, since they were given the same text.
  *
  * A (cell, lane) pair FAILS when its result differs from the oracle's.
  *
+ * A `<L>` lane ran each translation on core's direct path (compiled with
+ * `{ language: L }`) until Phase C2 of the engine cutover, which made text the
+ * multilingual interchange. It retired after the measurement the plan asked
+ * for: on the committed baseline no cell failed `<L>/eng` while passing `<L>`
+ * (the only foreign failures, three Italian `di` cells, failed all three).
+ * An `en` lane ran the English source on `@hyperfixi/core`'s parser and runtime
+ * until Phase C4, when it left with core's engine; its only failures were the
+ * eight accepted cells of core's `the X of Y as T` reading, which `eng` passes.
+ *
  * ## The ratchet
  * `baselines/value-matrix.json` lists every failing pair, per cell. The gate
  * fails on a failing pair it does not list AND on a listed pair that now
@@ -64,12 +76,13 @@
  * One jsdom window for the whole run; each lane resets `<body>` and the
  * globals, installs the handler on a fresh button, clicks it, and reads
  * `#out`. Variables are window globals (see GLOBALS), so no source needs a
- * `set` prefix. Upstream runs synchronously; hyperfixi's handler settles
- * within one macrotask.
+ * `set` prefix. Both engines run synchronously unless the source waits.
  *
- * A translation that loses a loop's condition can loop forever. Core caps a
- * loop at 10,000 iterations; upstream has no cap and blocks the thread, so
- * every upstream run gets an evaluation budget (see EVAL_BUDGET).
+ * A translation that loses a loop's condition can loop forever. Upstream has
+ * no cap and blocks the thread, so every upstream run gets an evaluation
+ * budget (see EVAL_BUDGET). The new engine has no cap and no hook for one: a
+ * string that spent the budget on upstream is not run on it, and its `/eng`
+ * lane reads `✗budget` too.
  *
  * Node-only: imports the real `hyperscript.org` build off disk and needs the
  * node vitest environment (see the shipped-examples gate for why).
@@ -731,11 +744,14 @@ export const FOREIGN_LANGUAGES = [
 
 /** Every lane, in report order. */
 export const LANES: readonly string[] = [
-  'en',
   'en-rt',
-  ...FOREIGN_LANGUAGES.flatMap(language => [language, `${language}/up`]),
+  'eng',
+  ...FOREIGN_LANGUAGES.flatMap(language => [`${language}/up`, `${language}/eng`]),
 ];
 
+/** The lanes a cell runs per language it does not skip. */
+export const LANES_PER_LANGUAGE = 2;
+
 /**
  * Upstream evaluations allowed per run. A cell uses well under a hundred; a
  * loop whose condition a translation lost uses them all and stops, where it
@@ -743,10 +759,14 @@ export const LANES: readonly string[] = [
  */
 const EVAL_BUDGET = 20_000;
 
-/** The upstream surface this uses (`hyperscript.org`'s ESM default export). */
-interface UpstreamEngine {
+/** What a lane needs of a host that reads scripts off attributes: upstream, or the new engine. */
+interface ScriptHost {
   parse(source: string): { errors?: Array<{ message: string }> } | undefined;
   processNode(element: Element): void;
+}
+
+/** The upstream surface this uses (`hyperscript.org`'s ESM default export). */
+interface UpstreamEngine extends ScriptHost {
   internals: {
     runtime: { unifiedEval(parseElement: unknown, context: unknown): unknown };
   };
@@ -775,8 +795,8 @@ export interface MatrixEngines {
  *
  * Until `close()`, the console is silenced and unhandled rejections are
  * trapped: thousands of lanes fail by design, both engines report a failure
- * on the console, and hyperfixi's handler is an async listener, so an error
- * in it rejects a promise nobody holds. The process's own rejection
+ * on the console, and a handler that waits on something runs on after its
+ * click, so an error in it rejects a promise nobody holds. The process's own rejection
  * listeners (vitest's, under test) are set aside for the run and restored.
  */
 export async function initMatrixEngines(): Promise<MatrixEngines> {
@@ -803,12 +823,14 @@ export async function initMatrixEngines(): Promise<MatrixEngines> {
   };
   process.on('unhandledRejection', trap);
 
-  const { hyperscript } = await import('@hyperfixi/core');
   const { parseSemantic, render } = await import('@lokascript/semantic');
   const { preprocess } = await import('@lokascript/hyperscript-adapter');
   const require = createRequire(import.meta.url);
   const esm = require.resolve('hyperscript.org').replace(/[^/\\]+$/, '_hyperscript.esm.js');
   const upstream: UpstreamEngine = (await import(pathToFileURL(esm).href)).default;
+  const engineModule = await import('@hyperfixi/engine');
+  engineModule.register(...engineModule.everything);
+  const engine: ScriptHost = engineModule.api;
 
   // The evaluation budget: every upstream evaluation goes through unifiedEval.
   const runtime = upstream.internals.runtime;
@@ -821,13 +843,6 @@ export async function initMatrixEngines(): Promise<MatrixEngines> {
 
   const window = dom.window;
   const document = window.document;
-  type Ast = Parameters<typeof hyperscript.execute>[0];
-
-  // hyperfixi keeps its global variables in one Map that every context shares,
-  // and writes a window global there (`increment n`). It outlives the run, and
-  // it shadows window's copy, so every later lane would read the write.
-  const coreGlobals = hyperscript.createContext().globals;
-  const coreGlobalsAtStart = new Map(coreGlobals);
   const headAtStart = document.head.innerHTML;
   const names = collidingNames();
 
@@ -844,8 +859,6 @@ export async function initMatrixEngines(): Promise<MatrixEngines> {
       html.innerHTML = `<head>${headAtStart}</head><body></body>`;
     }
     document.body.innerHTML = FIXTURE;
-    coreGlobals.clear();
-    for (const [name, value] of coreGlobalsAtStart) coreGlobals.set(name, value);
     for (const [name, make] of Object.entries(GLOBALS)) {
       const value = make();
       Reflect.set(window, name, value);
@@ -862,29 +875,21 @@ export async function initMatrixEngines(): Promise<MatrixEngines> {
   };
   const read = (): string => document.getElementById('out')?.textContent ?? '✗no #out';
 
-  /** Run English on upstream. */
-  const onUpstream = (source: string): string => {
-    const errors = upstream.parse(source)?.errors ?? [];
+  /** Run English on a host that reads the script off the button. */
+  const onHost = (host: ScriptHost, source: string): string => {
+    const errors = host.parse(source)?.errors ?? [];
     if (errors.length) return `✗parse: ${errors[0]?.message.split('\n')[0] ?? ''}`;
     const button = reset();
     button.setAttribute('_', source);
     evaluations = 0;
     const before = reportedErrors;
-    upstream.processNode(button);
+    host.processNode(button);
     click(button);
     if (evaluations > EVAL_BUDGET) return '✗budget';
     const got = read();
     return reportedErrors > before && got === '∅' ? '✗threw' : got;
   };
-
-  /** Install a compiled handler on hyperfixi, click, and settle. */
-  const onHyperfixi = async (ast: Ast): Promise<string> => {
-    const button = reset();
-    await hyperscript.execute(ast, hyperscript.createContext(button));
-    click(button);
-    await new Promise(resolve => setTimeout(resolve, 0));
-    return read();
-  };
+  const onUpstream = (source: string): string => onHost(upstream, source);
 
   const guard = async (run: () => Promise<string> | string): Promise<string> => {
     try {
@@ -904,16 +909,11 @@ export async function initMatrixEngines(): Promise<MatrixEngines> {
       }
       const lanes = result.lanes;
 
-      lanes.en = await guard(async () => {
-        const compiled = hyperscript.compileSync(cell.source);
-        if (!compiled.ok || !compiled.ast) return '✗compile';
-        return onHyperfixi(compiled.ast);
-      });
-
       const english = parseSemantic(cell.source, 'en').node;
       lanes['en-rt'] = await guard(() =>
         english ? onUpstream(render(english, 'en')) : '✗untranslatable'
       );
+      lanes.eng = await guard(() => onHost(engine, cell.source));
 
       for (const language of FOREIGN_LANGUAGES) {
         if (cell.skip?.includes(language)) continue;
@@ -924,17 +924,20 @@ export async function initMatrixEngines(): Promise<MatrixEngines> {
           code = null;
         }
         if (code === null) {
-          lanes[language] = '✗untranslatable';
           lanes[`${language}/up`] = '✗untranslatable';
+          lanes[`${language}/eng`] = '✗untranslatable';
           continue;
         }
         const translated = code;
-        lanes[language] = await guard(async () => {
-          const compiled = await hyperscript.compile(translated, { language });
-          if (!compiled.ok || !compiled.ast) return '✗compile';
-          return onHyperfixi(compiled.ast);
-        });
-        lanes[`${language}/up`] = await guard(() => onUpstream(preprocess(translated, language)));
+        // The adapter's English, once, for both hosts.
+        const adapted = await guard(() => preprocess(translated, language));
+        const up = (lanes[`${language}/up`] = adapted.startsWith('✗')
+          ? adapted
+          : await guard(() => onUpstream(adapted)));
+        lanes[`${language}/eng`] =
+          adapted.startsWith('✗') || up === '✗budget'
+            ? up
+            : await guard(() => onHost(engine, adapted));
       }
       return result;
     },
@@ -967,7 +970,7 @@ export async function runValueMatrix(cells: readonly MatrixCell[]): Promise<Cell
 export interface BaselineEntry {
   /**
    * The failing lanes, space-separated, in LANES order, with two shorthands:
-   * `*direct` for all 23 languages on hyperfixi, `*up` for all 23 on upstream.
+   * `*up` for all 23 languages on upstream, `*eng` for all 23 on the new engine.
    */
   lanes: string;
   /** Where the loss sits, for reading the burn-down (see familyOf); not asserted. */
@@ -987,23 +990,23 @@ export interface ValueMatrixBaseline {
   entries: Record<string, BaselineEntry>;
 }
 
-const DIRECT_LANES: readonly string[] = FOREIGN_LANGUAGES;
 const ADAPTER_LANES: readonly string[] = FOREIGN_LANGUAGES.map(language => `${language}/up`);
+const ENGINE_LANES: readonly string[] = FOREIGN_LANGUAGES.map(language => `${language}/eng`);
 
 /** Lanes (in LANES order) as a baseline string, with a shorthand for each full group. */
 export function compressLanes(lanes: readonly string[]): string {
   const set = new Set(lanes);
   const out: string[] = [];
-  const direct = DIRECT_LANES.every(l => set.has(l));
   const adapter = ADAPTER_LANES.every(l => set.has(l));
+  const engine = ENGINE_LANES.every(l => set.has(l));
   for (const lane of LANES) {
     if (!set.has(lane)) continue;
-    if (direct && DIRECT_LANES.includes(lane)) continue;
     if (adapter && ADAPTER_LANES.includes(lane)) continue;
+    if (engine && ENGINE_LANES.includes(lane)) continue;
     out.push(lane);
   }
-  if (direct) out.push('*direct');
   if (adapter) out.push('*up');
+  if (engine) out.push('*eng');
   return out.join(' ');
 }
 
@@ -1011,8 +1014,8 @@ export function compressLanes(lanes: readonly string[]): string {
 export function expandLanes(text: string): string[] {
   const out: string[] = [];
   for (const token of text.split(' ').filter(Boolean)) {
-    if (token === '*direct') out.push(...DIRECT_LANES);
-    else if (token === '*up') out.push(...ADAPTER_LANES);
+    if (token === '*up') out.push(...ADAPTER_LANES);
+    else if (token === '*eng') out.push(...ENGINE_LANES);
     else out.push(token);
   }
   return out;
@@ -1026,27 +1029,23 @@ export function failingLanes(result: CellResult): string[] {
 /**
  * Where a cell's failure sits, from which lanes fail:
  *
- *   - `core`         core's English run differs from upstream's;
  *   - `semantic-en`  semantic's English parse loses it (`en-rt`), so every
  *                    translation inherits the loss;
- *   - `translation`  some foreign lanes, on both engines;
- *   - `direct-path`  hyperfixi's foreign lanes only;
- *   - `adapter`      upstream's foreign lanes only.
+ *   - `translation`  some foreign lanes on upstream: the adapter's English
+ *                    for that language says something else;
+ *   - `engine`       the new engine differs from upstream on the same text:
+ *                    its English run, or a language's `/eng` lane without
+ *                    its `/up` lane.
  *
  * Several can hold at once; they are joined in that order.
  */
 export function familyOf(lanes: readonly string[]): string {
   const set = new Set(lanes);
   const parts: string[] = [];
-  if (set.has('en')) parts.push('core');
   if (set.has('en-rt')) parts.push('semantic-en');
-  else {
-    const direct = FOREIGN_LANGUAGES.filter(l => set.has(l));
-    const adapter = FOREIGN_LANGUAGES.filter(l => set.has(`${l}/up`));
-    const both = direct.filter(l => adapter.includes(l));
-    if (both.length) parts.push('translation');
-    if (direct.length > both.length && !set.has('en')) parts.push('direct-path');
-    if (adapter.length > both.length) parts.push('adapter');
+  else if (FOREIGN_LANGUAGES.some(l => set.has(`${l}/up`))) parts.push('translation');
+  if (set.has('eng') || FOREIGN_LANGUAGES.some(l => set.has(`${l}/eng`) !== set.has(`${l}/up`))) {
+    parts.push('engine');
   }
   return parts.join('+') || 'none';
 }
@@ -1063,26 +1062,13 @@ export const ACCEPTED: ReadonlyArray<{
   lanes: string;
   reason: string;
 }> = [
-  {
-    cells: [
-      ...['put', 'set', 'while', 'times', 'increment'].map(
-        p => `${p}|the textContent of #a as Int`
-      ),
-      // The same difference with a reference owner (PR 108); upstream's
-      // `window as Int` is null, and in a loop bound both read 0 iterations.
-      ...['put', 'set', 'increment'].map(p => `${p}|the scrollY of window as Int`),
-    ],
-    lanes: 'en *direct',
-    reason:
-      'known difference: core converts the property, upstream the target (core/docs/UPSTREAM-KNOWN-DIFFS.md)',
-  },
   {
     cells: [
       'increment|#a.textContent',
       'increment|#a.textContent + 2',
       'increment|#a.textContent as Int',
     ],
-    lanes: 'it it/up',
+    lanes: 'it/up it/eng',
     reason:
       'ambiguity: it `di` is both `by` and `of`, so `incrementare i di #a.textContent` also says `increment i of #a.textContent`',
   },

package/src/multilingual/shipped-sources-validity.test.ts

@@ -1,95 +0,0 @@
-/**
- * Shipped-sources validity gate (see shipped-sources-validity.ts for the why).
- *
- * Compiles every hyperscript source we ship in `examples/` and the doc trees,
- * and ratchets on the recovers-with-errors state (`ok: true` with a non-empty
- * `errors`). Three assertions, matching the canonical-validity gates:
- *   1. sanity — sources were actually found and mostly compile clean;
- *   2. no NEW recovering source appears outside the committed allowlist;
- *   3. no allowlisted key has silently become clean (stale entries must be
- *      removed so the list only ever shrinks).
- *
- * To update after an intentional fix: re-run and rewrite
- * `baselines/shipped-sources-validity.json`. Note the allowlist key embeds a
- * hash of the source, so FIXING a source changes its key — the entry goes
- * stale and assertion 3 makes removing it mandatory.
- */
-
-import { readFileSync } from 'node:fs';
-import { fileURLToPath } from 'node:url';
-import path from 'node:path';
-import { JSDOM } from 'jsdom';
-import { describe, it, expect, beforeAll } from 'vitest';
-import {
-  checkShippedSourcesValidity,
-  type ShippedSourcesResult,
-  type CompileForValidity,
-} from './shipped-sources-validity';
-
-interface AllowlistDoc {
-  allowedRecovered: Array<{
-    key: string;
-    file: string;
-    error: string;
-    upstream: string;
-    reason: string;
-  }>;
-}
-
-const baselinePath = path.resolve(
-  path.dirname(fileURLToPath(import.meta.url)),
-  '../../baselines/shipped-sources-validity.json'
-);
-const allowlist = JSON.parse(readFileSync(baselinePath, 'utf8')) as AllowlistDoc;
-const allowed = new Set(allowlist.allowedRecovered.map(e => e.key));
-
-describe('shipped-sources validity gate', () => {
-  let result: ShippedSourcesResult;
-
-  beforeAll(async () => {
-    // Import core through its built entry, the same surface a consumer gets.
-    const core = (await import('@hyperfixi/core')) as unknown as {
-      hyperscript: { compileSync: CompileForValidity };
-    };
-    const dom = new JSDOM('<!doctype html><html><body></body></html>');
-    result = checkShippedSourcesValidity(
-      code => core.hyperscript.compileSync(code),
-      dom.window.document as unknown as Parameters<typeof checkShippedSourcesValidity>[1]
-    );
-  }, 120_000);
-
-  it('finds and compiles the shipped sources (sanity: trees walked, extraction working)', () => {
-    // Guards the silent-zero failure mode: a broken walk or extractor would
-    // make every other assertion vacuously pass.
-    expect(result.checked).toBeGreaterThan(100);
-    expect(result.clean).toBeGreaterThan(100);
-  });
-
-  it('has no NEW source that parses with recovered errors outside the allowlist', () => {
-    const unexpected = result.findings.filter(f => !allowed.has(f.key));
-    expect(
-      unexpected,
-      unexpected.length
-        ? `\nNew shipped sources that parse ok:true WITH errors (fix the source, or allowlist with a reason):\n` +
-            unexpected
-              .map(f => `  [${f.key}]\n      "${f.excerpt}"\n      -> ${f.error}`)
-              .join('\n') +
-            `\n\nAsk the real hyperscript.org engine for a second opinion before deciding:\n` +
-            `upstream REJECTING means the source is malformed; upstream ACCEPTING means this is a hyperfixi parser defect.\n` +
-            `See loadCanonicalParser() in canonical-validity.ts.`
-        : ''
-    ).toEqual([]);
-  });
-
-  it('has no stale allowlist entries (a now-clean source must be removed so the list ratchets down)', () => {
-    const stillFailing = new Set(result.findings.map(f => f.key));
-    const stale = allowlist.allowedRecovered.map(e => e.key).filter(key => !stillFailing.has(key));
-    expect(
-      stale,
-      stale.length
-        ? `\nThese allowlisted sources no longer recover-with-errors (fixed, or edited — the key embeds a source hash).\n` +
-            `Remove them from baselines/shipped-sources-validity.json:\n  ${stale.join('\n  ')}`
-        : ''
-    ).toEqual([]);
-  });
-});