@hyperfixi/testing-framework 2.9.4 → 2.11.0

package/src/agent-bench/harness.ts

@@ -0,0 +1,331 @@
+/**
+ * Agent-loop benchmark — the scoring engine.
+ *
+ * Answers two questions per candidate, deliberately separately:
+ *
+ *   1. **Does it parse?**  `CompilationService.validate()` — the exact call the
+ *      `validate_and_compile` MCP tool makes, so the number reflects what an
+ *      agent actually sees.
+ *   2. **Does it DO the right thing?**  The candidate is executed in jsdom
+ *      against the task's fixture and its DOM effect signature compared to the
+ *      reference's, byte for byte.
+ *
+ * Keeping them separate is the point of the benchmark rather than an
+ * implementation detail: hyperscript's parser degrades instead of failing, so a
+ * candidate can parse at confidence 1.0 with zero diagnostics and still target
+ * the wrong element (a dropped `on` silently rebinds the destination to `me`).
+ * A single "success rate" would hide exactly the failure mode the loop needs to
+ * catch, so `parseRate` and `behaviorRate` are reported side by side and their
+ * GAP is a headline number in its own right.
+ *
+ * Determinism: fresh JSDOM + fresh Runtime per execution, no network, no timers
+ * beyond a fixed settle window. Effect-signature primitives are imported from
+ * ../multilingual/effect-signature so this harness and the R2 ratchet can never
+ * disagree about what a DOM effect is.
+ */
+
+import { JSDOM } from 'jsdom';
+import { parseSemantic, buildAST } from '@lokascript/semantic';
+import { snapshot, diffSnapshots } from '../multilingual/effect-signature.js';
+import { SHARED_FIXTURE, type BenchTask } from './tasks.js';
+
+/** Settle window for the dispatched handler. No task waits/fetches. */
+const SETTLE_MS = 20;
+/** Per-execution hard timeout — a hung candidate must not hang the run. */
+const EXECUTION_TIMEOUT_MS = 5000;
+
+export interface Diagnostic {
+  severity?: string;
+  code?: string;
+  message?: string;
+  suggestion?: string;
+}
+
+export interface ValidationOutcome {
+  ok: boolean;
+  confidence?: number | undefined;
+  diagnostics: Diagnostic[];
+  /** Flattened `action.role=value` view of the parse, for eyeballing intent drift. */
+  summary?: string | undefined;
+}
+
+export interface ExecutionOutcome {
+  effects: string[];
+  error?: string | undefined;
+}
+
+export interface TaskScore {
+  taskId: string;
+  code: string;
+  /** CompilationService said ok — what `validate_and_compile` reports. */
+  parsed: boolean;
+  /** Effect signature identical to the reference's. */
+  behaviorMatch: boolean;
+  /** parsed && !behaviorMatch && no visible diagnostic — the band the loop cannot see. */
+  silentlyWrong: boolean;
+  validation: ValidationOutcome;
+  execution: ExecutionOutcome;
+  referenceEffects: string[];
+}
+
+/**
+ * The five outcome bands, ordered best → worst. One function, used by the
+ * probe, the JSON baseline, and the ratchet test alike — a fork between them
+ * would let the ratchet pass against a baseline the probe can no longer
+ * produce.
+ *
+ * `warned-*`: wrong behavior, but validation carried a warning/error-severity
+ * diagnostic — VISIBLE to the loop, which can react (arc 3b's unconsumed-input
+ * propagation moves rows from silent-* to here). `silent-*`: wrong behavior
+ * and nothing to react to; the band that bounds the loop's ceiling.
+ */
+export type Band = 'correct' | 'rejected' | 'warned-wrong' | 'silent-wrong' | 'silent-noop';
+
+export function bandOf(score: TaskScore): Band {
+  if (score.behaviorMatch) return 'correct';
+  if (!score.parsed) return 'rejected';
+  const visible = score.validation.diagnostics.some(
+    d => d.severity === 'warning' || d.severity === 'error'
+  );
+  if (visible) return 'warned-wrong';
+  return score.execution.effects.length === 0 ? 'silent-noop' : 'silent-wrong';
+}
+
+export interface ConditionScore {
+  condition: string;
+  scores: TaskScore[];
+  parseRate: number;
+  behaviorRate: number;
+  /** Tasks that parsed but behaved differently from the reference. */
+  silentlyWrongCount: number;
+  /** Tasks with no candidate submitted for this condition. */
+  missing: string[];
+}
+
+// =============================================================================
+// Validation (the agent-visible surface)
+// =============================================================================
+
+let servicePromise: Promise<any> | null = null;
+
+async function getService(): Promise<any> {
+  if (!servicePromise) {
+    servicePromise = import('@lokascript/compilation-service').then(m =>
+      m.CompilationService.create()
+    );
+  }
+  return servicePromise;
+}
+
+/** Render a parse as `action(role=value, …)` so intent drift is readable. */
+function summarize(semantic: unknown): string | undefined {
+  const node = semantic as { action?: string; roles?: Record<string, { value?: unknown }> };
+  if (!node?.action) return undefined;
+  const roles = Object.entries(node.roles ?? {})
+    .map(([k, v]) => `${k}=${String(v?.value ?? '')}`)
+    .sort()
+    .join(', ');
+  return `${node.action}(${roles})`;
+}
+
+export async function validateCandidate(code: string): Promise<ValidationOutcome> {
+  try {
+    const service = await getService();
+    const result = service.validate({ code, language: 'en' });
+    return {
+      ok: Boolean(result?.ok),
+      confidence: result?.confidence,
+      diagnostics: (result?.diagnostics ?? []) as Diagnostic[],
+      summary: summarize(result?.semantic),
+    };
+  } catch (e: unknown) {
+    return {
+      ok: false,
+      diagnostics: [
+        { severity: 'error', code: 'HARNESS', message: e instanceof Error ? e.message : String(e) },
+      ],
+    };
+  }
+}
+
+// =============================================================================
+// Execution (what it actually does)
+// =============================================================================
+
+function buildDocument(task: BenchTask): JSDOM {
+  return new JSDOM(`<!DOCTYPE html><html><body>${SHARED_FIXTURE}${task.fixture}</body></html>`);
+}
+
+function installGlobals(dom: JSDOM): void {
+  const g = globalThis as Record<string, unknown>;
+  g.window = dom.window;
+  g.document = dom.window.document;
+  g.Event = dom.window.Event;
+  g.CustomEvent = dom.window.CustomEvent;
+  g.HTMLElement = dom.window.HTMLElement;
+  g.Element = dom.window.Element;
+  g.Node = dom.window.Node;
+  g.MutationObserver = dom.window.MutationObserver;
+  g.getComputedStyle = dom.window.getComputedStyle;
+}
+
+let core: {
+  Runtime: new () => { execute(ast: unknown, ctx: unknown): Promise<unknown> };
+  createContext: (el: HTMLElement) => unknown;
+} | null = null;
+
+let listenerErrors: string[] = [];
+let trapInstalled = false;
+
+/** Bootstraps jsdom globals BEFORE loading core (its dist evaluates `Element`). */
+export async function initialize(): Promise<void> {
+  if (core) return;
+  installGlobals(new JSDOM('<!DOCTYPE html><html><body></body></html>'));
+  const mod = (await import('@hyperfixi/core')) as unknown as {
+    Runtime: new () => { execute(ast: unknown, ctx: unknown): Promise<unknown> };
+    createContext: (el: HTMLElement) => unknown;
+  };
+  core = { Runtime: mod.Runtime, createContext: mod.createContext };
+  if (!trapInstalled) {
+    // Handler bodies are async, so a throw inside one surfaces as an unhandled
+    // rejection rather than propagating to our await.
+    process.on('unhandledRejection', (reason: unknown) => {
+      listenerErrors.push(reason instanceof Error ? reason.message : String(reason));
+    });
+    trapInstalled = true;
+  }
+}
+
+async function executeInner(task: BenchTask, code: string): Promise<ExecutionOutcome> {
+  await initialize();
+  const dom = buildDocument(task);
+  installGlobals(dom);
+  const document = dom.window.document;
+  task.setup?.(document);
+  const btn = document.getElementById('btn')!;
+  listenerErrors = [];
+
+  // The runtime logs every failing command; across a full run that is noise.
+  const saved = { log: console.log, warn: console.warn, error: console.error };
+  console.log = console.warn = console.error = () => {};
+  try {
+    const parsed = parseSemantic(code, 'en') as { node?: unknown; confidence?: number };
+    if (!parsed.node || (parsed.confidence ?? 0) < 0.5) {
+      return { effects: [], error: `parse failed (confidence ${parsed.confidence ?? 0})` };
+    }
+    const built = buildAST(parsed.node as never) as { ast?: unknown };
+    if (!built.ast) return { effects: [], error: 'buildAST returned no AST' };
+
+    const runtime = new core!.Runtime();
+    const ctx = core!.createContext(btn as unknown as HTMLElement);
+    await runtime.execute(built.ast, ctx);
+
+    const before = snapshot(document);
+    const trig = task.trigger ?? { event: 'click' };
+    const target = trig.selector ? document.querySelector(trig.selector) : btn;
+    if (!target) return { effects: [], error: `trigger selector ${trig.selector} matched nothing` };
+    const event =
+      trig.detail !== undefined
+        ? new dom.window.CustomEvent(trig.event, { bubbles: true, detail: trig.detail })
+        : new dom.window.Event(trig.event, { bubbles: true });
+    target.dispatchEvent(event);
+    await new Promise(r => setTimeout(r, SETTLE_MS));
+
+    const effects = diffSnapshots(before, snapshot(document));
+    return listenerErrors.length > 0
+      ? { effects, error: `runtime: ${listenerErrors.join('; ')}` }
+      : { effects };
+  } catch (e: unknown) {
+    return { effects: [], error: e instanceof Error ? e.message : String(e) };
+  } finally {
+    console.log = saved.log;
+    console.warn = saved.warn;
+    console.error = saved.error;
+    dom.window.close();
+  }
+}
+
+/** Execute one candidate against a task's fixture. Never throws. */
+export async function executeCandidate(task: BenchTask, code: string): Promise<ExecutionOutcome> {
+  return Promise.race([
+    executeInner(task, code),
+    new Promise<ExecutionOutcome>(resolve =>
+      setTimeout(
+        () => resolve({ effects: [], error: `execution timed out (${EXECUTION_TIMEOUT_MS}ms)` }),
+        EXECUTION_TIMEOUT_MS
+      )
+    ),
+  ]);
+}
+
+// =============================================================================
+// Scoring
+// =============================================================================
+
+function sameEffects(a: string[], b: string[]): boolean {
+  return a.length === b.length && a.every((v, i) => v === b[i]);
+}
+
+/**
+ * Reference signatures are pure functions of (task, runtime) and every score
+ * needs one, so scoring N candidates for a task would otherwise execute its
+ * reference N times. Memoized per process; `initialize()` state is per-process
+ * too, so the cache can never outlive the runtime it describes.
+ */
+const referenceCache = new Map<string, string[]>();
+
+export async function referenceEffectsFor(task: BenchTask): Promise<string[]> {
+  const hit = referenceCache.get(task.id);
+  if (hit) return hit;
+  const effects = (await executeCandidate(task, task.reference)).effects;
+  referenceCache.set(task.id, effects);
+  return effects;
+}
+
+export async function scoreCandidate(task: BenchTask, code: string): Promise<TaskScore> {
+  const referenceEffects = await referenceEffectsFor(task);
+  const validation = await validateCandidate(code);
+  const execution = await executeCandidate(task, code);
+  const behaviorMatch =
+    referenceEffects.length > 0 && sameEffects(execution.effects, referenceEffects);
+  const score: TaskScore = {
+    taskId: task.id,
+    code,
+    parsed: validation.ok,
+    behaviorMatch,
+    silentlyWrong: false,
+    validation,
+    execution,
+    referenceEffects,
+  };
+  score.silentlyWrong = bandOf(score).startsWith('silent');
+  return score;
+}
+
+export async function scoreCondition(
+  condition: string,
+  candidates: Record<string, string>,
+  tasks: readonly BenchTask[]
+): Promise<ConditionScore> {
+  const scores: TaskScore[] = [];
+  const missing: string[] = [];
+  for (const task of tasks) {
+    const code = candidates[task.id];
+    if (code === undefined) {
+      missing.push(task.id);
+      continue;
+    }
+    scores.push(await scoreCandidate(task, code));
+  }
+  // Missing candidates count as failures in the denominator: a condition that
+  // simply declines to answer must not out-score one that tries and misses.
+  const denom = tasks.length;
+  return {
+    condition,
+    scores,
+    parseRate: scores.filter(s => s.parsed).length / denom,
+    behaviorRate: scores.filter(s => s.behaviorMatch).length / denom,
+    silentlyWrongCount: scores.filter(s => s.silentlyWrong).length,
+    missing,
+  };
+}

package/src/agent-bench/tasks.ts

@@ -0,0 +1,214 @@
+/**
+ * Agent-loop benchmark — the task corpus.
+ *
+ * Twenty natural-language UI requests, phrased the way a user would put them to
+ * an agent (no hyperscript syntax leaks into a prompt — that would hand the
+ * generator the answer). Each carries a reference implementation whose jsdom
+ * effect signature defines "behaviorally correct" for that task.
+ *
+ * Eligibility bar, same as R2's execution subset: a task is only usable if its
+ * reference PARSES and produces a NON-EMPTY effect signature in the current
+ * runtime. `cli.ts verify-references` enforces both, so a task whose reference
+ * rots (or whose fixture stops matching) fails loudly rather than scoring every
+ * candidate against an empty signature — which would make wrong answers look
+ * right.
+ *
+ * Seeded from the concepts in the gallery examples and the patterns corpus, but
+ * deliberately NOT copied from them verbatim: the corpus text is what the
+ * pattern database (and therefore `search_patterns`) already contains, so
+ * reusing it would measure recall of the corpus rather than generation.
+ */
+
+/** Trigger to dispatch after the handler is installed. */
+export interface BenchTrigger {
+  event: string;
+  /** Element to dispatch on (default `#btn`). */
+  selector?: string;
+  /** CustomEvent detail; when set, a CustomEvent is dispatched instead of Event. */
+  detail?: unknown;
+}
+
+export interface BenchTask {
+  id: string;
+  /** The request as a user would phrase it. No hyperscript in here. */
+  prompt: string;
+  /** Known-good hyperscript satisfying the prompt; defines the target behavior. */
+  reference: string;
+  /** Markup appended inside <body>, after the shared button. */
+  fixture: string;
+  /** Fixture preconditions applied before the handler is installed. */
+  setup?: (doc: Document) => void;
+  /** Defaults to a bubbling `click` on `#btn`. */
+  trigger?: BenchTrigger;
+  /** What the task probes — used to group the report. */
+  tags: readonly string[];
+}
+
+/** Present in every fixture; the default trigger target. */
+export const SHARED_FIXTURE = '<div class="card"><button id="btn">Click</button></div>';
+
+export const TASKS: readonly BenchTask[] = [
+  {
+    id: 'toggle-self-class',
+    prompt: 'When the button is clicked, toggle the CSS class "active" on the button itself.',
+    reference: 'on click toggle .active on me',
+    fixture: '',
+    tags: ['toggle', 'self-reference'],
+  },
+  {
+    id: 'toggle-other-class',
+    prompt:
+      'When the button is clicked, toggle the CSS class "open" on the element whose id is "panel".',
+    reference: 'on click toggle .open on #panel',
+    fixture: '<div id="panel">panel</div>',
+    tags: ['toggle', 'destination'],
+  },
+  {
+    id: 'add-class-other',
+    prompt:
+      'When the button is clicked, add the CSS class "highlight" to the element whose id is "item".',
+    reference: 'on click add .highlight to #item',
+    fixture: '<div id="item">item</div>',
+    tags: ['add', 'destination'],
+  },
+  {
+    id: 'remove-class-all',
+    prompt:
+      'When the button is clicked, remove the CSS class "active" from every element that has the class "row".',
+    reference: 'on click remove .active from .row',
+    fixture: '<div class="row active">a</div><div class="row active">b</div>',
+    tags: ['remove', 'multi-target'],
+  },
+  {
+    id: 'put-text',
+    prompt:
+      'When the button is clicked, replace the contents of the element with id "output" with the text Saved.',
+    reference: 'on click put "Saved" into #output',
+    fixture: '<div id="output">old</div>',
+    tags: ['put', 'literal'],
+  },
+  {
+    id: 'hide-element',
+    prompt: 'When the button is clicked, hide the element whose id is "menu".',
+    reference: 'on click hide #menu',
+    fixture: '<div id="menu">menu</div>',
+    tags: ['hide'],
+  },
+  {
+    id: 'show-element',
+    prompt: 'When the button is clicked, make the hidden element with id "modal" visible again.',
+    reference: 'on click show #modal',
+    fixture: '<div id="modal" style="display: none">modal</div>',
+    tags: ['show'],
+  },
+  {
+    id: 'two-commands',
+    prompt:
+      'When the button is clicked, add the class "busy" to the button and also put the text Loading into the element with id "output".',
+    reference: 'on click add .busy to me then put "Loading" into #output',
+    fixture: '<div id="output">idle</div>',
+    tags: ['sequence', 'multi-command'],
+  },
+  {
+    id: 'closest-ancestor',
+    prompt:
+      'When the button is clicked, add the class "selected" to the nearest enclosing element that has the class "card".',
+    reference: 'on click add .selected to closest .card',
+    fixture: '',
+    tags: ['add', 'positional', 'closest'],
+  },
+  {
+    id: 'remove-element',
+    prompt: 'When the button is clicked, delete the element with id "item" from the page.',
+    reference: 'on click remove #item',
+    fixture: '<div id="item">doomed</div>',
+    tags: ['remove', 'element-removal'],
+  },
+  {
+    id: 'toggle-attribute',
+    prompt:
+      'When the button is clicked, toggle the "hidden" attribute on the element with id "message".',
+    reference: 'on click toggle @hidden on #message',
+    fixture: '<div id="message">msg</div>',
+    tags: ['toggle', 'attribute'],
+  },
+  {
+    id: 'set-attribute',
+    prompt:
+      'When the button is clicked, set the aria-expanded attribute of the element with id "panel" to true.',
+    // NB: `set @aria-expanded of #panel to "true"` — the phrasing an LLM reaches
+    // for first — parses clean and does NOTHING. The possessive form is the one
+    // that works. See runs/README findings.
+    reference: 'on click set #panel\'s @aria-expanded to "true"',
+    fixture: '<div id="panel" aria-expanded="false">panel</div>',
+    tags: ['set', 'attribute'],
+  },
+  {
+    id: 'set-style',
+    prompt:
+      'When the button is clicked, change the inline background colour of the element with id "swatch" to red.',
+    reference: 'on click set *background-color of #swatch to "red"',
+    fixture: '<div id="swatch">swatch</div>',
+    tags: ['set', 'style'],
+  },
+  {
+    id: 'set-inner-html',
+    prompt:
+      'When the button is clicked, set the innerHTML of the element with id "output" to the text Done.',
+    reference: 'on click set #output\'s innerHTML to "Done"',
+    fixture: '<div id="output">old</div>',
+    tags: ['set', 'possessive', 'property'],
+  },
+  {
+    id: 'add-class-multiple',
+    prompt:
+      'When the button is clicked, add the class "done" to every element that has the class "todo".',
+    reference: 'on click add .done to .todo',
+    fixture: '<li class="todo">a</li><li class="todo">b</li>',
+    tags: ['add', 'multi-target'],
+  },
+  {
+    id: 'mouseenter-hover',
+    prompt: 'When the mouse pointer enters the button, add the class "hover" to it.',
+    reference: 'on mouseenter add .hover to me',
+    fixture: '',
+    trigger: { event: 'mouseenter' },
+    tags: ['event', 'non-click-trigger'],
+  },
+  {
+    id: 'tabs-switch',
+    prompt:
+      'When the button is clicked, remove the class "active" from all elements with class "tab", then add that class to the element with id "tab2".',
+    reference: 'on click remove .active from .tab then add .active to #tab2',
+    fixture: '<div class="tab active" id="tab1">1</div><div class="tab" id="tab2">2</div>',
+    tags: ['sequence', 'multi-target', 'multi-command'],
+  },
+  {
+    id: 'conditional-class',
+    prompt:
+      'When the button is clicked, add the class "warned" to the element with id "box" only if that element already has the class "danger".',
+    reference: 'on click if #box matches .danger add .warned to #box end',
+    fixture: '<div id="box" class="danger">box</div>',
+    tags: ['conditional', 'if'],
+  },
+  {
+    id: 'custom-event',
+    prompt:
+      'When the button receives a custom event named "refresh", put the text Refreshed into the element with id "output".',
+    reference: 'on refresh put "Refreshed" into #output',
+    fixture: '<div id="output">stale</div>',
+    trigger: { event: 'refresh' },
+    tags: ['event', 'custom-event'],
+  },
+  {
+    id: 'add-to-body',
+    prompt: 'When the button is clicked, add the class "modal-open" to the page body.',
+    reference: 'on click add .modal-open to body',
+    fixture: '',
+    tags: ['add', 'body-target'],
+  },
+] as const;
+
+export function taskById(id: string): BenchTask | undefined {
+  return TASKS.find(t => t.id === id);
+}