@ggui-ai/ui-gen 0.20.0 → 0.21.0

package/src/evaluation/visual-evaluator.ts

@@ -24,7 +24,7 @@ import { createHash } from 'node:crypto';
 import { tmpdir } from 'os';
 import { createVisionAgent, type AgentConfig } from '../harness/llm-router';
 import type { EvaluationResult, EvaluationIssue, DimensionScores } from './types';
-import type { CanvasJudgeRecord, CanvasVisualSummary, EvalIssue, VisualEvalSummary } from './types-public.js';
+import type { CanvasJudgeRecord, CanvasVisualSummary, EvalIssue, VisualCoverage, VisualEvalSummary, VisualFitStamp } from './types-public.js';
 import type { LaunchOptions } from 'puppeteer-core';
 import { CANVAS_VIEWPORTS, displayModeForCanvas, type CanvasClass, type CanvasViewport } from '../design-mode.js';
 
@@ -55,6 +55,20 @@ export interface VisualEvalConfig {
    * single screenshot at `viewport`; no `canvases` field on the result.
    */
   canvases?: readonly CanvasClass[];
+  /**
+   * ggui#1195 — the box a canvas is COMPOSED for and CAPTURED at when the
+   * order declared one (the chat card at the visitor's real column, not the
+   * class's 400×640). Keyed by canvas; a canvas without an entry is judged
+   * at `CANVAS_VIEWPORTS[canvas]` as before. The composer's
+   * `rendering.viewport` and this entry are ONE value written once (the
+   * seam derives it), so the judge never scores a box the mint did not
+   * target. Only read in per-canvas mode. Captured at WHOLE CSS px: the
+   * browser's device-metrics override takes integers (measured — a 383.5 px
+   * width is refused by CDP), so a fractional declared box is rounded once
+   * where it enters and the capture, the ceiling and the reported
+   * `viewport` all carry that same integer box.
+   */
+  canvasViewports?: Partial<Readonly<Record<CanvasClass, CanvasViewport>>>;
   /**
    * Optional provider-routing override — see
    * `AgentConfig.routeOverride`. Threaded onto the agent this
@@ -110,6 +124,8 @@ export interface CanvasVisualResult {
   judge: CanvasJudgeRecord;
   /** How the judge composed the mount (ggui#1100): `'fill'` on every fullscreen canvas, absent on the inline card. */
   fit?: 'fill';
+  /** ggui#1195 — `true` when the order DECLARED this canvas's box (`config.canvasViewports` carried an entry), whatever its value. */
+  declared?: true;
 }
 
 /** How a canvas is captured for the judge and what an overflow means there (ggui#1027). */
@@ -158,6 +174,23 @@ export function canvasFit(canvas: CanvasClass): 'fill' | undefined {
   return displayModeForCanvas(canvas) === 'fullscreen' ? 'fill' : undefined;
 }
 
+/**
+ * The box a browser can be asked for: whole CSS px. CDP's
+ * `Emulation.setDeviceMetricsOverride` deserialises width/height as int32 and
+ * refuses a fraction (measured on Chrome: `383.5` → "int32 value expected"),
+ * while a host measuring its chat column with `getBoundingClientRect()` may
+ * legitimately declare one. Rounded once, at the judge's boundary.
+ */
+export function integerBox(viewport: { width: number; height: number }): { width: number; height: number } {
+  return { width: Math.round(viewport.width), height: Math.round(viewport.height) };
+}
+
+/** ggui#1195 — `true` when a canvas was judged at a box other than its class viewport (the order declared one). */
+export function isDeclaredViewport(canvas: CanvasClass, viewport: CanvasViewport): boolean {
+  const classBox = CANVAS_VIEWPORTS[canvas];
+  return viewport.width !== classBox.width || viewport.height !== classBox.height;
+}
+
 /** The deterministic fit issue — one per overflowing canvas, in the judge's issue shape so it rides the same channel. */
 export function canvasOverflowIssue(
   canvas: CanvasClass,
@@ -166,11 +199,18 @@ export function canvasOverflowIssue(
   verdict: 'fail' | 'warn',
 ): EvaluationIssue {
   const hidden = contentHeight - viewport.height;
+  // ggui#1195 — when the judge captured at a DECLARED box, the refusal names
+  // both boxes: the ceiling the content was measured against and the class
+  // box a reader would otherwise assume.
+  const classBox = CANVAS_VIEWPORTS[canvas];
+  const box = isDeclaredViewport(canvas, viewport)
+    ? `declared ${viewport.width}×${viewport.height}; class box ${classBox.width}×${classBox.height}`
+    : `${viewport.width}×${viewport.height}`;
   return {
     dimension: 'canvas-overflow',
     severity: verdict === 'fail' ? 'critical' : 'major',
     description:
-      `Rendered content is ${contentHeight}px tall on the ${canvas} canvas (${viewport.width}×${viewport.height}) — ` +
+      `Rendered content is ${contentHeight}px tall on the ${canvas} canvas (${box}) — ` +
       `${hidden}px ${verdict === 'fail' ? 'is cut off: the inline card does not scroll' : 'sits below the first screen'}.`,
     fix:
       'Fit the composition to the canvas: fewer and shorter sections, one compact row of chips, no hero taller than ' +
@@ -427,7 +467,10 @@ function buildRenderHTML(
     *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
     body {
       font-family: var(--ggui-font-family-sans, system-ui, -apple-system, sans-serif);
-      background: var(--ggui-color-neutral-50, #ffffff);
+      /* ggui#1083 — the host stand-in (neutral-50) under the theme's scrim (its tint at
+         its opacity): the ground a frosted host puts under the card, so the judge scores
+         the card where the visitor sees it. */
+      background: color-mix(in oklch, var(--ggui-scrim-tint, var(--ggui-color-ground, #ffffff)) calc(var(--ggui-scrim-opacity, 0.45) * 100%), var(--ggui-color-neutral-50, #ffffff));
       color: var(--ggui-color-neutral-900, #111827);
     }
     .error { color: #dc2626; padding: 16px; font-family: monospace; white-space: pre-wrap; }
@@ -523,18 +566,20 @@ export async function resolveLaunchOptions(
 ): Promise<LaunchOptions> {
   const env = deps.env ?? process.env;
   const override = env.PUPPETEER_EXECUTABLE_PATH?.trim();
+  // Whole px at the one place `defaultViewport` is built — the single-shot path takes the caller's box verbatim.
+  const defaultViewport = integerBox(viewport);
   if (override) {
     return {
       executablePath: override,
       args: [...EXECUTABLE_PATH_LAUNCH_ARGS],
-      defaultViewport: viewport,
+      defaultViewport,
       headless: true,
     };
   }
   const chromium = await (deps.loadChromium ?? loadSparticuzChromium)();
   return {
     args: chromium.args,
-    defaultViewport: viewport,
+    defaultViewport,
     executablePath: await chromium.executablePath(),
     headless: true,
   };
@@ -742,6 +787,51 @@ export interface VisualEvalDetailed {
 }
 
 
+/**
+ * One canvas, framed: the box it is captured at (ggui#1195 — the DECLARED box
+ * when the order carried one, else the class box), the canvas's fit policy,
+ * the runtime's fit for the mount (ggui#1100 — fullscreen canvases fill, the
+ * inline card does not) and the capture. The judge path and the fit-only path
+ * ({@link runVisualFit}) frame through this ONE function, so a fit verdict
+ * reached without the judge is taken on the frame the judge would have seen.
+ */
+interface CanvasFrame {
+  readonly viewport: CanvasViewport;
+  readonly policy: CanvasFitPolicy;
+  readonly fit: 'fill' | undefined;
+  /** `true` when `canvasViewports` carried an entry for this canvas, whatever its value. */
+  readonly declared: boolean;
+  readonly attempt: ScreenshotAttempt;
+}
+
+async function frameCanvas(
+  bundledCode: string,
+  context: VisualEvalContext,
+  html: string,
+  canvas: CanvasClass,
+  canvasViewports: VisualEvalConfig['canvasViewports'],
+  deps: ScreenshotDeps,
+): Promise<CanvasFrame> {
+  const declaredBox = canvasViewports?.[canvas];
+  const viewport = declaredBox !== undefined ? integerBox(declaredBox) : CANVAS_VIEWPORTS[canvas];
+  const policy = canvasFitPolicy(canvas);
+  const fit = canvasFit(canvas);
+  const canvasHtml = fit !== undefined ? buildRenderHTML(bundledCode, context.cssTokens, fit) : html;
+  const attempt = await captureScreenshotDetailed(canvasHtml, viewport, deps, policy.capture);
+  return { viewport, policy, fit, declared: declaredBox !== undefined, attempt };
+}
+
+/**
+ * The deterministic fit verdict on one frame (ggui#1027): the issue the
+ * canvas's policy gives an overflow, or `null` when the content fits, could
+ * not be measured, or the canvas scrolls by design.
+ */
+function fitVerdict(canvas: CanvasClass, frame: CanvasFrame): EvaluationIssue | null {
+  const contentHeight = frame.attempt.contentHeight;
+  if (contentHeight === null || contentHeight <= frame.viewport.height || frame.policy.overflow === 'none') return null;
+  return canvasOverflowIssue(canvas, frame.viewport, contentHeight, frame.policy.overflow);
+}
+
 /** One judge call parsed, retried ONCE on a malformed answer (#1017); the FIRST reason is kept verbatim. */
 type JudgedAnswer =
   | { readonly kind: 'ok'; readonly result: EvaluationResult; readonly inputTokens: number; readonly outputTokens: number }
@@ -848,12 +938,8 @@ export async function runVisualEvaluationDetailed(
     const perCanvas: CanvasVisualResult[] = [];
     const perCanvasResults: EvaluationResult[] = [];
     for (const canvas of config.canvases) {
-      const viewport = CANVAS_VIEWPORTS[canvas];
-      const policy = canvasFitPolicy(canvas);
-      // ggui#1100: the page is composed per canvas with the runtime's fit — fullscreen canvases fill, the inline card does not.
-      const fit = canvasFit(canvas);
-      const canvasHtml = fit !== undefined ? buildRenderHTML(bundledCode, context.cssTokens, fit) : html;
-      const attempt = await captureScreenshotDetailed(canvasHtml, viewport, deps, policy.capture);
+      const frame = await frameCanvas(bundledCode, context, html, canvas, config.canvasViewports, deps);
+      const { viewport, policy, fit, attempt } = frame;
       const screenshot = attempt.png;
       if (!screenshot) {
         const unavailableReason = attempt.reason ?? 'no browser available';
@@ -890,8 +976,9 @@ export async function runVisualEvaluationDetailed(
       // The fit verdict (ggui#1027): deterministic, in the judge's issue channel.
       const contentHeight = attempt.contentHeight;
       const overflow = contentHeight !== null && contentHeight > viewport.height;
-      if (contentHeight !== null && contentHeight > viewport.height && policy.overflow !== 'none') {
-        result.issues.push(canvasOverflowIssue(canvas, viewport, contentHeight, policy.overflow));
+      const fitIssue = fitVerdict(canvas, frame);
+      if (fitIssue !== null) {
+        result.issues.push(fitIssue);
         if (policy.overflow === 'fail') result.passed = false;
       }
       perCanvasResults.push(result);
@@ -905,6 +992,7 @@ export async function runVisualEvaluationDetailed(
         overflow,
         judge: judgeRecord,
         ...(fit !== undefined ? { fit } : {}),
+        ...(frame.declared ? { declared: true as const } : {}),
       });
       console.log(
         `[visual-eval] canvas=${canvas} ${viewport.width}×${viewport.height} score=${result.finalScore}${k > 1 ? ` (median of ${judgeRecord.samples.length}/${k}: ${judgeRecord.samples.join(',')} σ=${judgeRecord.sigma})` : ''} ` +
@@ -960,6 +1048,11 @@ export async function runVisualEvaluationDetailed(
  * per-dimension means, `issues` = union (description prefixed with the
  * canvas), `critique` = one line per canvas, tokens summed.
  */
+/** A canvas's issue as the aggregate carries it: the description prefixed `[<canvas>]`. */
+function canvasScopedIssue(canvas: CanvasClass, issue: EvaluationIssue): EvaluationIssue {
+  return { ...issue, description: `[${canvas}] ${issue.description}` };
+}
+
 function aggregateCanvasResults(
   perCanvas: readonly CanvasVisualResult[],
   results: readonly EvaluationResult[],
@@ -980,7 +1073,7 @@ function aggregateCanvasResults(
   let outputTokens = 0;
   results.forEach((r, i) => {
     const canvas = perCanvas[i]!.canvas;
-    for (const issue of r.issues) issues.push({ ...issue, description: `[${canvas}] ${issue.description}` });
+    for (const issue of r.issues) issues.push(canvasScopedIssue(canvas, issue));
     if (r.critique) critiques.push(`${canvas}: ${r.critique}`);
     inputTokens += r.inputTokens ?? 0;
     outputTokens += r.outputTokens ?? 0;
@@ -1010,10 +1103,24 @@ export function summarizeVisualResult(result: VisualEvaluationResult): VisualEva
     judge: c.judge,
     ...(c.fit !== undefined ? { fit: c.fit } : {}),
   }));
+  // ggui#1195 — the fit stamp: the first canvas whose policy FAILS an
+  // overflow (the inline card), judged with a measurable height.
+  const fitCanvas = result.canvases.find((c) => canvasFitPolicy(c.canvas).overflow === 'fail' && c.contentHeight !== null);
+  const fit: VisualFitStamp | undefined =
+    fitCanvas !== undefined && fitCanvas.contentHeight !== null
+      ? {
+          canvas: fitCanvas.canvas,
+          ceiling: { width: fitCanvas.viewport.width, height: fitCanvas.viewport.height },
+          // Declared = the order carried a box, even one equal to the class box (the review's point).
+          declared: fitCanvas.declared === true,
+          overflowPx: Math.max(0, fitCanvas.contentHeight - fitCanvas.viewport.height),
+        }
+      : undefined;
   return {
     score: result.finalScore,
     passed: result.passed,
     canvases,
+    ...(fit !== undefined ? { fit } : {}),
     ...(result.design !== undefined ? { design: result.design } : {}),
     ...(result.themeMode !== undefined ? { themeMode: result.themeMode } : {}),
   };
@@ -1148,6 +1255,12 @@ export interface VisualEvalOutcome {
    * the single-shot path (byte-identical to the pre-canvas result).
    */
   summary?: VisualEvalSummary;
+  /**
+   * ggui#1221 — whether the judge leg ran or was skipped (with the reason
+   * `runVisualEvaluationDetailed` names). Always present: the absence of a
+   * summary used to be indistinguishable from a clean single-shot run.
+   */
+  coverage: VisualCoverage;
 }
 
 /**
@@ -1157,19 +1270,113 @@ export interface VisualEvalOutcome {
 export async function runVisualEval(
   context: VisualEvalContext,
   config: VisualEvalConfig,
+  deps: VisualEvalDeps = {},
 ): Promise<VisualEvalOutcome> {
-  const result = await runVisualEvaluation(context, config);
-  if (!result) return { issues: [] }; // puppeteer not available
+  const detailed = await runVisualEvaluationDetailed(context, config, deps);
+  const result = detailed.result;
+  if (!result) {
+    // ggui#1221 — no browser / launch failure / unbundlable / unparsable judge:
+    // the leg is SKIPPED, and says so on the outcome instead of leaving `{ issues: [] }`.
+    const reason = detailed.unavailableReason ?? 'no browser available';
+    return { issues: [], coverage: { status: 'skipped', reason } };
+  }
 
-  const issues: EvalIssue[] = (result.issues || []).map(issue => ({
-    tier: 2 as const,
-    result: (issue.severity === 'critical' ? 'fail' : 'warn') as 'fail' | 'warn',
-    category: 'visual' as const,
+  const issues: EvalIssue[] = (result.issues || []).map(toEvalIssue);
+  const summary = summarizeVisualResult(result);
+  const coverage: VisualCoverage = { status: 'ran' };
+  return summary === undefined ? { issues, coverage } : { issues, summary, coverage };
+}
+
+/** A judge-shaped issue as the harness's tier-2 issue: critical ⇒ a blocking fail, anything else ⇒ a warn. */
+function toEvalIssue(issue: EvaluationIssue): EvalIssue {
+  return {
+    tier: 2,
+    result: issue.severity === 'critical' ? 'fail' : 'warn',
+    category: 'visual',
     subcategory: issue.dimension,
-    severity: (issue.severity === 'critical' ? 'critical' : 'major') as 'critical' | 'major',
+    severity: issue.severity === 'critical' ? 'critical' : 'major',
     description: issue.description,
     fix: issue.fix || '',
-  }));
-  const summary = summarizeVisualResult(result);
-  return summary === undefined ? { issues } : { issues, summary };
+  };
+}
+
+// ---------------------------------------------------------------------------
+// The fit half without the judge
+// ---------------------------------------------------------------------------
+
+/** What the fit half reads — the frame's inputs; nothing about a judge. */
+export type VisualFitConfig = Pick<VisualEvalConfig, 'sampleProps' | 'canvases' | 'canvasViewports' | 'designSrcDir'>;
+
+/** One canvas's fit reading: the box, the measure, whether it overflowed — no score. */
+export interface CanvasFitReading {
+  readonly canvas: CanvasClass;
+  readonly viewport: CanvasViewport;
+  readonly contentHeight: number | null;
+  readonly overflow: boolean;
+  /** `true` when the order declared this canvas's box. */
+  readonly declared?: true;
+}
+
+/** `measured` = every canvas was framed (its fit issues, possibly none); `unavailable` = no verdict, with why. */
+export type VisualFitOutcome =
+  | { readonly status: 'measured'; readonly issues: EvalIssue[]; readonly readings: CanvasFitReading[] }
+  | { readonly status: 'unavailable'; readonly reason: string };
+
+/**
+ * The visual leg's deterministic half, taken WITHOUT the vision judge.
+ *
+ * The fit verdict needs a browser and nothing else: each canvas is framed at
+ * its box (the declared one when the order carried it), its scroll height is
+ * measured against the box's height, and the canvas's fit policy decides what
+ * an overflow earns — the same frame ({@link frameCanvas}), the same measure
+ * and the same issue ({@link fitVerdict}, canvas-scoped) as the judge path, so
+ * the two reach one verdict. What is absent is the score. A caller whose
+ * generation provider has no vision judge takes this half alone, so a chat
+ * card cut off at its declared box is still reported in the loop, where the
+ * round can buy the composition fix.
+ *
+ * Per-canvas only: without `canvases` there is no box to hold the content
+ * against, and the outcome is `unavailable`. A frame that cannot be taken is
+ * `unavailable` with the capture's reason — never a pass.
+ */
+export async function runVisualFit(
+  context: VisualEvalContext,
+  config: VisualFitConfig,
+  deps: ScreenshotDeps = {},
+): Promise<VisualFitOutcome> {
+  if (config.canvases === undefined || config.canvases.length === 0) {
+    return { status: 'unavailable', reason: 'no canvases — the fit verdict is taken per canvas' };
+  }
+  const designSrc = config.designSrcDir ?? resolve(resolveDesignPackageDir(), 'src');
+  let bundledCode: string;
+  try {
+    bundledCode = await bundleForRendering(context.compiledCode, config.sampleProps ?? {}, designSrc);
+  } catch (e) {
+    return { status: 'unavailable', reason: `bundle failed: ${e instanceof Error ? e.message : String(e)}` };
+  }
+  const html = buildRenderHTML(bundledCode, context.cssTokens);
+  const issues: EvalIssue[] = [];
+  const readings: CanvasFitReading[] = [];
+  for (const canvas of config.canvases) {
+    const frame = await frameCanvas(bundledCode, context, html, canvas, config.canvasViewports, deps);
+    if (!frame.attempt.png) {
+      return { status: 'unavailable', reason: `${frame.attempt.reason ?? 'no browser available'} at canvas ${canvas}` };
+    }
+    const contentHeight = frame.attempt.contentHeight;
+    const overflow = contentHeight !== null && contentHeight > frame.viewport.height;
+    const verdict = fitVerdict(canvas, frame);
+    if (verdict !== null) issues.push(toEvalIssue(canvasScopedIssue(canvas, verdict)));
+    readings.push({
+      canvas,
+      viewport: frame.viewport,
+      contentHeight,
+      overflow,
+      ...(frame.declared ? { declared: true as const } : {}),
+    });
+    console.log(
+      `[visual-fit] canvas=${canvas} ${frame.viewport.width}×${frame.viewport.height} ` +
+        `content=${contentHeight ?? '?'}px${overflow ? ` OVERFLOW (${frame.policy.overflow})` : ''} | no judge`,
+    );
+  }
+  return { status: 'measured', issues, readings };
 }

package/src/harness/check/runtime-render/adapter.ts

@@ -203,6 +203,15 @@ function toEvalIssue(issue: RenderCheckIssue): EvalIssue | null {
   if (diag?.actionsFiredFromClicks?.length) {
     diagParts.push(`other actions fired from clicks: ${diag.actionsFiredFromClicks.join(", ")}`);
   }
+  if (diag?.inputPriming) {
+    // ggui#1187: an input-gated action that "did not dispatch" reads
+    // differently beside "primed 0" / a priming error than beside "primed 1".
+    diagParts.push(
+      "primed" in diag.inputPriming
+        ? `input priming: ${diag.inputPriming.primed} control(s) filled before the click`
+        : `input priming failed: ${diag.inputPriming.error}`,
+    );
+  }
   const diagSuffix = diagParts.length ? ` [observed: ${diagParts.join("; ")}]` : "";
 
   switch (issue.check) {

package/src/harness/check/runtime-render/render-check.ts

@@ -99,9 +99,25 @@ export interface RenderCheckIssue {
     readonly actionsFiredFromClicks?: readonly string[];
     /** Surfaced to the agent: prop sourceTool hints from the contract (only if present). */
     readonly sourceToolHints?: Readonly<Record<string, string>>;
+    /**
+     * For action-wiring (ggui#1187): what input priming did before the
+     * synthetic dispatch — how many controls it filled, or why it could not
+     * run. A "did not dispatch it" on an input-gated action reads differently
+     * beside `{ primed: 0 }` or `{ error }` than beside `{ primed: 1 }`.
+     */
+    readonly inputPriming?: InputPrimingDiagnostic;
   };
 }
 
+/**
+ * Outcome of `primeInputs` before the action-wiring probe (ggui#1187). A
+ * priming failure never aborts the check — it is recorded here, on every
+ * action-wiring issue, so it is visible in the feedback instead of vanishing.
+ */
+export type InputPrimingDiagnostic =
+  | { readonly primed: number }
+  | { readonly error: string };
+
 export interface RenderCheckResult {
   readonly ok: boolean;
   readonly issues: readonly RenderCheckIssue[];
@@ -513,11 +529,16 @@ export async function runRenderCheckInProcess(
       // sets input VALUES (not textContent) and fires input/change before the
       // per-candidate fire-log window is opened, so it neither perturbs the
       // prop-sensitivity text baseline nor false-positives the wiring probe.
-      // Best-effort: a priming failure must never abort the check.
+      // A priming failure never aborts the check — it is RECORDED on every
+      // action-wiring issue as `diagnostics.inputPriming`. (The first cut
+      // swallowed it, and a `ReferenceError` in the check worker's realm —
+      // no `HTMLInputElement` on `globalThis` — hid behind that catch while
+      // every chat cell kept warning.)
+      let inputPriming: InputPrimingDiagnostic;
       try {
-        primeInputs(renderResult.container);
-      } catch {
-        // Un-primeable DOM falls back to the prior (un-primed) behavior.
+        inputPriming = { primed: primeInputs(renderResult.container) };
+      } catch (err) {
+        inputPriming = { error: err instanceof Error ? err.message : String(err) };
       }
       // ── Check 2: Action wiring (BLOCK / unverified-as-warn) ───────────
       if (input.contract?.actionSpec) {
@@ -537,7 +558,7 @@ export async function runRenderCheckInProcess(
             probe,
             user,
           });
-          if (issue) issues.push(issue);
+          if (issue) issues.push({ ...issue, diagnostics: { ...issue.diagnostics, inputPriming } });
         }
       }