@ggui-ai/ui-gen 0.20.0 → 0.21.0

package/src/harness/coding/run-eval-round.ts

@@ -22,13 +22,20 @@ import { listContractGadgets } from "@ggui-ai/protocol";
 import type { Classification } from "../../classifier/index.js";
 import type { AgentWorkspace } from "../../coding-agent/workspace.js";
 import type { CostTracker } from "../../evaluation/cost-tracker.js";
-import type { EvalIssue, EvalResult, RuntimeProbeMeta, VisualEvalSummary } from "../../evaluation/types-public.js";
+import type { EvalIssue, EvalResult, RuntimeProbeMeta, VisualEvalSummary, VisualCoverage } from "../../evaluation/types-public.js";
 import { notApplicableCoverage } from "../../evaluation/types-public.js";
-import { mapProviderForEvaluator } from "../enforced-coding.js";
+import { mapProviderForEvaluator, visionJudgeProvider } from "../enforced-coding.js";
 import { runCheck } from "../index.js";
 import { isRecoverableRenderCrash } from "../check/runtime-render/index.js";
 import type { Harness } from "../types-public.js";
 import type { AgentSpec, SingleComponentParams } from "../runtime.js";
+import type {
+  CanvasFitReading,
+  VisualEvalContext,
+  VisualEvalOutcome,
+  VisualFitConfig,
+  VisualFitOutcome,
+} from "../../evaluation/visual-evaluator.js";
 
 type PreWarmedEvalContext =
   import("../../evaluation/llm-evaluator.js").PreWarmedEvalContext;
@@ -122,6 +129,23 @@ export interface EvalRoundResult {
 /** Cap feedback to N issues per eval-fix turn so the LLM stays surgical. */
 const MAX_FEEDBACK_ISSUES = 3;
 
+/**
+ * ggui#1195 — the visual leg's deterministic fit verdict: the inline card's
+ * content is taller than the box it was judged at (`canvas-overflow`,
+ * critical ⇒ fail). Recognised by the marker the evaluator emits, never by
+ * the description's wording.
+ */
+function isFitFail(issue: EvalIssue): boolean {
+  return issue.result === "fail" && issue.category === "visual" && issue.subcategory === "canvas-overflow";
+}
+/** The same marker as {@link isFitFail}, on the fingerprint `fingerprintFail` builds (`category|subcategory|…`). */
+const FIT_FINGERPRINT_PREFIX = "visual|canvas-overflow|";
+
+/** `[fit]`-tagged so the LLM can see the violation is the box, not the judge's taste. */
+function formatFitFeedback(issue: EvalIssue): string {
+  return `[fit] ${issue.category}/${issue.subcategory ?? ""}: ${issue.description}\n  Fix: ${issue.fix ?? ""}`;
+}
+
 /**
  * `criteriaCoverage` reason stamped on the same-image low-risk bypass
  * exits: tier-1/2 evaluation is never invoked there BY DESIGN, so every
@@ -131,6 +155,65 @@ const MAX_FEEDBACK_ISSUES = 3;
 /** Why an eval result carries no criteria: the round itself threw (ggui#1110). */
 const EVAL_ROUND_THREW_REASON = "eval round threw";
 
+/**
+ * The in-loop visual leg, made total (ggui#1248). The judge runs on a VISION
+ * provider: the agent's provider narrowed by `visionJudgeProvider`, never
+ * asserted into the vision union — a lane whose generation provider has no
+ * vision path (OpenAI, OpenRouter) gets a leg recorded as SKIPPED with the
+ * reason, and the judge is not called. And whatever the leg does, it resolves:
+ * a rejection becomes the same skipped coverage with the error, so the round
+ * never takes its thrown path on account of the visual leg and the runtime
+ * probe still runs — a bar that requires the probe decides on the probe, not
+ * on a judge that could not be reached.
+ *
+ * The leg's FIT half does not need the judge (ggui#1195): the fit verdict is a
+ * frame at the declared box and a scroll height, so a no-vision lane with
+ * canvases runs `measureFit` alone. Its canvas-overflow fail rides the round's
+ * issues like the judge path's would — so the `[fit]` round the cap grants
+ * can fire on that lane — and the coverage still reads `skipped` for the judge,
+ * with what the fit half measured (or why it could not) appended. Without
+ * that half, a card taller than its declared chat box on such a lane was
+ * never told so in the loop, so the composition fix was never asked for.
+ */
+function runVisualLeg(
+  visualEvalAgent: AgentSpec,
+  run: (provider: "claude" | "google") => Promise<VisualEvalOutcome>,
+  measureFit: (() => Promise<VisualFitOutcome>) | undefined,
+): Promise<VisualEvalOutcome> {
+  const provider = visionJudgeProvider(visualEvalAgent.provider);
+  if (provider === undefined) {
+    const noJudge = `no vision judge for provider '${visualEvalAgent.provider}' (model ${visualEvalAgent.model}) — the in-loop visual judge runs on a vision provider (anthropic, google)`;
+    if (measureFit === undefined) {
+      return Promise.resolve({ issues: [], coverage: { status: "skipped", reason: noJudge } });
+    }
+    const notMeasured = (why: string): VisualEvalOutcome => ({
+      issues: [],
+      coverage: { status: "skipped", reason: `${noJudge}; fit not measured: ${why}` },
+    });
+    return measureFit().then(
+      (fit): VisualEvalOutcome =>
+        fit.status === "measured"
+          ? {
+              issues: fit.issues,
+              coverage: { status: "skipped", reason: `${noJudge}; fit measured without the judge: ${describeFitReadings(fit.readings)}` },
+            }
+          : notMeasured(fit.reason),
+      (e: unknown): VisualEvalOutcome => notMeasured(e instanceof Error ? e.message : String(e)),
+    );
+  }
+  return run(provider).catch((e: unknown): VisualEvalOutcome => ({
+    issues: [],
+    coverage: { status: "skipped", reason: `visual leg failed: ${e instanceof Error ? e.message : String(e)}` },
+  }));
+}
+
+/** The fit half's receipt on the coverage: `xs-chat-card 384×516 content 535px (overflow)`, one per canvas. */
+function describeFitReadings(readings: readonly CanvasFitReading[]): string {
+  return readings
+    .map((r) => `${r.canvas} ${r.viewport.width}×${r.viewport.height} content ${r.contentHeight ?? "?"}px${r.overflow ? " (overflow)" : ""}`)
+    .join(", ");
+}
+
 const LOW_RISK_BYPASS_REASON =
   "same-image low-risk bypass: axis checks clean, tier-1/2 evaluation not invoked";
 
@@ -448,6 +531,30 @@ export async function runEvalRound(
     let llmResult: (EvalResult & { inputTokens: number; outputTokens: number }) | null = null;
     let visualIssues: EvalIssue[] | null = null;
 
+    // The visual leg's page and frame — ONE value each, read by the judge and
+    // by its fit half alike, so both frame the same component the same way.
+    const visualContext: VisualEvalContext = {
+      compiledCode,
+      originalPrompt: userPrompt,
+      profile: harness.profile,
+      // The caller's theme, when named — the round paints what the app paints.
+      ...(visualEvaluation?.cssTokens !== undefined ? { cssTokens: visualEvaluation.cssTokens } : {}),
+    };
+    const visualFrame: VisualFitConfig = {
+      // The in-loop visual round renders the same sample the runtime probe
+      // does (the caller's fixture) unless the config names its own.
+      sampleProps: visualEvaluation?.sampleProps ?? fixtureProps,
+      // Per-canvas judging (arm-neutral) — absent = single shot.
+      ...(visualEvaluation?.canvases !== undefined ? { canvases: visualEvaluation.canvases } : {}),
+      // ggui#1195 — the declared box per canvas (same-target with the composer).
+      ...(visualEvaluation?.canvasViewports !== undefined ? { canvasViewports: visualEvaluation.canvasViewports } : {}),
+    };
+    // The fit verdict is per canvas: a single-shot leg has no box to hold content against.
+    const measureFit =
+      visualMod && visualFrame.canvases !== undefined && visualFrame.canvases.length > 0
+        ? () => visualMod.runVisualFit(visualContext, visualFrame)
+        : undefined;
+
     const evalLlmStart = Date.now();
     const [llm, visual] = await Promise.all([
       llmEvalMod
@@ -483,35 +590,25 @@ export async function runEvalRound(
           )
         : null,
       visualMod
-        ? visualMod.runVisualEval(
-            {
-              compiledCode,
-              originalPrompt: userPrompt,
-              profile: harness.profile,
-              // The caller's theme, when named — the round paints what the app paints.
-              ...(visualEvaluation?.cssTokens !== undefined ? { cssTokens: visualEvaluation.cssTokens } : {}),
-            },
-            {
-              provider: mapProviderForEvaluator(visualEvalAgent.provider) as "claude" | "google",
-              model: visualEvalAgent.model,
-              passThreshold: visualThreshold,
-              // The in-loop visual round renders the same sample the runtime probe
-              // does (the caller's fixture) unless the config names its own.
-              sampleProps: visualEvaluation?.sampleProps ?? fixtureProps,
-              viewport: visualEvaluation?.viewport,
-              // Per-canvas judging (arm-neutral) — absent = single shot.
-              ...(visualEvaluation?.canvases !== undefined
-                ? { canvases: visualEvaluation.canvases }
-                : {}),
-              // #484/#489 — same threading as the LLM-eval leg above:
-              // the visual eval's multimodal call runs inside the same
-              // concurrent generation, so it needs the same routing
-              // override (never fall back to `process.env`) and the
-              // same provider-429-retry observer (retries reach the
-              // host application's structured log).
-              routeOverride: visualEvalAgent.routeOverride,
-              onRetry: visualEvalAgent.onRetry,
-            },
+        ? runVisualLeg(
+            visualEvalAgent,
+            (provider) =>
+              visualMod.runVisualEval(visualContext, {
+                provider,
+                model: visualEvalAgent.model,
+                passThreshold: visualThreshold,
+                viewport: visualEvaluation?.viewport,
+                ...visualFrame,
+                // #484/#489 — same threading as the LLM-eval leg above:
+                // the visual eval's multimodal call runs inside the same
+                // concurrent generation, so it needs the same routing
+                // override (never fall back to `process.env`) and the
+                // same provider-429-retry observer (retries reach the
+                // host application's structured log).
+                routeOverride: visualEvalAgent.routeOverride,
+                onRetry: visualEvalAgent.onRetry,
+              }),
+            measureFit,
           )
         : null,
     ]);
@@ -519,6 +616,12 @@ export async function runEvalRound(
     llmResult = llm;
     visualIssues = visual?.issues ?? null;
     const visualSummary: VisualEvalSummary | undefined = visual?.summary;
+    // ggui#1221 — the judge leg's coverage is ALWAYS stamped: ran / skipped (with the
+    // reason) / not-applicable when no visual leg was configured for this round.
+    const visualCoverage: VisualCoverage = visual?.coverage ?? { status: 'not-applicable', reason: 'visual leg not configured' };
+    if (visualCoverage.status === 'skipped') {
+      console.log(`[eval] visual-judge: SKIPPED (${visualCoverage.reason ?? 'no reason recorded'})`);
+    }
 
     // Track costs from LLM eval calls
     if (llmResult) {
@@ -549,6 +652,7 @@ export async function runEvalRound(
       // requested and the leg ran; every later re-stamp spreads this
       // base so it survives to the assembled result.
       ...(visualSummary !== undefined ? { visual: visualSummary } : {}),
+      visualCoverage,
     };
 
     // ── Log merged results, with mode-check subcategory breakdown ──
@@ -683,13 +787,19 @@ export async function runEvalRound(
     // to localize them; the +2 max-rounds extension is the right escape
     // hatch, not stuck-loop exit. Keep stuck-loop active for non-runtime
     // fail-sets (the original tier-1 nitpick case it was built for).
-    const onlyRuntimeFails =
+    // ggui#1195 follow-up: a FIT fail (`visual|canvas-overflow|…`) is exempt
+    // the same way — its escape hatch is the one `[fit]` round at the cap, and
+    // a card whose measured height did not move between rounds would otherwise
+    // exit here as "stuck" before that round is ever reached (the adversarial
+    // review ran the commit's own test and found round 2's break WAS the stuck
+    // exit). Bounded: the cap block grants the fit round exactly once.
+    const onlyExtensionExemptFails =
       currFailFingerprints.size > 0 &&
-      [...currFailFingerprints].every((fp) => fp.includes("|runtime:"));
+      [...currFailFingerprints].every((fp) => fp.includes("|runtime:") || fp.startsWith(FIT_FINGERPRINT_PREFIX));
     if (
       evalRoundsUsed > 1 &&
       currFailFingerprints.size > 0 &&
-      !onlyRuntimeFails &&
+      !onlyExtensionExemptFails &&
       [...currFailFingerprints].every((fp) => prevFailFingerprints.has(fp))
     ) {
       console.log(
@@ -747,6 +857,11 @@ export async function runEvalRound(
     // stuck-loop detector won't fire (some fails change), but the cap
     // hits with the runtime fail unresolved.
     const RUNTIME_EXTENSION_BONUS = 2;
+    // ggui#1195 — a fit fail (the hello does not fit its declared first
+    // screen — the founder's ruling is FAIL, not warn) active at the cap is
+    // worth ONE more round: the fix is a composition change the LLM can make
+    // in a turn, and without the round the card ships cut off.
+    const FIT_EXTENSION_BONUS = 1;
     if (evalRoundsUsed >= maxEvalRounds) {
       // 2026-04-27 (in-loop probe trigger): probe is now off the per-turn
       // hot path; `blocking` from runCheck won't contain `runtime:*` fails
@@ -764,6 +879,36 @@ export async function runEvalRound(
       const allowRuntimeExtension =
         hasRuntimeFail &&
         evalRoundsUsed < maxEvalRounds + RUNTIME_EXTENSION_BONUS;
+      const fitFails = evalResult.issues.filter(isFitFail);
+      const allowFitExtension =
+        !allowRuntimeExtension &&
+        fitFails.length > 0 &&
+        evalRoundsUsed < maxEvalRounds + FIT_EXTENSION_BONUS;
+      if (allowFitExtension) {
+        const fitLines = fitFails.slice(0, MAX_FEEDBACK_ISSUES).map(formatFitFeedback);
+        evalResult = {
+          ...evalResult,
+          issues: [...evalResult.issues, ...exitProbe.probeIssues],
+          runtimeProbe: exitProbe.meta,
+        };
+        console.log(
+          `[simple] eval round ${evalRoundsUsed}: fit fail active at cap — granting +1 retry (ggui#1195; ${fitFails.length} canvas-overflow)`,
+        );
+        return {
+          control: "feedback",
+          evalDone: false,
+          evalResult,
+          evalRoundsUsed,
+          prevModeSubcats: updatedPrevModeSubcats,
+          prevFailFingerprints: new Set([...currFailFingerprints, ...fitFails.map(fingerprintFail)]),
+          preWarmedContext,
+          evalTokens,
+          evalLlmMs,
+          lastResultText: fitLines.join("\n\n"),
+          isEvalFeedback: true,
+          lastDiffFailed: false,
+        };
+      }
       if (!allowRuntimeExtension) {
         // Fold probe issues (if any) into evalResult for telemetry and
         // ALWAYS stamp the probe's execution meta, even though we're not

package/src/harness/enforced-coding.ts

@@ -18,3 +18,23 @@
 export function mapProviderForEvaluator(provider: 'anthropic' | 'openai' | 'google' | 'openrouter'): 'claude' | 'openai' | 'google' | 'openrouter' {
   return provider === 'anthropic' ? 'claude' : provider;
 }
+
+/**
+ * The visual judge's provider for an agent's provider (ggui#1248) — only the
+ * providers whose SDK path takes an image (`createVisionAgent`: Anthropic,
+ * Google). `undefined` for a provider with no vision path, so a caller must
+ * decide what the visual leg does there instead of asserting it away: the
+ * in-loop judge inherits the generation's agent by default, and an OpenAI or
+ * OpenRouter lane reaching `createVisionAgent` got `undefined` back and threw.
+ */
+export function visionJudgeProvider(provider: 'anthropic' | 'openai' | 'google' | 'openrouter'): 'claude' | 'google' | undefined {
+  switch (provider) {
+    case 'anthropic':
+      return 'claude';
+    case 'google':
+      return 'google';
+    case 'openai':
+    case 'openrouter':
+      return undefined;
+  }
+}