@ggui-ai/ui-gen 0.25.0 → 0.26.0

package/src/harness/check/runtime-render/walk.ts

@@ -0,0 +1,200 @@
+/**
+ * The action walk (ggui#1652): find a control that dispatches one action by
+ * walking the card's screens, as a person would, in a mount of its own.
+ *
+ * A screen is the card's set of controls, told apart by each control's tag,
+ * type, label (its aria-label, placeholder or text) and disabled state, never
+ * by typed values, selection or generated names.
+ * Picking an answer is the same screen unless it enables or disables a
+ * control; a wizard's next step, or an Edit button that now reads Save, is a
+ * new one. Each control is pressed once per screen. When every control on
+ * the current screen has been pressed, the walk returns over controls it has
+ * already pressed to the nearest screen that still has one to press. A press
+ * that leads back to a screen already seen is not progress, so Back and Next
+ * cannot loop, in any language. The walk stops at the first press that
+ * dispatches the action, when every reachable control has been pressed, at
+ * a cap on presses, or at its deadline. Neither the cap nor the deadline
+ * stops it while it is on the first screen with a control there still
+ * unpressed; once a press leaves the first screen both apply, and a
+ * first-screen control left unpressed is pressed only if a known route
+ * leads back to it (a one-way wizard has none).
+ */
+import type { MinimalElement } from "./host-boundary.js";
+
+/** One control the walk can press, and how. */
+export interface WalkControl {
+  readonly element: MinimalElement;
+  /** Whether the control names the action (its label, aria-label or `data-action`). */
+  readonly namesAction: boolean;
+}
+
+/** A fresh copy of the card, and how to fill its screen's empty form controls. */
+export interface WalkMount {
+  readonly container: MinimalElement;
+  /** Fill the current screen's empty form controls; filled ones are left as they are. */
+  readonly prime: () => void;
+}
+
+export interface WalkInput<C extends WalkControl> {
+  /** Mount a fresh copy of the card; undefined when it cannot be mounted. */
+  readonly mount: () => Promise<WalkMount | undefined>;
+  /** The screen's controls, in the order they are tried. */
+  readonly controls: (container: MinimalElement) => readonly C[];
+  /** Press one control; true when the press dispatched the action. */
+  readonly press: (control: C) => Promise<boolean>;
+  readonly describe: (el: MinimalElement) => string;
+  /** Epoch ms after which no further press is made. */
+  readonly stopAt: number;
+  readonly maxPresses: number;
+}
+
+export type WalkStop = "dispatched" | "explored" | "press-cap" | "deadline" | "no-mount";
+
+export interface WalkOutcome {
+  readonly fired: boolean;
+  /** The control whose press dispatched the action, and the mount it is in. */
+  readonly element?: MinimalElement;
+  readonly container?: MinimalElement;
+  readonly presses: number;
+  readonly screens: number;
+  readonly stoppedBy: WalkStop;
+  /** Whether any screen rendered a control that names the action. */
+  readonly namedControlSeen: boolean;
+  /** The last few controls pressed, oldest first. */
+  readonly lastPressed: readonly string[];
+  /** The text of every screen the walk reached, once each, in the order reached. */
+  readonly screenTexts: readonly string[];
+}
+
+/** How many of the last presses an outcome keeps. */
+const LAST_PRESSED_KEPT = 4;
+
+function isDisabled(el: MinimalElement): boolean {
+  return el.getAttribute("disabled") !== null || el.getAttribute("aria-disabled") === "true";
+}
+
+/**
+ * A control's label for the fingerprint. Never its `name` or `id`: React
+ * generates those (`useId`) afresh when a step re-mounts, so a screen keyed
+ * on them would look new on every return and the walk would loop on it.
+ */
+function labelOf(el: MinimalElement): string {
+  const named = el.getAttribute("aria-label") ?? el.getAttribute("placeholder");
+  return (named ?? el.textContent ?? "").replace(/\s+/g, " ").trim().slice(0, 60);
+}
+
+/** The screen's identity: its controls' structure and disabled state, in document order. */
+export function screenFingerprint(container: MinimalElement): string {
+  const parts: string[] = [];
+  for (const el of container.querySelectorAll(
+    'button, [role="button"], a[href], input, select, textarea, form'
+  )) {
+    parts.push(
+      [el.tagName.toLowerCase(), el.getAttribute("type") ?? "", labelOf(el), isDisabled(el) ? "disabled" : ""].join("|")
+    );
+  }
+  return parts.join("\n");
+}
+
+interface Screen {
+  /** Each control's key, in try order. */
+  readonly order: readonly string[];
+  /** A pressed control's key → the screen its press led to. */
+  readonly led: Map<string, string>;
+}
+
+/** Each control's key on a screen: its description, numbered among controls described alike. */
+function keyed<C extends WalkControl>(controls: readonly C[], describe: (el: MinimalElement) => string): Map<string, C> {
+  const seen = new Map<string, number>();
+  const out = new Map<string, C>();
+  for (const c of controls) {
+    const d = describe(c.element);
+    const n = seen.get(d) ?? 0;
+    seen.set(d, n + 1);
+    out.set(`${d}#${n}`, c);
+  }
+  return out;
+}
+
+/** The first press of the shortest known route to a screen with a control not yet pressed. */
+function firstStepToUnpressed(screens: ReadonlyMap<string, Screen>, from: string): string | undefined {
+  const firstStep = new Map<string, string | undefined>([[from, undefined]]);
+  const queue = [from];
+  while (queue.length > 0) {
+    const at = queue.shift()!;
+    const screen = screens.get(at);
+    if (screen === undefined) continue;
+    if (at !== from && screen.order.some((k) => !screen.led.has(k))) return firstStep.get(at);
+    for (const [key, to] of screen.led) {
+      if (to === at || firstStep.has(to)) continue;
+      firstStep.set(to, firstStep.get(at) ?? key);
+      queue.push(to);
+    }
+  }
+  return undefined;
+}
+
+export async function walkToAction<C extends WalkControl>(input: WalkInput<C>): Promise<WalkOutcome> {
+  const lastPressed: string[] = [];
+  const screenTexts: string[] = [];
+  const screens = new Map<string, Screen>();
+  let presses = 0;
+  let namedControlSeen = false;
+  const outcome = (stoppedBy: WalkStop, extra: Partial<WalkOutcome> = {}): WalkOutcome => ({
+    fired: false,
+    presses,
+    screens: screens.size,
+    stoppedBy,
+    namedControlSeen,
+    lastPressed: [...lastPressed],
+    screenTexts: [...screenTexts],
+    ...extra,
+  });
+
+  const mounted = await input.mount();
+  if (mounted === undefined) return outcome("no-mount");
+  const { container } = mounted;
+
+  const arrive = (): { id: string; controls: Map<string, C> } => {
+    mounted.prime();
+    const controls = keyed(input.controls(container), input.describe);
+    if ([...controls.values()].some((c) => c.namesAction)) namedControlSeen = true;
+    const id = screenFingerprint(container);
+    if (!screens.has(id)) {
+      screens.set(id, { order: [...controls.keys()], led: new Map() });
+      screenTexts.push(container.textContent ?? "");
+    }
+    return { id, controls };
+  };
+
+  let here = arrive();
+  const first = here.id;
+  for (;;) {
+    const screen = screens.get(here.id)!;
+    const onOpenFirstScreen = here.id === first && screen.order.some((k) => !screen.led.has(k));
+    if (!onOpenFirstScreen) {
+      if (presses >= input.maxPresses) return outcome("press-cap");
+      if (Date.now() >= input.stopAt) return outcome("deadline");
+    }
+    const key = screen.order.find((k) => !screen.led.has(k)) ?? firstStepToUnpressed(screens, here.id);
+    if (key === undefined) return outcome("explored");
+    const control = here.controls.get(key);
+    if (control === undefined) {
+      // Listed when the screen was first seen, gone now: nothing to press.
+      // It now leads nowhere, which a route never takes, so this cannot
+      // pick the same control again.
+      screen.led.set(key, here.id);
+      continue;
+    }
+    presses += 1;
+    lastPressed.push(input.describe(control.element));
+    if (lastPressed.length > LAST_PRESSED_KEPT) lastPressed.shift();
+    if (await input.press(control)) {
+      return outcome("dispatched", { fired: true, element: control.element, container });
+    }
+    here = arrive();
+    // The first press records where a control leads; a route that later
+    // leads elsewhere is simply followed on from where it led.
+    if (!screen.led.has(key)) screen.led.set(key, here.id);
+  }
+}

package/src/harness/coding/assemble-result.ts

@@ -119,6 +119,8 @@ export async function assembleGenerationResult(
     ...(telemetry.cacheCreationTokens !== undefined
       ? { cacheCreationTokens: telemetry.cacheCreationTokens }
       : {}),
+    // ggui#1522 — the in-loop visual judge's spend, beside `tokens` and never inside it.
+    ...(telemetry.inLoopVisualTokens !== undefined ? { inLoopVisualTokens: telemetry.inLoopVisualTokens } : {}),
     generationTimeMs: totalMs,
     turnsUsed: telemetry.turnsUsed,
     passesUsed: 1,

package/src/harness/coding/generate-task-runner.ts

@@ -24,6 +24,7 @@ import {
 } from "./run-coding-turn.js";
 import { runEvalRound } from "./run-eval-round.js";
 import type { TokenUsage } from "./run-eval-round.js";
+import type { VisualLegTokens } from "../../evaluation/visual-evaluator.js";
 import type { SameExchangeBreak } from "../result-types.js";
 
 type PreWarmedEvalContext =
@@ -75,6 +76,12 @@ export interface GenerateTelemetry {
    */
   cacheReadTokens?: number;
   cacheCreationTokens?: number;
+  /**
+   * ggui#1522 — the in-loop visual judge's spend across eval rounds, apart from {@link totalIn} / {@link totalOut}
+   * (priced at the coding and text-evaluator models): this part is priced at the VISUAL agent's model. Stays
+   * `undefined` while no round reported one.
+   */
+  inLoopVisualTokens?: VisualLegTokens;
   /** Latest evalResult — undefined when eval was disabled or never ran. */
   evalResult: EvalResult | undefined;
   /**
@@ -167,6 +174,25 @@ export function absorbTokens(telemetry: GenerateTelemetry, usage: TokenUsage): v
   }
 }
 
+/** ggui#1522 — fold one round's in-loop visual spend into the run's own part (never into `totalIn` / `totalOut`). */
+export function absorbInLoopVisualTokens(telemetry: GenerateTelemetry, part: VisualLegTokens | undefined): void {
+  if (part === undefined) return;
+  const prev = telemetry.inLoopVisualTokens;
+  const prevCriteria = prev?.criteria;
+  const criteria =
+    part.criteria === undefined
+      ? prevCriteria
+      : {
+          inputTokens: (prevCriteria?.inputTokens ?? 0) + part.criteria.inputTokens,
+          outputTokens: (prevCriteria?.outputTokens ?? 0) + part.criteria.outputTokens,
+        };
+  telemetry.inLoopVisualTokens = {
+    inputTokens: (prev?.inputTokens ?? 0) + part.inputTokens,
+    outputTokens: (prev?.outputTokens ?? 0) + part.outputTokens,
+    ...(criteria !== undefined ? { criteria } : {}),
+  };
+}
+
 /**
  * Fold one coding turn's measurements into the run's telemetry — tokens (via
  * {@link absorbTokens}), timings, the phase/outcome counters, and the
@@ -413,6 +439,7 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
         if (round.contractFeedback) telemetry.contractFeedback = round.contractFeedback;
         evalDone = round.evalDone;
         absorbTokens(telemetry, round.evalTokens);
+        absorbInLoopVisualTokens(telemetry, round.inLoopVisualTokens);
 
         if (round.control === "break") break;
         // round.control === "feedback" — set next coding turn input.

package/src/harness/coding/init-session.ts

@@ -19,6 +19,7 @@ import type { AgentSpec, SingleComponentParams } from "../runtime.js";
 import type { ResolvedRunPolicy } from "../policy.js";
 import { DEFAULT_HARNESS_POLICY } from "../policy.js";
 import { createDupeBreakState, type DupeBreakState } from "./dupe-break.js";
+import { inLoopEvaluation } from "../in-loop-evaluation.js";
 
 type PreWarmedEvalContext =
   import("../../evaluation/llm-evaluator.js").PreWarmedEvalContext;
@@ -165,10 +166,13 @@ export async function initSession(input: {
     "required",
   );
 
+  // ggui#1513 — the one rule for which evaluation legs run; a host reads the same export.
+  const { text: codeEvalEnabled, visual: visualEvalEnabled } = inLoopEvaluation(params);
+
   // ── Pre-warm eval (overlaps with coding turn 1) ──
   // Uses a separate agent instance so it doesn't disturb the coding cache.
   let preWarmPromise: Promise<PreWarmedEvalContext | null> | undefined;
-  if (params.evaluation?.enabled) {
+  if (codeEvalEnabled) {
     const evalSpec = agents.evaluation;
     preWarmPromise = import("../../evaluation/llm-evaluator.js")
       .then((mod) =>
@@ -202,8 +206,6 @@ export async function initSession(input: {
   // ── Eval gate + lazy-loaded modules ──
   const qualityConfig = params.qualityConfig;
   const qualityMode = qualityConfig?.quality ?? "fast";
-  const codeEvalEnabled = !!params.evaluation?.enabled;
-  const visualEvalEnabled = !!(params.visualEvaluation?.enabled || qualityConfig?.visualEval);
   const probeOnlyEnabled =
     harness.check.runtimeRender !== undefined && !codeEvalEnabled && !visualEvalEnabled;
   const maxEvalRounds =

package/src/harness/coding/run-coding-turn.ts

@@ -544,7 +544,11 @@ ${closingInstruction}`;
   );
   const llmMs = Date.now() - llmStart;
   if (costTracker) {
-    costTracker.record(codingModel, response.inputTokens, response.outputTokens);
+    // ggui#1524 — the cache counts reach the cap: a cached prefix is billed, so it is spend.
+    costTracker.record(codingModel, response.inputTokens, response.outputTokens, {
+      ...(response.cacheReadTokens !== undefined ? { read: response.cacheReadTokens } : {}),
+      ...(response.cacheCreationTokens !== undefined ? { write: response.cacheCreationTokens } : {}),
+    });
   }
 
   const tokens = { input: response.inputTokens, output: response.outputTokens };

package/src/harness/coding/run-eval-round.ts

@@ -52,6 +52,7 @@ import type {
   VisualEvalOutcome,
   VisualFitConfig,
   VisualFitOutcome,
+  VisualLegTokens,
 } from "../../evaluation/visual-evaluator.js";
 
 type PreWarmedEvalContext =
@@ -181,6 +182,12 @@ export interface EvalRoundResult {
   readonly preWarmedContext: PreWarmedEvalContext | null | undefined;
   /** Tokens added during this round — caller adds to its own totals. */
   readonly evalTokens: TokenUsage;
+  /**
+   * ggui#1522 — the in-loop visual judge's spend this round, apart from {@link evalTokens} (the text evaluator's,
+   * priced at its model): already recorded in the cost tracker at the visual agent's model. Absent when no visual
+   * leg ran or it reported no tokens.
+   */
+  readonly inLoopVisualTokens?: VisualLegTokens;
   /** Wall-clock of the parallel LLM + visual eval block (zero on the
    *  low-risk bypass path, which exits before `Promise.all`). Caller
    *  accumulates into its `cumulativeEvalLlmMs`. */
@@ -417,6 +424,7 @@ function probeMetaOf(outcome: RuntimeRenderOutcome, probeIssues: readonly EvalIs
     ...(outcome.renderMs !== undefined ? { renderMs: outcome.renderMs } : {}),
     ...(outcome.hostLoad !== undefined ? { hostLoad: outcome.hostLoad } : {}),
     ...(outcome.queuedMs !== undefined ? { queuedMs: outcome.queuedMs } : {}),
+    ...(outcome.pendingAffordance !== undefined ? { pendingAffordance: outcome.pendingAffordance } : {}),
   };
   const elapsed = outcome.elapsedMs !== undefined ? { elapsedMs: outcome.elapsedMs } : {};
   if (outcome.status !== "ran") {
@@ -591,6 +599,7 @@ export async function runEvalRound(
 
   let evalResult: EvalResult | undefined;
   let evalTokens: TokenUsage = { input: 0, output: 0 };
+  let inLoopVisualTokens: VisualLegTokens | undefined = undefined;
   let evalLlmMs = 0;
 
   try {
@@ -641,6 +650,7 @@ export async function runEvalRound(
           prevFailFingerprints,
           preWarmedContext,
           evalTokens,
+          ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
           evalLlmMs,
           lastResultText: "",
           isEvalFeedback: false,
@@ -665,6 +675,7 @@ export async function runEvalRound(
           prevFailFingerprints,
           preWarmedContext,
           evalTokens,
+          ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
           evalLlmMs,
           lastResultText: lines.join("\n\n"),
           isEvalFeedback: true,
@@ -683,6 +694,7 @@ export async function runEvalRound(
         prevFailFingerprints,
         preWarmedContext,
         evalTokens,
+        ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
         evalLlmMs,
         lastResultText: "",
         isEvalFeedback: false,
@@ -772,6 +784,7 @@ export async function runEvalRound(
           prevFailFingerprints,
           preWarmedContext,
           evalTokens,
+          ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
           evalLlmMs,
           lastResultText: lines.join("\n\n"),
           isEvalFeedback: true,
@@ -797,6 +810,7 @@ export async function runEvalRound(
         prevFailFingerprints,
         preWarmedContext,
         evalTokens,
+        ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
         evalLlmMs,
         lastResultText: "",
         isEvalFeedback: false,
@@ -917,6 +931,18 @@ export async function runEvalRound(
     evalLlmMs = Date.now() - evalLlmStart;
     llmResult = llm;
     visualIssues = visual?.issues ?? null;
+    // ggui#1522 — the in-loop visual judge's own spend: recorded in the tracker at the VISUAL agent's model, so
+    // `canContinue()` sees it, and carried apart from `evalTokens` (the text evaluator's model).
+    // The report-only criteria calls (ggui#1436) count toward the cap too, deliberately: the cap bounds MONEY SPENT,
+    // and a cap that left out real spend would under-count exactly as this leg did before. So under a tight cap a
+    // configured criteria bank's calls can end the eval loop earlier — the one place a report-only block reaches an
+    // outcome, and it does so through its cost, never through its answers. The tracker's total feeds no metered figure.
+    inLoopVisualTokens = visual?.tokens;
+    if (inLoopVisualTokens !== undefined) {
+      costTracker.record(visualEvalAgent.model, inLoopVisualTokens.inputTokens, inLoopVisualTokens.outputTokens);
+      const criteriaSpend = inLoopVisualTokens.criteria;
+      if (criteriaSpend !== undefined) costTracker.record(visualEvalAgent.model, criteriaSpend.inputTokens, criteriaSpend.outputTokens);
+    }
     const visualSummary: VisualEvalSummary | undefined = visual?.summary;
     // ggui#1221 — the judge leg's coverage is ALWAYS stamped: ran / skipped (with the
     // reason) / not-applicable when no visual leg was configured for this round.
@@ -927,7 +953,11 @@ export async function runEvalRound(
 
     // Track costs from LLM eval calls
     if (llmResult) {
-      costTracker.record(evaluationAgent.model, llmResult.inputTokens, llmResult.outputTokens);
+      // ggui#1524 — the text evaluator's cache counts reach the cap too.
+      costTracker.record(evaluationAgent.model, llmResult.inputTokens, llmResult.outputTokens, {
+        ...(llmResult.cacheReadTokens !== undefined ? { read: llmResult.cacheReadTokens } : {}),
+        ...(llmResult.cacheCreationTokens !== undefined ? { write: llmResult.cacheCreationTokens } : {}),
+      });
       evalTokens = {
         input: llmResult.inputTokens,
         output: llmResult.outputTokens,
@@ -996,12 +1026,22 @@ export async function runEvalRound(
     );
 
     // ── Decide whether to continue the loop ──
-    // In fast mode: only fails trigger another iteration.
+    // In fast mode: only fails trigger another iteration, plus (ggui#1542) the visual JUDGE's major findings when
+    // `visualEvaluation.actOn` is 'major'. They are taken from the visual outcome itself, never by category over the
+    // merged list, because the text evaluator has a tier-2 `visual` criterion of its own. Only findings whose
+    // `origin` is `judge` act (ggui#1545): the instruments' warn-level majors (fit's `canvas-overflow*` today) are
+    // deterministic and run with or without a judge, so acting on them would switch a second feedback source on
+    // with the judge's. Selecting by origin rather than excluding by name fails closed: a new instrument check, or
+    // any issue with no origin, never joins the treatment unless it is marked a judge finding.
     // In auto-improve / high-quality: fails + warns trigger iterations.
     const blocking = evalResult.issues.filter((i) => i.result === "fail");
+    const visualMajorsActed =
+      visualEvaluation?.actOn === "major"
+        ? (visualIssues ?? []).filter((i) => i.origin === "judge" && i.result === "warn" && i.severity === "major")
+        : [];
     const shouldContinue =
       qualityMode === "fast"
-        ? blocking.length > 0
+        ? blocking.length > 0 || visualMajorsActed.length > 0
         : blocking.length > 0 || warns.length > 0;
 
     // Compute current round's fail fingerprints — used by the stuck-loop
@@ -1046,6 +1086,7 @@ export async function runEvalRound(
           prevFailFingerprints: enrichedFingerprints,
           preWarmedContext,
           evalTokens,
+          ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
           evalLlmMs,
           lastResultText: lines.join("\n\n"),
           isEvalFeedback: true,
@@ -1079,6 +1120,7 @@ export async function runEvalRound(
           prevFailFingerprints: new Set([...currFailFingerprints, ...contractFails.map(fingerprintFail)]),
           preWarmedContext,
           evalTokens,
+          ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
           evalLlmMs,
           lastResultText: lines.join("\n\n"),
           isEvalFeedback: true,
@@ -1105,6 +1147,7 @@ export async function runEvalRound(
         prevFailFingerprints: currFailFingerprints,
         preWarmedContext,
         evalTokens,
+        ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
         evalLlmMs,
         lastResultText: "",
         isEvalFeedback: false,
@@ -1186,6 +1229,7 @@ export async function runEvalRound(
           prevFailFingerprints: new Set([...currFailFingerprints, ...stuckContractFails.map(fingerprintFail)]),
           preWarmedContext,
           evalTokens,
+          ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
           evalLlmMs,
           lastResultText: lines.join("\n\n"),
           isEvalFeedback: true,
@@ -1202,6 +1246,7 @@ export async function runEvalRound(
         prevFailFingerprints: currFailFingerprints,
         preWarmedContext,
         evalTokens,
+        ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
         evalLlmMs,
         lastResultText: "",
         isEvalFeedback: false,
@@ -1272,6 +1317,7 @@ export async function runEvalRound(
           prevFailFingerprints: new Set([...currFailFingerprints, ...fitFails.map(fingerprintFail)]),
           preWarmedContext,
           evalTokens,
+          ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
           evalLlmMs,
           lastResultText: fitLines.join("\n\n"),
           isEvalFeedback: true,
@@ -1301,6 +1347,7 @@ export async function runEvalRound(
           prevFailFingerprints: new Set([...currFailFingerprints, ...capContractFails.map(fingerprintFail)]),
           preWarmedContext,
           evalTokens,
+          ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
           evalLlmMs,
           lastResultText: contractLines.join("\n\n"),
           isEvalFeedback: true,
@@ -1327,6 +1374,7 @@ export async function runEvalRound(
           prevFailFingerprints: currFailFingerprints,
           preWarmedContext,
           evalTokens,
+          ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
           evalLlmMs,
           lastResultText: "",
           isEvalFeedback: false,
@@ -1359,6 +1407,7 @@ export async function runEvalRound(
         ]),
         preWarmedContext,
         evalTokens,
+        ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
         evalLlmMs,
         lastResultText: probeLines.join("\n\n"),
         isEvalFeedback: true,
@@ -1380,6 +1429,7 @@ export async function runEvalRound(
         prevFailFingerprints: currFailFingerprints,
         preWarmedContext,
         evalTokens,
+        ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
         evalLlmMs,
         lastResultText: "",
         isEvalFeedback: false,
@@ -1412,6 +1462,14 @@ export async function runEvalRound(
     console.log(
       `[simple] eval round ${evalRoundsUsed}: ${allActionable.length} issues (${blocking.length} fail, ${warns.length} warn) → feeding back to coding loop`,
     );
+    // ggui#1542 — WHY this round continued, for a caller that set `actOn`: the manipulation check of an experiment on
+    // the setting reads this line. `visual majors acted` counts the judge's majors only (fit's are never acted on).
+    if (visualEvaluation?.actOn !== undefined) {
+      console.log(
+        `[simple] eval round ${evalRoundsUsed}: continuing: fails=${blocking.length}, visual majors acted=${visualMajorsActed.length} ` +
+          `(actOn=${visualEvaluation.actOn})`,
+      );
+    }
 
     return {
       control: "feedback",
@@ -1431,6 +1489,7 @@ export async function runEvalRound(
       ),
       preWarmedContext,
       evalTokens,
+      ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
       evalLlmMs,
       lastResultText: issueLines.join("\n\n"),
       isEvalFeedback: true,
@@ -1463,6 +1522,7 @@ export async function runEvalRound(
       prevFailFingerprints,
       preWarmedContext,
       evalTokens,
+      ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
       evalLlmMs,
       lastResultText: "",
       isEvalFeedback: false,

package/src/harness/in-loop-evaluation.ts

@@ -0,0 +1,39 @@
+/**
+ * Which in-loop evaluation legs a generation runs (ggui#1513).
+ *
+ * The harness gates its evaluation rounds on this, and nothing else does:
+ * the text evaluator runs when `evaluation.enabled`, and the visual judge
+ * runs when `visualEvaluation.enabled` or when a quality config asks for it
+ * (`qualityConfig.visualEval`). A host that needs to know whether a
+ * generation's token counts (`GenerationMetadata`) can include evaluation
+ * calls reads it here, with the options it configured the generator with,
+ * instead of keeping its own copy of the rule.
+ */
+import type { QualityConfig } from "../evaluation/types-public.js";
+
+/** The switches that turn an in-loop evaluation leg on. */
+export interface InLoopEvaluationSwitches {
+  readonly evaluation?: { readonly enabled: boolean };
+  readonly visualEvaluation?: { readonly enabled: boolean };
+  readonly qualityConfig?: Pick<QualityConfig, "visualEval">;
+}
+
+/** The in-loop evaluation legs these options turn on. */
+export function inLoopEvaluation(options: InLoopEvaluationSwitches): {
+  readonly text: boolean;
+  readonly visual: boolean;
+} {
+  return {
+    text: options.evaluation?.enabled === true,
+    visual: options.visualEvaluation?.enabled === true || options.qualityConfig?.visualEval === true,
+  };
+}
+
+/**
+ * True when these options turn either leg on: a generation built with them
+ * can spend tokens on evaluation calls as well as on its coding turns.
+ */
+export function runsInLoopEvaluation(options: InLoopEvaluationSwitches): boolean {
+  const legs = inLoopEvaluation(options);
+  return legs.text || legs.visual;
+}