@ggui-ai/ui-gen 0.24.0 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/README.md +2 -2
  2. package/dist/{adapter-Du7P_5Fx.d.ts → adapter-BJwn74rx.d.ts} +49 -1
  3. package/dist/adapters/generation-dispatch.d.ts +32 -7
  4. package/dist/adapters/generation-dispatch.js +1997 -216
  5. package/dist/adapters/generation-dispatch.js.map +1 -1
  6. package/dist/adapters/index.js +29 -6
  7. package/dist/adapters/index.js.map +1 -1
  8. package/dist/advanced/index.d.ts +15 -12
  9. package/dist/advanced/index.js +2013 -270
  10. package/dist/advanced/index.js.map +1 -1
  11. package/dist/{axes-CzLEMDeB.d.ts → axes-CKVCzR_x.d.ts} +1 -1
  12. package/dist/bank-Ctdp3FUA.d.ts +118 -0
  13. package/dist/blueprint-validator.js +416 -92
  14. package/dist/blueprint-validator.js.map +1 -1
  15. package/dist/boilerplate.d.ts +2 -2
  16. package/dist/boilerplate.js +34 -15
  17. package/dist/boilerplate.js.map +1 -1
  18. package/dist/check/index.d.ts +13 -5
  19. package/dist/check/index.js +56 -3
  20. package/dist/check/index.js.map +1 -1
  21. package/dist/classifier/index.d.ts +2 -2
  22. package/dist/compose.d.ts +2 -2
  23. package/dist/{create-ui-generator-GIhzx08x.d.ts → create-ui-generator-N4gRXpcL.d.ts} +11 -4
  24. package/dist/{design-mode-DLufhrNp.d.ts → design-mode-DGWN_Gkm.d.ts} +1 -1
  25. package/dist/{dispatch-BnAH56p4.d.ts → dispatch-BUoXRj_r.d.ts} +6 -4
  26. package/dist/evaluation/axis-checks/index.d.ts +7 -6
  27. package/dist/evaluation/axis-checks/index.js +90 -2
  28. package/dist/evaluation/axis-checks/index.js.map +1 -1
  29. package/dist/evaluation/axis-checks/registry.d.ts +6 -5
  30. package/dist/evaluation/axis-checks/registry.js +88 -1
  31. package/dist/evaluation/axis-checks/registry.js.map +1 -1
  32. package/dist/evaluation/index.d.ts +189 -344
  33. package/dist/evaluation/index.js +1272 -59
  34. package/dist/evaluation/index.js.map +1 -1
  35. package/dist/evaluation/types.d.ts +26 -1
  36. package/dist/evaluation/types.js.map +1 -1
  37. package/dist/fragments/index.d.ts +3 -3
  38. package/dist/{generate-9IeNT4ZY.d.ts → generate-tSO_shQ0.d.ts} +1 -1
  39. package/dist/harness/check/runtime-render/index.d.ts +7 -7
  40. package/dist/harness/check/runtime-render/index.js +416 -92
  41. package/dist/harness/check/runtime-render/index.js.map +1 -1
  42. package/dist/harness/check/runtime-render/render-check-worker.js +393 -88
  43. package/dist/harness/check/runtime-render/render-check-worker.js.map +1 -1
  44. package/dist/harness/index.d.ts +5 -6
  45. package/dist/harness/index.js +69 -20
  46. package/dist/harness/index.js.map +1 -1
  47. package/dist/harness/prompts.d.ts +2 -2
  48. package/dist/harness/prompts.js +10 -1
  49. package/dist/harness/prompts.js.map +1 -1
  50. package/dist/harness/result-types.d.ts +19 -5
  51. package/dist/harness/result-types.js.map +1 -1
  52. package/dist/harness/runtime.d.ts +21 -7
  53. package/dist/harness/runtime.js +36 -17
  54. package/dist/harness/runtime.js.map +1 -1
  55. package/dist/harness/types-public.d.ts +6 -4
  56. package/dist/harness/types-public.js +43 -18
  57. package/dist/harness/types-public.js.map +1 -1
  58. package/dist/harness/validator-trace-sink.d.ts +6 -5
  59. package/dist/hash.d.ts +1 -1
  60. package/dist/index.d.ts +64 -13
  61. package/dist/index.js +2023 -270
  62. package/dist/index.js.map +1 -1
  63. package/dist/patch.d.ts +5 -5
  64. package/dist/policy.d.ts +1 -1
  65. package/dist/{prompts-YU9piOgs.d.ts → prompts-BhNUuRoZ.d.ts} +1 -1
  66. package/dist/tools/get-primitives-ts.d.ts +1 -1
  67. package/dist/tools/get-primitives-ts.js +1 -1
  68. package/dist/tools/get-primitives-ts.js.map +1 -1
  69. package/dist/tools/render-check-worker.js +8 -1
  70. package/dist/tools/render-check-worker.js.map +1 -1
  71. package/dist/{types-BOvHNG7K.d.ts → types-Dh1s4K9j.d.ts} +1 -1
  72. package/dist/types-public-B7wn4ce5.d.ts +959 -0
  73. package/dist/validation/index.d.ts +1 -1
  74. package/dist/validation/index.js +1 -1
  75. package/dist/validation/index.js.map +1 -1
  76. package/dist/visual-evaluator-C9L-o6ap.d.ts +639 -0
  77. package/dist/workflows.d.ts +5 -5
  78. package/package.json +12 -12
  79. package/src/adapters/generation-dispatch.ts +38 -2
  80. package/src/adapters/tools.ts +2 -1
  81. package/src/boilerplate/free-design-prompt.ts +4 -4
  82. package/src/boilerplate/generate.ts +26 -1
  83. package/src/boilerplate/hard-sections.ts +14 -14
  84. package/src/boilerplate/system-prompt.ts +1 -1
  85. package/src/boilerplate/templates/layouts/chat-mobile.tsx.tmpl +3 -1
  86. package/src/boilerplate/templates/layouts/chat-universal.tsx.tmpl +3 -1
  87. package/src/boilerplate/templates/layouts/fullscreen-desktop.tsx.tmpl +3 -1
  88. package/src/boilerplate/templates/layouts/fullscreen-universal.tsx.tmpl +3 -1
  89. package/src/check/index.ts +1 -0
  90. package/src/check/run-tier0.ts +6 -3
  91. package/src/check/sample-props.ts +63 -0
  92. package/src/coding-agent/tools.ts +4 -1
  93. package/src/contract-context.ts +1 -1
  94. package/src/create-harness.ts +3 -3
  95. package/src/create-ui-generator.ts +20 -1
  96. package/src/evaluation/__fixtures__/png.ts +100 -0
  97. package/src/evaluation/axis-checks/checks/state-merge.ts +1 -1
  98. package/src/evaluation/axis-checks/checks/universal.ts +111 -0
  99. package/src/evaluation/axis-checks/dispatch.ts +4 -1
  100. package/src/evaluation/cost-tracker.ts +54 -6
  101. package/src/evaluation/criteria/bank.ts +85 -0
  102. package/src/evaluation/criteria/context.ts +52 -0
  103. package/src/evaluation/criteria/edge-probe.ts +286 -0
  104. package/src/evaluation/criteria/edge.ts +202 -0
  105. package/src/evaluation/criteria/resolve.ts +207 -0
  106. package/src/evaluation/criteria/select.ts +79 -0
  107. package/src/evaluation/index.ts +27 -0
  108. package/src/evaluation/ink-extent.ts +200 -0
  109. package/src/evaluation/judge-spend.ts +48 -0
  110. package/src/evaluation/theme-css.ts +12 -5
  111. package/src/evaluation/types-public.ts +221 -3
  112. package/src/evaluation/types.ts +19 -0
  113. package/src/evaluation/visual-evaluator.ts +853 -51
  114. package/src/harness/check/runtime-render/adapter.ts +18 -0
  115. package/src/harness/check/runtime-render/probe-pending.ts +111 -0
  116. package/src/harness/check/runtime-render/render-check-host.ts +10 -6
  117. package/src/harness/check/runtime-render/render-check-worker.ts +6 -3
  118. package/src/harness/check/runtime-render/render-check.ts +417 -133
  119. package/src/harness/check/runtime-render/walk.ts +200 -0
  120. package/src/harness/coding/assemble-result.ts +2 -0
  121. package/src/harness/coding/generate-task-runner.ts +30 -0
  122. package/src/harness/coding/init-session.ts +5 -3
  123. package/src/harness/coding/run-coding-turn.ts +12 -4
  124. package/src/harness/coding/run-eval-round.ts +82 -3
  125. package/src/harness/in-loop-evaluation.ts +39 -0
  126. package/src/harness/llm-router.ts +58 -1
  127. package/src/harness/result-types.ts +9 -0
  128. package/src/harness/runtime.ts +14 -0
  129. package/src/harness/types-public.ts +10 -1
  130. package/src/harness-tool-names.ts +38 -0
  131. package/src/index.ts +17 -0
  132. package/src/internal/verdict-line.ts +33 -0
  133. package/src/run-check.ts +2 -0
  134. package/src/taught-tokens.ts +32 -0
  135. package/src/tools/get-primitives-ts.ts +1 -1
  136. package/src/tools/get-wire.ts +1 -1
  137. package/src/tools/render-check-worker.ts +2 -1
  138. package/src/tools/render-check.ts +8 -67
  139. package/src/tools.ts +11 -1
  140. package/src/validation/primitives.ts +1 -1
  141. package/dist/llm-router-DwwUK1bz.d.ts +0 -265
  142. package/dist/types-public-Bk6d1WLN.d.ts +0 -451
@@ -0,0 +1,200 @@
1
+ /**
2
+ * The action walk (ggui#1652): find a control that dispatches one action by
3
+ * walking the card's screens, as a person would, in a mount of its own.
4
+ *
5
+ * A screen is the card's set of controls, told apart by each control's tag,
6
+ * type, label (its aria-label, placeholder or text) and disabled state, never
7
+ * by typed values, selection or generated names.
8
+ * Picking an answer is the same screen unless it enables or disables a
9
+ * control; a wizard's next step, or an Edit button that now reads Save, is a
10
+ * new one. Each control is pressed once per screen. When every control on
11
+ * the current screen has been pressed, the walk returns over controls it has
12
+ * already pressed to the nearest screen that still has one to press. A press
13
+ * that leads back to a screen already seen is not progress, so Back and Next
14
+ * cannot loop, in any language. The walk stops at the first press that
15
+ * dispatches the action, when every reachable control has been pressed, at
16
+ * a cap on presses, or at its deadline. Neither the cap nor the deadline
17
+ * stops it while it is on the first screen with a control there still
18
+ * unpressed; once a press leaves the first screen both apply, and a
19
+ * first-screen control left unpressed is pressed only if a known route
20
+ * leads back to it (a one-way wizard has none).
21
+ */
22
+ import type { MinimalElement } from "./host-boundary.js";
23
+
24
+ /** One control the walk can press, and how. */
25
+ export interface WalkControl {
26
+ readonly element: MinimalElement;
27
+ /** Whether the control names the action (its label, aria-label or `data-action`). */
28
+ readonly namesAction: boolean;
29
+ }
30
+
31
+ /** A fresh copy of the card, and how to fill its screen's empty form controls. */
32
+ export interface WalkMount {
33
+ readonly container: MinimalElement;
34
+ /** Fill the current screen's empty form controls; filled ones are left as they are. */
35
+ readonly prime: () => void;
36
+ }
37
+
38
+ export interface WalkInput<C extends WalkControl> {
39
+ /** Mount a fresh copy of the card; undefined when it cannot be mounted. */
40
+ readonly mount: () => Promise<WalkMount | undefined>;
41
+ /** The screen's controls, in the order they are tried. */
42
+ readonly controls: (container: MinimalElement) => readonly C[];
43
+ /** Press one control; true when the press dispatched the action. */
44
+ readonly press: (control: C) => Promise<boolean>;
45
+ readonly describe: (el: MinimalElement) => string;
46
+ /** Epoch ms after which no further press is made. */
47
+ readonly stopAt: number;
48
+ readonly maxPresses: number;
49
+ }
50
+
51
+ export type WalkStop = "dispatched" | "explored" | "press-cap" | "deadline" | "no-mount";
52
+
53
+ export interface WalkOutcome {
54
+ readonly fired: boolean;
55
+ /** The control whose press dispatched the action, and the mount it is in. */
56
+ readonly element?: MinimalElement;
57
+ readonly container?: MinimalElement;
58
+ readonly presses: number;
59
+ readonly screens: number;
60
+ readonly stoppedBy: WalkStop;
61
+ /** Whether any screen rendered a control that names the action. */
62
+ readonly namedControlSeen: boolean;
63
+ /** The last few controls pressed, oldest first. */
64
+ readonly lastPressed: readonly string[];
65
+ /** The text of every screen the walk reached, once each, in the order reached. */
66
+ readonly screenTexts: readonly string[];
67
+ }
68
+
69
+ /** How many of the last presses an outcome keeps. */
70
+ const LAST_PRESSED_KEPT = 4;
71
+
72
+ function isDisabled(el: MinimalElement): boolean {
73
+ return el.getAttribute("disabled") !== null || el.getAttribute("aria-disabled") === "true";
74
+ }
75
+
76
+ /**
77
+ * A control's label for the fingerprint. Never its `name` or `id`: React
78
+ * generates those (`useId`) afresh when a step re-mounts, so a screen keyed
79
+ * on them would look new on every return and the walk would loop on it.
80
+ */
81
+ function labelOf(el: MinimalElement): string {
82
+ const named = el.getAttribute("aria-label") ?? el.getAttribute("placeholder");
83
+ return (named ?? el.textContent ?? "").replace(/\s+/g, " ").trim().slice(0, 60);
84
+ }
85
+
86
+ /** The screen's identity: its controls' structure and disabled state, in document order. */
87
+ export function screenFingerprint(container: MinimalElement): string {
88
+ const parts: string[] = [];
89
+ for (const el of container.querySelectorAll(
90
+ 'button, [role="button"], a[href], input, select, textarea, form'
91
+ )) {
92
+ parts.push(
93
+ [el.tagName.toLowerCase(), el.getAttribute("type") ?? "", labelOf(el), isDisabled(el) ? "disabled" : ""].join("|")
94
+ );
95
+ }
96
+ return parts.join("\n");
97
+ }
98
+
99
+ interface Screen {
100
+ /** Each control's key, in try order. */
101
+ readonly order: readonly string[];
102
+ /** A pressed control's key → the screen its press led to. */
103
+ readonly led: Map<string, string>;
104
+ }
105
+
106
+ /** Each control's key on a screen: its description, numbered among controls described alike. */
107
+ function keyed<C extends WalkControl>(controls: readonly C[], describe: (el: MinimalElement) => string): Map<string, C> {
108
+ const seen = new Map<string, number>();
109
+ const out = new Map<string, C>();
110
+ for (const c of controls) {
111
+ const d = describe(c.element);
112
+ const n = seen.get(d) ?? 0;
113
+ seen.set(d, n + 1);
114
+ out.set(`${d}#${n}`, c);
115
+ }
116
+ return out;
117
+ }
118
+
119
+ /** The first press of the shortest known route to a screen with a control not yet pressed. */
120
+ function firstStepToUnpressed(screens: ReadonlyMap<string, Screen>, from: string): string | undefined {
121
+ const firstStep = new Map<string, string | undefined>([[from, undefined]]);
122
+ const queue = [from];
123
+ while (queue.length > 0) {
124
+ const at = queue.shift()!;
125
+ const screen = screens.get(at);
126
+ if (screen === undefined) continue;
127
+ if (at !== from && screen.order.some((k) => !screen.led.has(k))) return firstStep.get(at);
128
+ for (const [key, to] of screen.led) {
129
+ if (to === at || firstStep.has(to)) continue;
130
+ firstStep.set(to, firstStep.get(at) ?? key);
131
+ queue.push(to);
132
+ }
133
+ }
134
+ return undefined;
135
+ }
136
+
137
+ export async function walkToAction<C extends WalkControl>(input: WalkInput<C>): Promise<WalkOutcome> {
138
+ const lastPressed: string[] = [];
139
+ const screenTexts: string[] = [];
140
+ const screens = new Map<string, Screen>();
141
+ let presses = 0;
142
+ let namedControlSeen = false;
143
+ const outcome = (stoppedBy: WalkStop, extra: Partial<WalkOutcome> = {}): WalkOutcome => ({
144
+ fired: false,
145
+ presses,
146
+ screens: screens.size,
147
+ stoppedBy,
148
+ namedControlSeen,
149
+ lastPressed: [...lastPressed],
150
+ screenTexts: [...screenTexts],
151
+ ...extra,
152
+ });
153
+
154
+ const mounted = await input.mount();
155
+ if (mounted === undefined) return outcome("no-mount");
156
+ const { container } = mounted;
157
+
158
+ const arrive = (): { id: string; controls: Map<string, C> } => {
159
+ mounted.prime();
160
+ const controls = keyed(input.controls(container), input.describe);
161
+ if ([...controls.values()].some((c) => c.namesAction)) namedControlSeen = true;
162
+ const id = screenFingerprint(container);
163
+ if (!screens.has(id)) {
164
+ screens.set(id, { order: [...controls.keys()], led: new Map() });
165
+ screenTexts.push(container.textContent ?? "");
166
+ }
167
+ return { id, controls };
168
+ };
169
+
170
+ let here = arrive();
171
+ const first = here.id;
172
+ for (;;) {
173
+ const screen = screens.get(here.id)!;
174
+ const onOpenFirstScreen = here.id === first && screen.order.some((k) => !screen.led.has(k));
175
+ if (!onOpenFirstScreen) {
176
+ if (presses >= input.maxPresses) return outcome("press-cap");
177
+ if (Date.now() >= input.stopAt) return outcome("deadline");
178
+ }
179
+ const key = screen.order.find((k) => !screen.led.has(k)) ?? firstStepToUnpressed(screens, here.id);
180
+ if (key === undefined) return outcome("explored");
181
+ const control = here.controls.get(key);
182
+ if (control === undefined) {
183
+ // Listed when the screen was first seen, gone now: nothing to press.
184
+ // It now leads nowhere, which a route never takes, so this cannot
185
+ // pick the same control again.
186
+ screen.led.set(key, here.id);
187
+ continue;
188
+ }
189
+ presses += 1;
190
+ lastPressed.push(input.describe(control.element));
191
+ if (lastPressed.length > LAST_PRESSED_KEPT) lastPressed.shift();
192
+ if (await input.press(control)) {
193
+ return outcome("dispatched", { fired: true, element: control.element, container });
194
+ }
195
+ here = arrive();
196
+ // The first press records where a control leads; a route that later
197
+ // leads elsewhere is simply followed on from where it led.
198
+ if (!screen.led.has(key)) screen.led.set(key, here.id);
199
+ }
200
+ }
@@ -119,6 +119,8 @@ export async function assembleGenerationResult(
119
119
  ...(telemetry.cacheCreationTokens !== undefined
120
120
  ? { cacheCreationTokens: telemetry.cacheCreationTokens }
121
121
  : {}),
122
+ // ggui#1522 — the in-loop visual judge's spend, beside `tokens` and never inside it.
123
+ ...(telemetry.inLoopVisualTokens !== undefined ? { inLoopVisualTokens: telemetry.inLoopVisualTokens } : {}),
122
124
  generationTimeMs: totalMs,
123
125
  turnsUsed: telemetry.turnsUsed,
124
126
  passesUsed: 1,
@@ -24,6 +24,7 @@ import {
24
24
  } from "./run-coding-turn.js";
25
25
  import { runEvalRound } from "./run-eval-round.js";
26
26
  import type { TokenUsage } from "./run-eval-round.js";
27
+ import type { VisualLegTokens } from "../../evaluation/visual-evaluator.js";
27
28
  import type { SameExchangeBreak } from "../result-types.js";
28
29
 
29
30
  type PreWarmedEvalContext =
@@ -75,6 +76,12 @@ export interface GenerateTelemetry {
75
76
  */
76
77
  cacheReadTokens?: number;
77
78
  cacheCreationTokens?: number;
79
+ /**
80
+ * ggui#1522 — the in-loop visual judge's spend across eval rounds, apart from {@link totalIn} / {@link totalOut}
81
+ * (priced at the coding and text-evaluator models): this part is priced at the VISUAL agent's model. Stays
82
+ * `undefined` while no round reported one.
83
+ */
84
+ inLoopVisualTokens?: VisualLegTokens;
78
85
  /** Latest evalResult — undefined when eval was disabled or never ran. */
79
86
  evalResult: EvalResult | undefined;
80
87
  /**
@@ -167,6 +174,25 @@ export function absorbTokens(telemetry: GenerateTelemetry, usage: TokenUsage): v
167
174
  }
168
175
  }
169
176
 
177
+ /** ggui#1522 — fold one round's in-loop visual spend into the run's own part (never into `totalIn` / `totalOut`). */
178
+ export function absorbInLoopVisualTokens(telemetry: GenerateTelemetry, part: VisualLegTokens | undefined): void {
179
+ if (part === undefined) return;
180
+ const prev = telemetry.inLoopVisualTokens;
181
+ const prevCriteria = prev?.criteria;
182
+ const criteria =
183
+ part.criteria === undefined
184
+ ? prevCriteria
185
+ : {
186
+ inputTokens: (prevCriteria?.inputTokens ?? 0) + part.criteria.inputTokens,
187
+ outputTokens: (prevCriteria?.outputTokens ?? 0) + part.criteria.outputTokens,
188
+ };
189
+ telemetry.inLoopVisualTokens = {
190
+ inputTokens: (prev?.inputTokens ?? 0) + part.inputTokens,
191
+ outputTokens: (prev?.outputTokens ?? 0) + part.outputTokens,
192
+ ...(criteria !== undefined ? { criteria } : {}),
193
+ };
194
+ }
195
+
170
196
  /**
171
197
  * Fold one coding turn's measurements into the run's telemetry — tokens (via
172
198
  * {@link absorbTokens}), timings, the phase/outcome counters, and the
@@ -260,6 +286,8 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
260
286
  originalPrompt: params.userPrompt,
261
287
  commitMeta: session.commitMeta,
262
288
  originalProps: session.originalProps,
289
+ // ggui#1285: the fixture the probe and judge render with, for the echo check at auto-commit.
290
+ ...(params.fixtureProps !== undefined ? { fixtureProps: params.fixtureProps } : {}),
263
291
  costTracker: session.costTracker,
264
292
  // Thread the dispatch-resolved policy so experimental
265
293
  // profiles take effect on both preflight and tier-0 retry
@@ -379,6 +407,7 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
379
407
  evaluationAgent: session.agents.evaluation,
380
408
  visualEvalAgent: session.agents.visualEval,
381
409
  visualEvaluation: params.visualEvaluation,
410
+ shellType: params.shellType,
382
411
  visualThreshold: session.visualThreshold,
383
412
  qualityMode: session.qualityMode,
384
413
  maxEvalRounds: session.maxEvalRounds,
@@ -410,6 +439,7 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
410
439
  if (round.contractFeedback) telemetry.contractFeedback = round.contractFeedback;
411
440
  evalDone = round.evalDone;
412
441
  absorbTokens(telemetry, round.evalTokens);
442
+ absorbInLoopVisualTokens(telemetry, round.inLoopVisualTokens);
413
443
 
414
444
  if (round.control === "break") break;
415
445
  // round.control === "feedback" — set next coding turn input.
@@ -19,6 +19,7 @@ import type { AgentSpec, SingleComponentParams } from "../runtime.js";
19
19
  import type { ResolvedRunPolicy } from "../policy.js";
20
20
  import { DEFAULT_HARNESS_POLICY } from "../policy.js";
21
21
  import { createDupeBreakState, type DupeBreakState } from "./dupe-break.js";
22
+ import { inLoopEvaluation } from "../in-loop-evaluation.js";
22
23
 
23
24
  type PreWarmedEvalContext =
24
25
  import("../../evaluation/llm-evaluator.js").PreWarmedEvalContext;
@@ -165,10 +166,13 @@ export async function initSession(input: {
165
166
  "required",
166
167
  );
167
168
 
169
+ // ggui#1513 — the one rule for which evaluation legs run; a host reads the same export.
170
+ const { text: codeEvalEnabled, visual: visualEvalEnabled } = inLoopEvaluation(params);
171
+
168
172
  // ── Pre-warm eval (overlaps with coding turn 1) ──
169
173
  // Uses a separate agent instance so it doesn't disturb the coding cache.
170
174
  let preWarmPromise: Promise<PreWarmedEvalContext | null> | undefined;
171
- if (params.evaluation?.enabled) {
175
+ if (codeEvalEnabled) {
172
176
  const evalSpec = agents.evaluation;
173
177
  preWarmPromise = import("../../evaluation/llm-evaluator.js")
174
178
  .then((mod) =>
@@ -202,8 +206,6 @@ export async function initSession(input: {
202
206
  // ── Eval gate + lazy-loaded modules ──
203
207
  const qualityConfig = params.qualityConfig;
204
208
  const qualityMode = qualityConfig?.quality ?? "fast";
205
- const codeEvalEnabled = !!params.evaluation?.enabled;
206
- const visualEvalEnabled = !!(params.visualEvaluation?.enabled || qualityConfig?.visualEval);
207
209
  const probeOnlyEnabled =
208
210
  harness.check.runtimeRender !== undefined && !codeEvalEnabled && !visualEvalEnabled;
209
211
  const maxEvalRounds =
@@ -14,7 +14,7 @@
14
14
  // Behavior MUST match the inlined original byte-for-byte — this is a
15
15
  // mechanical extraction, not a redesign.
16
16
 
17
- import type { DataContract } from "@ggui-ai/protocol";
17
+ import type { DataContract, JsonObject } from "@ggui-ai/protocol";
18
18
  import type { CommitMetadata } from "../../coding-agent/types.js";
19
19
  import type { AgentWorkspace } from "../../coding-agent/workspace.js";
20
20
  import { executeTool } from "../../coding-agent/tools.js";
@@ -30,9 +30,10 @@ import {
30
30
  APPLY_CHANGES_TOOL_SCOPED,
31
31
  GET_COMPONENTS_INFO_TOOL,
32
32
  GET_ICONS_TOOL,
33
+ ICON_LIST_NO_MATCH_GUIDANCE,
33
34
  REWRITE_TOOL,
34
- WRITE_PLAN_TOOL,
35
35
  type LLMToolDef,
36
+ WRITE_PLAN_TOOL,
36
37
  } from "../../tools.js";
37
38
  import { LUCIDE_ICON_NAMES } from "@ggui-ai/design";
38
39
  import { formatWithHashlines } from "../hashline.js";
@@ -165,6 +166,8 @@ export interface CodingTurnContext {
165
166
  readonly originalPrompt: string;
166
167
  readonly commitMeta: Map<string, CommitMetadata>;
167
168
  readonly originalProps: string | undefined;
169
+ /** The props the probe and judge render with (ggui#1285) — handed to auto-commit's axis checks; absent = the echo check stands down. */
170
+ readonly fixtureProps?: JsonObject;
168
171
  readonly costTracker: CostTracker | null;
169
172
  /**
170
173
  * Runtime-resolved context policy. Defaults to `harness.policy.context`
@@ -541,7 +544,11 @@ ${closingInstruction}`;
541
544
  );
542
545
  const llmMs = Date.now() - llmStart;
543
546
  if (costTracker) {
544
- costTracker.record(codingModel, response.inputTokens, response.outputTokens);
547
+ // ggui#1524 — the cache counts reach the cap: a cached prefix is billed, so it is spend.
548
+ costTracker.record(codingModel, response.inputTokens, response.outputTokens, {
549
+ ...(response.cacheReadTokens !== undefined ? { read: response.cacheReadTokens } : {}),
550
+ ...(response.cacheCreationTokens !== undefined ? { write: response.cacheCreationTokens } : {}),
551
+ });
545
552
  }
546
553
 
547
554
  const tokens = { input: response.inputTokens, output: response.outputTokens };
@@ -614,7 +621,7 @@ ${closingInstruction}`;
614
621
  iconNamesCache = LUCIDE_ICON_NAMES.join(", ");
615
622
  }
616
623
  console.log(`[simple] turn ${turnsUsed}: get_available_icons`);
617
- let iconResult = `Available Lucide icon names (use with <Icon name="...">):\n${iconNamesCache}`;
624
+ let iconResult = `Available Lucide icon names (use with <Icon name="...">):\n${iconNamesCache}\n\n${ICON_LIST_NO_MATCH_GUIDANCE}`;
618
625
  // ggui#404 same-call circuit breaker. The icon list is constant by
619
626
  // construction, so every repeat is an identical exchange — record on
620
627
  // the BASE text (pre-nudge) so the chain keys stay stable across the
@@ -690,6 +697,7 @@ ${closingInstruction}`;
690
697
  classification: harness.classification,
691
698
  originalPrompt: ctx.originalPrompt,
692
699
  ...(harness.canvas !== undefined ? { canvas: harness.canvas } : {}),
700
+ ...(ctx.fixtureProps !== undefined ? { fixtureProps: ctx.fixtureProps } : {}),
693
701
  },
694
702
  );
695
703
  const toolMs = Date.now() - toolStart;
@@ -20,6 +20,7 @@
20
20
  // mechanical extraction, not a redesign.
21
21
 
22
22
  import type { DataContract, JsonObject } from "@ggui-ai/protocol";
23
+ import { criteriaContextFor } from "../../evaluation/criteria/context.js";
23
24
  import { listContractGadgets } from "@ggui-ai/protocol";
24
25
  import type { Classification } from "../../classifier/index.js";
25
26
  import type { AgentWorkspace } from "../../coding-agent/workspace.js";
@@ -51,6 +52,7 @@ import type {
51
52
  VisualEvalOutcome,
52
53
  VisualFitConfig,
53
54
  VisualFitOutcome,
55
+ VisualLegTokens,
54
56
  } from "../../evaluation/visual-evaluator.js";
55
57
 
56
58
  type PreWarmedEvalContext =
@@ -74,6 +76,8 @@ export interface EvalRoundContext {
74
76
  readonly evaluationAgent: AgentSpec;
75
77
  readonly visualEvalAgent: AgentSpec;
76
78
  readonly visualEvaluation: SingleComponentParams["visualEvaluation"];
79
+ /** ggui#1436 — the shell as the caller named it, for the criteria selector's context; absent reads `unknown`. */
80
+ readonly shellType?: SingleComponentParams["shellType"];
77
81
  readonly visualThreshold: number;
78
82
  readonly qualityMode: "fast" | "auto-improve" | "high-quality";
79
83
  readonly maxEvalRounds: number;
@@ -178,6 +182,12 @@ export interface EvalRoundResult {
178
182
  readonly preWarmedContext: PreWarmedEvalContext | null | undefined;
179
183
  /** Tokens added during this round — caller adds to its own totals. */
180
184
  readonly evalTokens: TokenUsage;
185
+ /**
186
+ * ggui#1522 — the in-loop visual judge's spend this round, apart from {@link evalTokens} (the text evaluator's,
187
+ * priced at its model): already recorded in the cost tracker at the visual agent's model. Absent when no visual
188
+ * leg ran or it reported no tokens.
189
+ */
190
+ readonly inLoopVisualTokens?: VisualLegTokens;
181
191
  /** Wall-clock of the parallel LLM + visual eval block (zero on the
182
192
  * low-risk bypass path, which exits before `Promise.all`). Caller
183
193
  * accumulates into its `cumulativeEvalLlmMs`. */
@@ -414,6 +424,7 @@ function probeMetaOf(outcome: RuntimeRenderOutcome, probeIssues: readonly EvalIs
414
424
  ...(outcome.renderMs !== undefined ? { renderMs: outcome.renderMs } : {}),
415
425
  ...(outcome.hostLoad !== undefined ? { hostLoad: outcome.hostLoad } : {}),
416
426
  ...(outcome.queuedMs !== undefined ? { queuedMs: outcome.queuedMs } : {}),
427
+ ...(outcome.pendingAffordance !== undefined ? { pendingAffordance: outcome.pendingAffordance } : {}),
417
428
  };
418
429
  const elapsed = outcome.elapsedMs !== undefined ? { elapsedMs: outcome.elapsedMs } : {};
419
430
  if (outcome.status !== "ran") {
@@ -588,6 +599,7 @@ export async function runEvalRound(
588
599
 
589
600
  let evalResult: EvalResult | undefined;
590
601
  let evalTokens: TokenUsage = { input: 0, output: 0 };
602
+ let inLoopVisualTokens: VisualLegTokens | undefined = undefined;
591
603
  let evalLlmMs = 0;
592
604
 
593
605
  try {
@@ -638,6 +650,7 @@ export async function runEvalRound(
638
650
  prevFailFingerprints,
639
651
  preWarmedContext,
640
652
  evalTokens,
653
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
641
654
  evalLlmMs,
642
655
  lastResultText: "",
643
656
  isEvalFeedback: false,
@@ -662,6 +675,7 @@ export async function runEvalRound(
662
675
  prevFailFingerprints,
663
676
  preWarmedContext,
664
677
  evalTokens,
678
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
665
679
  evalLlmMs,
666
680
  lastResultText: lines.join("\n\n"),
667
681
  isEvalFeedback: true,
@@ -680,6 +694,7 @@ export async function runEvalRound(
680
694
  prevFailFingerprints,
681
695
  preWarmedContext,
682
696
  evalTokens,
697
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
683
698
  evalLlmMs,
684
699
  lastResultText: "",
685
700
  isEvalFeedback: false,
@@ -769,6 +784,7 @@ export async function runEvalRound(
769
784
  prevFailFingerprints,
770
785
  preWarmedContext,
771
786
  evalTokens,
787
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
772
788
  evalLlmMs,
773
789
  lastResultText: lines.join("\n\n"),
774
790
  isEvalFeedback: true,
@@ -794,6 +810,7 @@ export async function runEvalRound(
794
810
  prevFailFingerprints,
795
811
  preWarmedContext,
796
812
  evalTokens,
813
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
797
814
  evalLlmMs,
798
815
  lastResultText: "",
799
816
  isEvalFeedback: false,
@@ -822,6 +839,22 @@ export async function runEvalRound(
822
839
  profile: harness.profile,
823
840
  // The caller's theme, when named — the round paints what the app paints.
824
841
  ...(visualEvaluation?.cssTokens !== undefined ? { cssTokens: visualEvaluation.cssTokens } : {}),
842
+ // ggui#1436 — the criteria bank and the card's context; the judge selects per canvas, report-only.
843
+ ...(visualEvaluation?.criteriaBank !== undefined
844
+ ? {
845
+ criteria: {
846
+ bank: visualEvaluation.criteriaBank,
847
+ context: criteriaContextFor({
848
+ classification,
849
+ contract,
850
+ cssTokens: visualEvaluation.cssTokens,
851
+ profile: harness.profile,
852
+ shell: ctx.shellType ?? "unknown",
853
+ ...(visualEvaluation.criteriaKind !== undefined ? { kind: visualEvaluation.criteriaKind } : {}),
854
+ }),
855
+ },
856
+ }
857
+ : {}),
825
858
  };
826
859
  const visualFrame: VisualFitConfig = {
827
860
  // The in-loop visual round renders the same sample the runtime probe
@@ -898,6 +931,18 @@ export async function runEvalRound(
898
931
  evalLlmMs = Date.now() - evalLlmStart;
899
932
  llmResult = llm;
900
933
  visualIssues = visual?.issues ?? null;
934
+ // ggui#1522 — the in-loop visual judge's own spend: recorded in the tracker at the VISUAL agent's model, so
935
+ // `canContinue()` sees it, and carried apart from `evalTokens` (the text evaluator's model).
936
+ // The report-only criteria calls (ggui#1436) count toward the cap too, deliberately: the cap bounds MONEY SPENT,
937
+ // and a cap that left out real spend would under-count exactly as this leg did before. So under a tight cap a
938
+ // configured criteria bank's calls can end the eval loop earlier — the one place a report-only block reaches an
939
+ // outcome, and it does so through its cost, never through its answers. The tracker's total feeds no metered figure.
940
+ inLoopVisualTokens = visual?.tokens;
941
+ if (inLoopVisualTokens !== undefined) {
942
+ costTracker.record(visualEvalAgent.model, inLoopVisualTokens.inputTokens, inLoopVisualTokens.outputTokens);
943
+ const criteriaSpend = inLoopVisualTokens.criteria;
944
+ if (criteriaSpend !== undefined) costTracker.record(visualEvalAgent.model, criteriaSpend.inputTokens, criteriaSpend.outputTokens);
945
+ }
901
946
  const visualSummary: VisualEvalSummary | undefined = visual?.summary;
902
947
  // ggui#1221 — the judge leg's coverage is ALWAYS stamped: ran / skipped (with the
903
948
  // reason) / not-applicable when no visual leg was configured for this round.
@@ -908,7 +953,11 @@ export async function runEvalRound(
908
953
 
909
954
  // Track costs from LLM eval calls
910
955
  if (llmResult) {
911
- costTracker.record(evaluationAgent.model, llmResult.inputTokens, llmResult.outputTokens);
956
+ // ggui#1524 — the text evaluator's cache counts reach the cap too.
957
+ costTracker.record(evaluationAgent.model, llmResult.inputTokens, llmResult.outputTokens, {
958
+ ...(llmResult.cacheReadTokens !== undefined ? { read: llmResult.cacheReadTokens } : {}),
959
+ ...(llmResult.cacheCreationTokens !== undefined ? { write: llmResult.cacheCreationTokens } : {}),
960
+ });
912
961
  evalTokens = {
913
962
  input: llmResult.inputTokens,
914
963
  output: llmResult.outputTokens,
@@ -977,12 +1026,22 @@ export async function runEvalRound(
977
1026
  );
978
1027
 
979
1028
  // ── Decide whether to continue the loop ──
980
- // In fast mode: only fails trigger another iteration.
1029
+ // In fast mode: only fails trigger another iteration, plus (ggui#1542) the visual JUDGE's major findings when
1030
+ // `visualEvaluation.actOn` is 'major'. They are taken from the visual outcome itself, never by category over the
1031
+ // merged list, because the text evaluator has a tier-2 `visual` criterion of its own. Only findings whose
1032
+ // `origin` is `judge` act (ggui#1545): the instruments' warn-level majors (fit's `canvas-overflow*` today) are
1033
+ // deterministic and run with or without a judge, so acting on them would switch a second feedback source on
1034
+ // with the judge's. Selecting by origin rather than excluding by name fails closed: a new instrument check, or
1035
+ // any issue with no origin, never joins the treatment unless it is marked a judge finding.
981
1036
  // In auto-improve / high-quality: fails + warns trigger iterations.
982
1037
  const blocking = evalResult.issues.filter((i) => i.result === "fail");
1038
+ const visualMajorsActed =
1039
+ visualEvaluation?.actOn === "major"
1040
+ ? (visualIssues ?? []).filter((i) => i.origin === "judge" && i.result === "warn" && i.severity === "major")
1041
+ : [];
983
1042
  const shouldContinue =
984
1043
  qualityMode === "fast"
985
- ? blocking.length > 0
1044
+ ? blocking.length > 0 || visualMajorsActed.length > 0
986
1045
  : blocking.length > 0 || warns.length > 0;
987
1046
 
988
1047
  // Compute current round's fail fingerprints — used by the stuck-loop
@@ -1027,6 +1086,7 @@ export async function runEvalRound(
1027
1086
  prevFailFingerprints: enrichedFingerprints,
1028
1087
  preWarmedContext,
1029
1088
  evalTokens,
1089
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
1030
1090
  evalLlmMs,
1031
1091
  lastResultText: lines.join("\n\n"),
1032
1092
  isEvalFeedback: true,
@@ -1060,6 +1120,7 @@ export async function runEvalRound(
1060
1120
  prevFailFingerprints: new Set([...currFailFingerprints, ...contractFails.map(fingerprintFail)]),
1061
1121
  preWarmedContext,
1062
1122
  evalTokens,
1123
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
1063
1124
  evalLlmMs,
1064
1125
  lastResultText: lines.join("\n\n"),
1065
1126
  isEvalFeedback: true,
@@ -1086,6 +1147,7 @@ export async function runEvalRound(
1086
1147
  prevFailFingerprints: currFailFingerprints,
1087
1148
  preWarmedContext,
1088
1149
  evalTokens,
1150
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
1089
1151
  evalLlmMs,
1090
1152
  lastResultText: "",
1091
1153
  isEvalFeedback: false,
@@ -1167,6 +1229,7 @@ export async function runEvalRound(
1167
1229
  prevFailFingerprints: new Set([...currFailFingerprints, ...stuckContractFails.map(fingerprintFail)]),
1168
1230
  preWarmedContext,
1169
1231
  evalTokens,
1232
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
1170
1233
  evalLlmMs,
1171
1234
  lastResultText: lines.join("\n\n"),
1172
1235
  isEvalFeedback: true,
@@ -1183,6 +1246,7 @@ export async function runEvalRound(
1183
1246
  prevFailFingerprints: currFailFingerprints,
1184
1247
  preWarmedContext,
1185
1248
  evalTokens,
1249
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
1186
1250
  evalLlmMs,
1187
1251
  lastResultText: "",
1188
1252
  isEvalFeedback: false,
@@ -1253,6 +1317,7 @@ export async function runEvalRound(
1253
1317
  prevFailFingerprints: new Set([...currFailFingerprints, ...fitFails.map(fingerprintFail)]),
1254
1318
  preWarmedContext,
1255
1319
  evalTokens,
1320
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
1256
1321
  evalLlmMs,
1257
1322
  lastResultText: fitLines.join("\n\n"),
1258
1323
  isEvalFeedback: true,
@@ -1282,6 +1347,7 @@ export async function runEvalRound(
1282
1347
  prevFailFingerprints: new Set([...currFailFingerprints, ...capContractFails.map(fingerprintFail)]),
1283
1348
  preWarmedContext,
1284
1349
  evalTokens,
1350
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
1285
1351
  evalLlmMs,
1286
1352
  lastResultText: contractLines.join("\n\n"),
1287
1353
  isEvalFeedback: true,
@@ -1308,6 +1374,7 @@ export async function runEvalRound(
1308
1374
  prevFailFingerprints: currFailFingerprints,
1309
1375
  preWarmedContext,
1310
1376
  evalTokens,
1377
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
1311
1378
  evalLlmMs,
1312
1379
  lastResultText: "",
1313
1380
  isEvalFeedback: false,
@@ -1340,6 +1407,7 @@ export async function runEvalRound(
1340
1407
  ]),
1341
1408
  preWarmedContext,
1342
1409
  evalTokens,
1410
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
1343
1411
  evalLlmMs,
1344
1412
  lastResultText: probeLines.join("\n\n"),
1345
1413
  isEvalFeedback: true,
@@ -1361,6 +1429,7 @@ export async function runEvalRound(
1361
1429
  prevFailFingerprints: currFailFingerprints,
1362
1430
  preWarmedContext,
1363
1431
  evalTokens,
1432
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
1364
1433
  evalLlmMs,
1365
1434
  lastResultText: "",
1366
1435
  isEvalFeedback: false,
@@ -1393,6 +1462,14 @@ export async function runEvalRound(
1393
1462
  console.log(
1394
1463
  `[simple] eval round ${evalRoundsUsed}: ${allActionable.length} issues (${blocking.length} fail, ${warns.length} warn) → feeding back to coding loop`,
1395
1464
  );
1465
+ // ggui#1542 — WHY this round continued, for a caller that set `actOn`: the manipulation check of an experiment on
1466
+ // the setting reads this line. `visual majors acted` counts the judge's majors only (fit's are never acted on).
1467
+ if (visualEvaluation?.actOn !== undefined) {
1468
+ console.log(
1469
+ `[simple] eval round ${evalRoundsUsed}: continuing: fails=${blocking.length}, visual majors acted=${visualMajorsActed.length} ` +
1470
+ `(actOn=${visualEvaluation.actOn})`,
1471
+ );
1472
+ }
1396
1473
 
1397
1474
  return {
1398
1475
  control: "feedback",
@@ -1412,6 +1489,7 @@ export async function runEvalRound(
1412
1489
  ),
1413
1490
  preWarmedContext,
1414
1491
  evalTokens,
1492
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
1415
1493
  evalLlmMs,
1416
1494
  lastResultText: issueLines.join("\n\n"),
1417
1495
  isEvalFeedback: true,
@@ -1444,6 +1522,7 @@ export async function runEvalRound(
1444
1522
  prevFailFingerprints,
1445
1523
  preWarmedContext,
1446
1524
  evalTokens,
1525
+ ...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
1447
1526
  evalLlmMs,
1448
1527
  lastResultText: "",
1449
1528
  isEvalFeedback: false,