@ggui-ai/ui-gen 0.24.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{adapter-Du7P_5Fx.d.ts → adapter-BJwn74rx.d.ts} +49 -1
- package/dist/adapters/generation-dispatch.d.ts +32 -7
- package/dist/adapters/generation-dispatch.js +1997 -216
- package/dist/adapters/generation-dispatch.js.map +1 -1
- package/dist/adapters/index.js +29 -6
- package/dist/adapters/index.js.map +1 -1
- package/dist/advanced/index.d.ts +15 -12
- package/dist/advanced/index.js +2013 -270
- package/dist/advanced/index.js.map +1 -1
- package/dist/{axes-CzLEMDeB.d.ts → axes-CKVCzR_x.d.ts} +1 -1
- package/dist/bank-Ctdp3FUA.d.ts +118 -0
- package/dist/blueprint-validator.js +416 -92
- package/dist/blueprint-validator.js.map +1 -1
- package/dist/boilerplate.d.ts +2 -2
- package/dist/boilerplate.js +34 -15
- package/dist/boilerplate.js.map +1 -1
- package/dist/check/index.d.ts +13 -5
- package/dist/check/index.js +56 -3
- package/dist/check/index.js.map +1 -1
- package/dist/classifier/index.d.ts +2 -2
- package/dist/compose.d.ts +2 -2
- package/dist/{create-ui-generator-GIhzx08x.d.ts → create-ui-generator-N4gRXpcL.d.ts} +11 -4
- package/dist/{design-mode-DLufhrNp.d.ts → design-mode-DGWN_Gkm.d.ts} +1 -1
- package/dist/{dispatch-BnAH56p4.d.ts → dispatch-BUoXRj_r.d.ts} +6 -4
- package/dist/evaluation/axis-checks/index.d.ts +7 -6
- package/dist/evaluation/axis-checks/index.js +90 -2
- package/dist/evaluation/axis-checks/index.js.map +1 -1
- package/dist/evaluation/axis-checks/registry.d.ts +6 -5
- package/dist/evaluation/axis-checks/registry.js +88 -1
- package/dist/evaluation/axis-checks/registry.js.map +1 -1
- package/dist/evaluation/index.d.ts +189 -344
- package/dist/evaluation/index.js +1272 -59
- package/dist/evaluation/index.js.map +1 -1
- package/dist/evaluation/types.d.ts +26 -1
- package/dist/evaluation/types.js.map +1 -1
- package/dist/fragments/index.d.ts +3 -3
- package/dist/{generate-9IeNT4ZY.d.ts → generate-tSO_shQ0.d.ts} +1 -1
- package/dist/harness/check/runtime-render/index.d.ts +7 -7
- package/dist/harness/check/runtime-render/index.js +416 -92
- package/dist/harness/check/runtime-render/index.js.map +1 -1
- package/dist/harness/check/runtime-render/render-check-worker.js +393 -88
- package/dist/harness/check/runtime-render/render-check-worker.js.map +1 -1
- package/dist/harness/index.d.ts +5 -6
- package/dist/harness/index.js +69 -20
- package/dist/harness/index.js.map +1 -1
- package/dist/harness/prompts.d.ts +2 -2
- package/dist/harness/prompts.js +10 -1
- package/dist/harness/prompts.js.map +1 -1
- package/dist/harness/result-types.d.ts +19 -5
- package/dist/harness/result-types.js.map +1 -1
- package/dist/harness/runtime.d.ts +21 -7
- package/dist/harness/runtime.js +36 -17
- package/dist/harness/runtime.js.map +1 -1
- package/dist/harness/types-public.d.ts +6 -4
- package/dist/harness/types-public.js +43 -18
- package/dist/harness/types-public.js.map +1 -1
- package/dist/harness/validator-trace-sink.d.ts +6 -5
- package/dist/hash.d.ts +1 -1
- package/dist/index.d.ts +64 -13
- package/dist/index.js +2023 -270
- package/dist/index.js.map +1 -1
- package/dist/patch.d.ts +5 -5
- package/dist/policy.d.ts +1 -1
- package/dist/{prompts-YU9piOgs.d.ts → prompts-BhNUuRoZ.d.ts} +1 -1
- package/dist/tools/get-primitives-ts.d.ts +1 -1
- package/dist/tools/get-primitives-ts.js +1 -1
- package/dist/tools/get-primitives-ts.js.map +1 -1
- package/dist/tools/render-check-worker.js +8 -1
- package/dist/tools/render-check-worker.js.map +1 -1
- package/dist/{types-BOvHNG7K.d.ts → types-Dh1s4K9j.d.ts} +1 -1
- package/dist/types-public-B7wn4ce5.d.ts +959 -0
- package/dist/validation/index.d.ts +1 -1
- package/dist/validation/index.js +1 -1
- package/dist/validation/index.js.map +1 -1
- package/dist/visual-evaluator-C9L-o6ap.d.ts +639 -0
- package/dist/workflows.d.ts +5 -5
- package/package.json +12 -12
- package/src/adapters/generation-dispatch.ts +38 -2
- package/src/adapters/tools.ts +2 -1
- package/src/boilerplate/free-design-prompt.ts +4 -4
- package/src/boilerplate/generate.ts +26 -1
- package/src/boilerplate/hard-sections.ts +14 -14
- package/src/boilerplate/system-prompt.ts +1 -1
- package/src/boilerplate/templates/layouts/chat-mobile.tsx.tmpl +3 -1
- package/src/boilerplate/templates/layouts/chat-universal.tsx.tmpl +3 -1
- package/src/boilerplate/templates/layouts/fullscreen-desktop.tsx.tmpl +3 -1
- package/src/boilerplate/templates/layouts/fullscreen-universal.tsx.tmpl +3 -1
- package/src/check/index.ts +1 -0
- package/src/check/run-tier0.ts +6 -3
- package/src/check/sample-props.ts +63 -0
- package/src/coding-agent/tools.ts +4 -1
- package/src/contract-context.ts +1 -1
- package/src/create-harness.ts +3 -3
- package/src/create-ui-generator.ts +20 -1
- package/src/evaluation/__fixtures__/png.ts +100 -0
- package/src/evaluation/axis-checks/checks/state-merge.ts +1 -1
- package/src/evaluation/axis-checks/checks/universal.ts +111 -0
- package/src/evaluation/axis-checks/dispatch.ts +4 -1
- package/src/evaluation/cost-tracker.ts +54 -6
- package/src/evaluation/criteria/bank.ts +85 -0
- package/src/evaluation/criteria/context.ts +52 -0
- package/src/evaluation/criteria/edge-probe.ts +286 -0
- package/src/evaluation/criteria/edge.ts +202 -0
- package/src/evaluation/criteria/resolve.ts +207 -0
- package/src/evaluation/criteria/select.ts +79 -0
- package/src/evaluation/index.ts +27 -0
- package/src/evaluation/ink-extent.ts +200 -0
- package/src/evaluation/judge-spend.ts +48 -0
- package/src/evaluation/theme-css.ts +12 -5
- package/src/evaluation/types-public.ts +221 -3
- package/src/evaluation/types.ts +19 -0
- package/src/evaluation/visual-evaluator.ts +853 -51
- package/src/harness/check/runtime-render/adapter.ts +18 -0
- package/src/harness/check/runtime-render/probe-pending.ts +111 -0
- package/src/harness/check/runtime-render/render-check-host.ts +10 -6
- package/src/harness/check/runtime-render/render-check-worker.ts +6 -3
- package/src/harness/check/runtime-render/render-check.ts +417 -133
- package/src/harness/check/runtime-render/walk.ts +200 -0
- package/src/harness/coding/assemble-result.ts +2 -0
- package/src/harness/coding/generate-task-runner.ts +30 -0
- package/src/harness/coding/init-session.ts +5 -3
- package/src/harness/coding/run-coding-turn.ts +12 -4
- package/src/harness/coding/run-eval-round.ts +82 -3
- package/src/harness/in-loop-evaluation.ts +39 -0
- package/src/harness/llm-router.ts +58 -1
- package/src/harness/result-types.ts +9 -0
- package/src/harness/runtime.ts +14 -0
- package/src/harness/types-public.ts +10 -1
- package/src/harness-tool-names.ts +38 -0
- package/src/index.ts +17 -0
- package/src/internal/verdict-line.ts +33 -0
- package/src/run-check.ts +2 -0
- package/src/taught-tokens.ts +32 -0
- package/src/tools/get-primitives-ts.ts +1 -1
- package/src/tools/get-wire.ts +1 -1
- package/src/tools/render-check-worker.ts +2 -1
- package/src/tools/render-check.ts +8 -67
- package/src/tools.ts +11 -1
- package/src/validation/primitives.ts +1 -1
- package/dist/llm-router-DwwUK1bz.d.ts +0 -265
- package/dist/types-public-Bk6d1WLN.d.ts +0 -451
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The action walk (ggui#1652): find a control that dispatches one action by
|
|
3
|
+
* walking the card's screens, as a person would, in a mount of its own.
|
|
4
|
+
*
|
|
5
|
+
* A screen is the card's set of controls, told apart by each control's tag,
|
|
6
|
+
* type, label (its aria-label, placeholder or text) and disabled state, never
|
|
7
|
+
* by typed values, selection or generated names.
|
|
8
|
+
* Picking an answer is the same screen unless it enables or disables a
|
|
9
|
+
* control; a wizard's next step, or an Edit button that now reads Save, is a
|
|
10
|
+
* new one. Each control is pressed once per screen. When every control on
|
|
11
|
+
* the current screen has been pressed, the walk returns over controls it has
|
|
12
|
+
* already pressed to the nearest screen that still has one to press. A press
|
|
13
|
+
* that leads back to a screen already seen is not progress, so Back and Next
|
|
14
|
+
* cannot loop, in any language. The walk stops at the first press that
|
|
15
|
+
* dispatches the action, when every reachable control has been pressed, at
|
|
16
|
+
* a cap on presses, or at its deadline. Neither the cap nor the deadline
|
|
17
|
+
* stops it while it is on the first screen with a control there still
|
|
18
|
+
* unpressed; once a press leaves the first screen both apply, and a
|
|
19
|
+
* first-screen control left unpressed is pressed only if a known route
|
|
20
|
+
* leads back to it (a one-way wizard has none).
|
|
21
|
+
*/
|
|
22
|
+
import type { MinimalElement } from "./host-boundary.js";
|
|
23
|
+
|
|
24
|
+
/** One control the walk can press, and how. */
|
|
25
|
+
export interface WalkControl {
|
|
26
|
+
readonly element: MinimalElement;
|
|
27
|
+
/** Whether the control names the action (its label, aria-label or `data-action`). */
|
|
28
|
+
readonly namesAction: boolean;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** A fresh copy of the card, and how to fill its screen's empty form controls. */
|
|
32
|
+
export interface WalkMount {
|
|
33
|
+
readonly container: MinimalElement;
|
|
34
|
+
/** Fill the current screen's empty form controls; filled ones are left as they are. */
|
|
35
|
+
readonly prime: () => void;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface WalkInput<C extends WalkControl> {
|
|
39
|
+
/** Mount a fresh copy of the card; undefined when it cannot be mounted. */
|
|
40
|
+
readonly mount: () => Promise<WalkMount | undefined>;
|
|
41
|
+
/** The screen's controls, in the order they are tried. */
|
|
42
|
+
readonly controls: (container: MinimalElement) => readonly C[];
|
|
43
|
+
/** Press one control; true when the press dispatched the action. */
|
|
44
|
+
readonly press: (control: C) => Promise<boolean>;
|
|
45
|
+
readonly describe: (el: MinimalElement) => string;
|
|
46
|
+
/** Epoch ms after which no further press is made. */
|
|
47
|
+
readonly stopAt: number;
|
|
48
|
+
readonly maxPresses: number;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export type WalkStop = "dispatched" | "explored" | "press-cap" | "deadline" | "no-mount";
|
|
52
|
+
|
|
53
|
+
export interface WalkOutcome {
|
|
54
|
+
readonly fired: boolean;
|
|
55
|
+
/** The control whose press dispatched the action, and the mount it is in. */
|
|
56
|
+
readonly element?: MinimalElement;
|
|
57
|
+
readonly container?: MinimalElement;
|
|
58
|
+
readonly presses: number;
|
|
59
|
+
readonly screens: number;
|
|
60
|
+
readonly stoppedBy: WalkStop;
|
|
61
|
+
/** Whether any screen rendered a control that names the action. */
|
|
62
|
+
readonly namedControlSeen: boolean;
|
|
63
|
+
/** The last few controls pressed, oldest first. */
|
|
64
|
+
readonly lastPressed: readonly string[];
|
|
65
|
+
/** The text of every screen the walk reached, once each, in the order reached. */
|
|
66
|
+
readonly screenTexts: readonly string[];
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** How many of the last presses an outcome keeps. */
|
|
70
|
+
const LAST_PRESSED_KEPT = 4;
|
|
71
|
+
|
|
72
|
+
function isDisabled(el: MinimalElement): boolean {
|
|
73
|
+
return el.getAttribute("disabled") !== null || el.getAttribute("aria-disabled") === "true";
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* A control's label for the fingerprint. Never its `name` or `id`: React
|
|
78
|
+
* generates those (`useId`) afresh when a step re-mounts, so a screen keyed
|
|
79
|
+
* on them would look new on every return and the walk would loop on it.
|
|
80
|
+
*/
|
|
81
|
+
function labelOf(el: MinimalElement): string {
|
|
82
|
+
const named = el.getAttribute("aria-label") ?? el.getAttribute("placeholder");
|
|
83
|
+
return (named ?? el.textContent ?? "").replace(/\s+/g, " ").trim().slice(0, 60);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** The screen's identity: its controls' structure and disabled state, in document order. */
|
|
87
|
+
export function screenFingerprint(container: MinimalElement): string {
|
|
88
|
+
const parts: string[] = [];
|
|
89
|
+
for (const el of container.querySelectorAll(
|
|
90
|
+
'button, [role="button"], a[href], input, select, textarea, form'
|
|
91
|
+
)) {
|
|
92
|
+
parts.push(
|
|
93
|
+
[el.tagName.toLowerCase(), el.getAttribute("type") ?? "", labelOf(el), isDisabled(el) ? "disabled" : ""].join("|")
|
|
94
|
+
);
|
|
95
|
+
}
|
|
96
|
+
return parts.join("\n");
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
interface Screen {
|
|
100
|
+
/** Each control's key, in try order. */
|
|
101
|
+
readonly order: readonly string[];
|
|
102
|
+
/** A pressed control's key → the screen its press led to. */
|
|
103
|
+
readonly led: Map<string, string>;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Each control's key on a screen: its description, numbered among controls described alike. */
|
|
107
|
+
function keyed<C extends WalkControl>(controls: readonly C[], describe: (el: MinimalElement) => string): Map<string, C> {
|
|
108
|
+
const seen = new Map<string, number>();
|
|
109
|
+
const out = new Map<string, C>();
|
|
110
|
+
for (const c of controls) {
|
|
111
|
+
const d = describe(c.element);
|
|
112
|
+
const n = seen.get(d) ?? 0;
|
|
113
|
+
seen.set(d, n + 1);
|
|
114
|
+
out.set(`${d}#${n}`, c);
|
|
115
|
+
}
|
|
116
|
+
return out;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** The first press of the shortest known route to a screen with a control not yet pressed. */
|
|
120
|
+
function firstStepToUnpressed(screens: ReadonlyMap<string, Screen>, from: string): string | undefined {
|
|
121
|
+
const firstStep = new Map<string, string | undefined>([[from, undefined]]);
|
|
122
|
+
const queue = [from];
|
|
123
|
+
while (queue.length > 0) {
|
|
124
|
+
const at = queue.shift()!;
|
|
125
|
+
const screen = screens.get(at);
|
|
126
|
+
if (screen === undefined) continue;
|
|
127
|
+
if (at !== from && screen.order.some((k) => !screen.led.has(k))) return firstStep.get(at);
|
|
128
|
+
for (const [key, to] of screen.led) {
|
|
129
|
+
if (to === at || firstStep.has(to)) continue;
|
|
130
|
+
firstStep.set(to, firstStep.get(at) ?? key);
|
|
131
|
+
queue.push(to);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return undefined;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export async function walkToAction<C extends WalkControl>(input: WalkInput<C>): Promise<WalkOutcome> {
|
|
138
|
+
const lastPressed: string[] = [];
|
|
139
|
+
const screenTexts: string[] = [];
|
|
140
|
+
const screens = new Map<string, Screen>();
|
|
141
|
+
let presses = 0;
|
|
142
|
+
let namedControlSeen = false;
|
|
143
|
+
const outcome = (stoppedBy: WalkStop, extra: Partial<WalkOutcome> = {}): WalkOutcome => ({
|
|
144
|
+
fired: false,
|
|
145
|
+
presses,
|
|
146
|
+
screens: screens.size,
|
|
147
|
+
stoppedBy,
|
|
148
|
+
namedControlSeen,
|
|
149
|
+
lastPressed: [...lastPressed],
|
|
150
|
+
screenTexts: [...screenTexts],
|
|
151
|
+
...extra,
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
const mounted = await input.mount();
|
|
155
|
+
if (mounted === undefined) return outcome("no-mount");
|
|
156
|
+
const { container } = mounted;
|
|
157
|
+
|
|
158
|
+
const arrive = (): { id: string; controls: Map<string, C> } => {
|
|
159
|
+
mounted.prime();
|
|
160
|
+
const controls = keyed(input.controls(container), input.describe);
|
|
161
|
+
if ([...controls.values()].some((c) => c.namesAction)) namedControlSeen = true;
|
|
162
|
+
const id = screenFingerprint(container);
|
|
163
|
+
if (!screens.has(id)) {
|
|
164
|
+
screens.set(id, { order: [...controls.keys()], led: new Map() });
|
|
165
|
+
screenTexts.push(container.textContent ?? "");
|
|
166
|
+
}
|
|
167
|
+
return { id, controls };
|
|
168
|
+
};
|
|
169
|
+
|
|
170
|
+
let here = arrive();
|
|
171
|
+
const first = here.id;
|
|
172
|
+
for (;;) {
|
|
173
|
+
const screen = screens.get(here.id)!;
|
|
174
|
+
const onOpenFirstScreen = here.id === first && screen.order.some((k) => !screen.led.has(k));
|
|
175
|
+
if (!onOpenFirstScreen) {
|
|
176
|
+
if (presses >= input.maxPresses) return outcome("press-cap");
|
|
177
|
+
if (Date.now() >= input.stopAt) return outcome("deadline");
|
|
178
|
+
}
|
|
179
|
+
const key = screen.order.find((k) => !screen.led.has(k)) ?? firstStepToUnpressed(screens, here.id);
|
|
180
|
+
if (key === undefined) return outcome("explored");
|
|
181
|
+
const control = here.controls.get(key);
|
|
182
|
+
if (control === undefined) {
|
|
183
|
+
// Listed when the screen was first seen, gone now: nothing to press.
|
|
184
|
+
// It now leads nowhere, which a route never takes, so this cannot
|
|
185
|
+
// pick the same control again.
|
|
186
|
+
screen.led.set(key, here.id);
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
presses += 1;
|
|
190
|
+
lastPressed.push(input.describe(control.element));
|
|
191
|
+
if (lastPressed.length > LAST_PRESSED_KEPT) lastPressed.shift();
|
|
192
|
+
if (await input.press(control)) {
|
|
193
|
+
return outcome("dispatched", { fired: true, element: control.element, container });
|
|
194
|
+
}
|
|
195
|
+
here = arrive();
|
|
196
|
+
// The first press records where a control leads; a route that later
|
|
197
|
+
// leads elsewhere is simply followed on from where it led.
|
|
198
|
+
if (!screen.led.has(key)) screen.led.set(key, here.id);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
@@ -119,6 +119,8 @@ export async function assembleGenerationResult(
|
|
|
119
119
|
...(telemetry.cacheCreationTokens !== undefined
|
|
120
120
|
? { cacheCreationTokens: telemetry.cacheCreationTokens }
|
|
121
121
|
: {}),
|
|
122
|
+
// ggui#1522 — the in-loop visual judge's spend, beside `tokens` and never inside it.
|
|
123
|
+
...(telemetry.inLoopVisualTokens !== undefined ? { inLoopVisualTokens: telemetry.inLoopVisualTokens } : {}),
|
|
122
124
|
generationTimeMs: totalMs,
|
|
123
125
|
turnsUsed: telemetry.turnsUsed,
|
|
124
126
|
passesUsed: 1,
|
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
} from "./run-coding-turn.js";
|
|
25
25
|
import { runEvalRound } from "./run-eval-round.js";
|
|
26
26
|
import type { TokenUsage } from "./run-eval-round.js";
|
|
27
|
+
import type { VisualLegTokens } from "../../evaluation/visual-evaluator.js";
|
|
27
28
|
import type { SameExchangeBreak } from "../result-types.js";
|
|
28
29
|
|
|
29
30
|
type PreWarmedEvalContext =
|
|
@@ -75,6 +76,12 @@ export interface GenerateTelemetry {
|
|
|
75
76
|
*/
|
|
76
77
|
cacheReadTokens?: number;
|
|
77
78
|
cacheCreationTokens?: number;
|
|
79
|
+
/**
|
|
80
|
+
* ggui#1522 — the in-loop visual judge's spend across eval rounds, apart from {@link totalIn} / {@link totalOut}
|
|
81
|
+
* (priced at the coding and text-evaluator models): this part is priced at the VISUAL agent's model. Stays
|
|
82
|
+
* `undefined` while no round reported one.
|
|
83
|
+
*/
|
|
84
|
+
inLoopVisualTokens?: VisualLegTokens;
|
|
78
85
|
/** Latest evalResult — undefined when eval was disabled or never ran. */
|
|
79
86
|
evalResult: EvalResult | undefined;
|
|
80
87
|
/**
|
|
@@ -167,6 +174,25 @@ export function absorbTokens(telemetry: GenerateTelemetry, usage: TokenUsage): v
|
|
|
167
174
|
}
|
|
168
175
|
}
|
|
169
176
|
|
|
177
|
+
/** ggui#1522 — fold one round's in-loop visual spend into the run's own part (never into `totalIn` / `totalOut`). */
|
|
178
|
+
export function absorbInLoopVisualTokens(telemetry: GenerateTelemetry, part: VisualLegTokens | undefined): void {
|
|
179
|
+
if (part === undefined) return;
|
|
180
|
+
const prev = telemetry.inLoopVisualTokens;
|
|
181
|
+
const prevCriteria = prev?.criteria;
|
|
182
|
+
const criteria =
|
|
183
|
+
part.criteria === undefined
|
|
184
|
+
? prevCriteria
|
|
185
|
+
: {
|
|
186
|
+
inputTokens: (prevCriteria?.inputTokens ?? 0) + part.criteria.inputTokens,
|
|
187
|
+
outputTokens: (prevCriteria?.outputTokens ?? 0) + part.criteria.outputTokens,
|
|
188
|
+
};
|
|
189
|
+
telemetry.inLoopVisualTokens = {
|
|
190
|
+
inputTokens: (prev?.inputTokens ?? 0) + part.inputTokens,
|
|
191
|
+
outputTokens: (prev?.outputTokens ?? 0) + part.outputTokens,
|
|
192
|
+
...(criteria !== undefined ? { criteria } : {}),
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
|
|
170
196
|
/**
|
|
171
197
|
* Fold one coding turn's measurements into the run's telemetry — tokens (via
|
|
172
198
|
* {@link absorbTokens}), timings, the phase/outcome counters, and the
|
|
@@ -260,6 +286,8 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
|
|
|
260
286
|
originalPrompt: params.userPrompt,
|
|
261
287
|
commitMeta: session.commitMeta,
|
|
262
288
|
originalProps: session.originalProps,
|
|
289
|
+
// ggui#1285: the fixture the probe and judge render with, for the echo check at auto-commit.
|
|
290
|
+
...(params.fixtureProps !== undefined ? { fixtureProps: params.fixtureProps } : {}),
|
|
263
291
|
costTracker: session.costTracker,
|
|
264
292
|
// Thread the dispatch-resolved policy so experimental
|
|
265
293
|
// profiles take effect on both preflight and tier-0 retry
|
|
@@ -379,6 +407,7 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
|
|
|
379
407
|
evaluationAgent: session.agents.evaluation,
|
|
380
408
|
visualEvalAgent: session.agents.visualEval,
|
|
381
409
|
visualEvaluation: params.visualEvaluation,
|
|
410
|
+
shellType: params.shellType,
|
|
382
411
|
visualThreshold: session.visualThreshold,
|
|
383
412
|
qualityMode: session.qualityMode,
|
|
384
413
|
maxEvalRounds: session.maxEvalRounds,
|
|
@@ -410,6 +439,7 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
|
|
|
410
439
|
if (round.contractFeedback) telemetry.contractFeedback = round.contractFeedback;
|
|
411
440
|
evalDone = round.evalDone;
|
|
412
441
|
absorbTokens(telemetry, round.evalTokens);
|
|
442
|
+
absorbInLoopVisualTokens(telemetry, round.inLoopVisualTokens);
|
|
413
443
|
|
|
414
444
|
if (round.control === "break") break;
|
|
415
445
|
// round.control === "feedback" — set next coding turn input.
|
|
@@ -19,6 +19,7 @@ import type { AgentSpec, SingleComponentParams } from "../runtime.js";
|
|
|
19
19
|
import type { ResolvedRunPolicy } from "../policy.js";
|
|
20
20
|
import { DEFAULT_HARNESS_POLICY } from "../policy.js";
|
|
21
21
|
import { createDupeBreakState, type DupeBreakState } from "./dupe-break.js";
|
|
22
|
+
import { inLoopEvaluation } from "../in-loop-evaluation.js";
|
|
22
23
|
|
|
23
24
|
type PreWarmedEvalContext =
|
|
24
25
|
import("../../evaluation/llm-evaluator.js").PreWarmedEvalContext;
|
|
@@ -165,10 +166,13 @@ export async function initSession(input: {
|
|
|
165
166
|
"required",
|
|
166
167
|
);
|
|
167
168
|
|
|
169
|
+
// ggui#1513 — the one rule for which evaluation legs run; a host reads the same export.
|
|
170
|
+
const { text: codeEvalEnabled, visual: visualEvalEnabled } = inLoopEvaluation(params);
|
|
171
|
+
|
|
168
172
|
// ── Pre-warm eval (overlaps with coding turn 1) ──
|
|
169
173
|
// Uses a separate agent instance so it doesn't disturb the coding cache.
|
|
170
174
|
let preWarmPromise: Promise<PreWarmedEvalContext | null> | undefined;
|
|
171
|
-
if (
|
|
175
|
+
if (codeEvalEnabled) {
|
|
172
176
|
const evalSpec = agents.evaluation;
|
|
173
177
|
preWarmPromise = import("../../evaluation/llm-evaluator.js")
|
|
174
178
|
.then((mod) =>
|
|
@@ -202,8 +206,6 @@ export async function initSession(input: {
|
|
|
202
206
|
// ── Eval gate + lazy-loaded modules ──
|
|
203
207
|
const qualityConfig = params.qualityConfig;
|
|
204
208
|
const qualityMode = qualityConfig?.quality ?? "fast";
|
|
205
|
-
const codeEvalEnabled = !!params.evaluation?.enabled;
|
|
206
|
-
const visualEvalEnabled = !!(params.visualEvaluation?.enabled || qualityConfig?.visualEval);
|
|
207
209
|
const probeOnlyEnabled =
|
|
208
210
|
harness.check.runtimeRender !== undefined && !codeEvalEnabled && !visualEvalEnabled;
|
|
209
211
|
const maxEvalRounds =
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
// Behavior MUST match the inlined original byte-for-byte — this is a
|
|
15
15
|
// mechanical extraction, not a redesign.
|
|
16
16
|
|
|
17
|
-
import type { DataContract } from "@ggui-ai/protocol";
|
|
17
|
+
import type { DataContract, JsonObject } from "@ggui-ai/protocol";
|
|
18
18
|
import type { CommitMetadata } from "../../coding-agent/types.js";
|
|
19
19
|
import type { AgentWorkspace } from "../../coding-agent/workspace.js";
|
|
20
20
|
import { executeTool } from "../../coding-agent/tools.js";
|
|
@@ -30,9 +30,10 @@ import {
|
|
|
30
30
|
APPLY_CHANGES_TOOL_SCOPED,
|
|
31
31
|
GET_COMPONENTS_INFO_TOOL,
|
|
32
32
|
GET_ICONS_TOOL,
|
|
33
|
+
ICON_LIST_NO_MATCH_GUIDANCE,
|
|
33
34
|
REWRITE_TOOL,
|
|
34
|
-
WRITE_PLAN_TOOL,
|
|
35
35
|
type LLMToolDef,
|
|
36
|
+
WRITE_PLAN_TOOL,
|
|
36
37
|
} from "../../tools.js";
|
|
37
38
|
import { LUCIDE_ICON_NAMES } from "@ggui-ai/design";
|
|
38
39
|
import { formatWithHashlines } from "../hashline.js";
|
|
@@ -165,6 +166,8 @@ export interface CodingTurnContext {
|
|
|
165
166
|
readonly originalPrompt: string;
|
|
166
167
|
readonly commitMeta: Map<string, CommitMetadata>;
|
|
167
168
|
readonly originalProps: string | undefined;
|
|
169
|
+
/** The props the probe and judge render with (ggui#1285) — handed to auto-commit's axis checks; absent = the echo check stands down. */
|
|
170
|
+
readonly fixtureProps?: JsonObject;
|
|
168
171
|
readonly costTracker: CostTracker | null;
|
|
169
172
|
/**
|
|
170
173
|
* Runtime-resolved context policy. Defaults to `harness.policy.context`
|
|
@@ -541,7 +544,11 @@ ${closingInstruction}`;
|
|
|
541
544
|
);
|
|
542
545
|
const llmMs = Date.now() - llmStart;
|
|
543
546
|
if (costTracker) {
|
|
544
|
-
|
|
547
|
+
// ggui#1524 — the cache counts reach the cap: a cached prefix is billed, so it is spend.
|
|
548
|
+
costTracker.record(codingModel, response.inputTokens, response.outputTokens, {
|
|
549
|
+
...(response.cacheReadTokens !== undefined ? { read: response.cacheReadTokens } : {}),
|
|
550
|
+
...(response.cacheCreationTokens !== undefined ? { write: response.cacheCreationTokens } : {}),
|
|
551
|
+
});
|
|
545
552
|
}
|
|
546
553
|
|
|
547
554
|
const tokens = { input: response.inputTokens, output: response.outputTokens };
|
|
@@ -614,7 +621,7 @@ ${closingInstruction}`;
|
|
|
614
621
|
iconNamesCache = LUCIDE_ICON_NAMES.join(", ");
|
|
615
622
|
}
|
|
616
623
|
console.log(`[simple] turn ${turnsUsed}: get_available_icons`);
|
|
617
|
-
let iconResult = `Available Lucide icon names (use with <Icon name="...">):\n${iconNamesCache}`;
|
|
624
|
+
let iconResult = `Available Lucide icon names (use with <Icon name="...">):\n${iconNamesCache}\n\n${ICON_LIST_NO_MATCH_GUIDANCE}`;
|
|
618
625
|
// ggui#404 same-call circuit breaker. The icon list is constant by
|
|
619
626
|
// construction, so every repeat is an identical exchange — record on
|
|
620
627
|
// the BASE text (pre-nudge) so the chain keys stay stable across the
|
|
@@ -690,6 +697,7 @@ ${closingInstruction}`;
|
|
|
690
697
|
classification: harness.classification,
|
|
691
698
|
originalPrompt: ctx.originalPrompt,
|
|
692
699
|
...(harness.canvas !== undefined ? { canvas: harness.canvas } : {}),
|
|
700
|
+
...(ctx.fixtureProps !== undefined ? { fixtureProps: ctx.fixtureProps } : {}),
|
|
693
701
|
},
|
|
694
702
|
);
|
|
695
703
|
const toolMs = Date.now() - toolStart;
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
// mechanical extraction, not a redesign.
|
|
21
21
|
|
|
22
22
|
import type { DataContract, JsonObject } from "@ggui-ai/protocol";
|
|
23
|
+
import { criteriaContextFor } from "../../evaluation/criteria/context.js";
|
|
23
24
|
import { listContractGadgets } from "@ggui-ai/protocol";
|
|
24
25
|
import type { Classification } from "../../classifier/index.js";
|
|
25
26
|
import type { AgentWorkspace } from "../../coding-agent/workspace.js";
|
|
@@ -51,6 +52,7 @@ import type {
|
|
|
51
52
|
VisualEvalOutcome,
|
|
52
53
|
VisualFitConfig,
|
|
53
54
|
VisualFitOutcome,
|
|
55
|
+
VisualLegTokens,
|
|
54
56
|
} from "../../evaluation/visual-evaluator.js";
|
|
55
57
|
|
|
56
58
|
type PreWarmedEvalContext =
|
|
@@ -74,6 +76,8 @@ export interface EvalRoundContext {
|
|
|
74
76
|
readonly evaluationAgent: AgentSpec;
|
|
75
77
|
readonly visualEvalAgent: AgentSpec;
|
|
76
78
|
readonly visualEvaluation: SingleComponentParams["visualEvaluation"];
|
|
79
|
+
/** ggui#1436 — the shell as the caller named it, for the criteria selector's context; absent reads `unknown`. */
|
|
80
|
+
readonly shellType?: SingleComponentParams["shellType"];
|
|
77
81
|
readonly visualThreshold: number;
|
|
78
82
|
readonly qualityMode: "fast" | "auto-improve" | "high-quality";
|
|
79
83
|
readonly maxEvalRounds: number;
|
|
@@ -178,6 +182,12 @@ export interface EvalRoundResult {
|
|
|
178
182
|
readonly preWarmedContext: PreWarmedEvalContext | null | undefined;
|
|
179
183
|
/** Tokens added during this round — caller adds to its own totals. */
|
|
180
184
|
readonly evalTokens: TokenUsage;
|
|
185
|
+
/**
|
|
186
|
+
* ggui#1522 — the in-loop visual judge's spend this round, apart from {@link evalTokens} (the text evaluator's,
|
|
187
|
+
* priced at its model): already recorded in the cost tracker at the visual agent's model. Absent when no visual
|
|
188
|
+
* leg ran or it reported no tokens.
|
|
189
|
+
*/
|
|
190
|
+
readonly inLoopVisualTokens?: VisualLegTokens;
|
|
181
191
|
/** Wall-clock of the parallel LLM + visual eval block (zero on the
|
|
182
192
|
* low-risk bypass path, which exits before `Promise.all`). Caller
|
|
183
193
|
* accumulates into its `cumulativeEvalLlmMs`. */
|
|
@@ -414,6 +424,7 @@ function probeMetaOf(outcome: RuntimeRenderOutcome, probeIssues: readonly EvalIs
|
|
|
414
424
|
...(outcome.renderMs !== undefined ? { renderMs: outcome.renderMs } : {}),
|
|
415
425
|
...(outcome.hostLoad !== undefined ? { hostLoad: outcome.hostLoad } : {}),
|
|
416
426
|
...(outcome.queuedMs !== undefined ? { queuedMs: outcome.queuedMs } : {}),
|
|
427
|
+
...(outcome.pendingAffordance !== undefined ? { pendingAffordance: outcome.pendingAffordance } : {}),
|
|
417
428
|
};
|
|
418
429
|
const elapsed = outcome.elapsedMs !== undefined ? { elapsedMs: outcome.elapsedMs } : {};
|
|
419
430
|
if (outcome.status !== "ran") {
|
|
@@ -588,6 +599,7 @@ export async function runEvalRound(
|
|
|
588
599
|
|
|
589
600
|
let evalResult: EvalResult | undefined;
|
|
590
601
|
let evalTokens: TokenUsage = { input: 0, output: 0 };
|
|
602
|
+
let inLoopVisualTokens: VisualLegTokens | undefined = undefined;
|
|
591
603
|
let evalLlmMs = 0;
|
|
592
604
|
|
|
593
605
|
try {
|
|
@@ -638,6 +650,7 @@ export async function runEvalRound(
|
|
|
638
650
|
prevFailFingerprints,
|
|
639
651
|
preWarmedContext,
|
|
640
652
|
evalTokens,
|
|
653
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
641
654
|
evalLlmMs,
|
|
642
655
|
lastResultText: "",
|
|
643
656
|
isEvalFeedback: false,
|
|
@@ -662,6 +675,7 @@ export async function runEvalRound(
|
|
|
662
675
|
prevFailFingerprints,
|
|
663
676
|
preWarmedContext,
|
|
664
677
|
evalTokens,
|
|
678
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
665
679
|
evalLlmMs,
|
|
666
680
|
lastResultText: lines.join("\n\n"),
|
|
667
681
|
isEvalFeedback: true,
|
|
@@ -680,6 +694,7 @@ export async function runEvalRound(
|
|
|
680
694
|
prevFailFingerprints,
|
|
681
695
|
preWarmedContext,
|
|
682
696
|
evalTokens,
|
|
697
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
683
698
|
evalLlmMs,
|
|
684
699
|
lastResultText: "",
|
|
685
700
|
isEvalFeedback: false,
|
|
@@ -769,6 +784,7 @@ export async function runEvalRound(
|
|
|
769
784
|
prevFailFingerprints,
|
|
770
785
|
preWarmedContext,
|
|
771
786
|
evalTokens,
|
|
787
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
772
788
|
evalLlmMs,
|
|
773
789
|
lastResultText: lines.join("\n\n"),
|
|
774
790
|
isEvalFeedback: true,
|
|
@@ -794,6 +810,7 @@ export async function runEvalRound(
|
|
|
794
810
|
prevFailFingerprints,
|
|
795
811
|
preWarmedContext,
|
|
796
812
|
evalTokens,
|
|
813
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
797
814
|
evalLlmMs,
|
|
798
815
|
lastResultText: "",
|
|
799
816
|
isEvalFeedback: false,
|
|
@@ -822,6 +839,22 @@ export async function runEvalRound(
|
|
|
822
839
|
profile: harness.profile,
|
|
823
840
|
// The caller's theme, when named — the round paints what the app paints.
|
|
824
841
|
...(visualEvaluation?.cssTokens !== undefined ? { cssTokens: visualEvaluation.cssTokens } : {}),
|
|
842
|
+
// ggui#1436 — the criteria bank and the card's context; the judge selects per canvas, report-only.
|
|
843
|
+
...(visualEvaluation?.criteriaBank !== undefined
|
|
844
|
+
? {
|
|
845
|
+
criteria: {
|
|
846
|
+
bank: visualEvaluation.criteriaBank,
|
|
847
|
+
context: criteriaContextFor({
|
|
848
|
+
classification,
|
|
849
|
+
contract,
|
|
850
|
+
cssTokens: visualEvaluation.cssTokens,
|
|
851
|
+
profile: harness.profile,
|
|
852
|
+
shell: ctx.shellType ?? "unknown",
|
|
853
|
+
...(visualEvaluation.criteriaKind !== undefined ? { kind: visualEvaluation.criteriaKind } : {}),
|
|
854
|
+
}),
|
|
855
|
+
},
|
|
856
|
+
}
|
|
857
|
+
: {}),
|
|
825
858
|
};
|
|
826
859
|
const visualFrame: VisualFitConfig = {
|
|
827
860
|
// The in-loop visual round renders the same sample the runtime probe
|
|
@@ -898,6 +931,18 @@ export async function runEvalRound(
|
|
|
898
931
|
evalLlmMs = Date.now() - evalLlmStart;
|
|
899
932
|
llmResult = llm;
|
|
900
933
|
visualIssues = visual?.issues ?? null;
|
|
934
|
+
// ggui#1522 — the in-loop visual judge's own spend: recorded in the tracker at the VISUAL agent's model, so
|
|
935
|
+
// `canContinue()` sees it, and carried apart from `evalTokens` (the text evaluator's model).
|
|
936
|
+
// The report-only criteria calls (ggui#1436) count toward the cap too, deliberately: the cap bounds MONEY SPENT,
|
|
937
|
+
// and a cap that left out real spend would under-count exactly as this leg did before. So under a tight cap a
|
|
938
|
+
// configured criteria bank's calls can end the eval loop earlier — the one place a report-only block reaches an
|
|
939
|
+
// outcome, and it does so through its cost, never through its answers. The tracker's total feeds no metered figure.
|
|
940
|
+
inLoopVisualTokens = visual?.tokens;
|
|
941
|
+
if (inLoopVisualTokens !== undefined) {
|
|
942
|
+
costTracker.record(visualEvalAgent.model, inLoopVisualTokens.inputTokens, inLoopVisualTokens.outputTokens);
|
|
943
|
+
const criteriaSpend = inLoopVisualTokens.criteria;
|
|
944
|
+
if (criteriaSpend !== undefined) costTracker.record(visualEvalAgent.model, criteriaSpend.inputTokens, criteriaSpend.outputTokens);
|
|
945
|
+
}
|
|
901
946
|
const visualSummary: VisualEvalSummary | undefined = visual?.summary;
|
|
902
947
|
// ggui#1221 — the judge leg's coverage is ALWAYS stamped: ran / skipped (with the
|
|
903
948
|
// reason) / not-applicable when no visual leg was configured for this round.
|
|
@@ -908,7 +953,11 @@ export async function runEvalRound(
|
|
|
908
953
|
|
|
909
954
|
// Track costs from LLM eval calls
|
|
910
955
|
if (llmResult) {
|
|
911
|
-
|
|
956
|
+
// ggui#1524 — the text evaluator's cache counts reach the cap too.
|
|
957
|
+
costTracker.record(evaluationAgent.model, llmResult.inputTokens, llmResult.outputTokens, {
|
|
958
|
+
...(llmResult.cacheReadTokens !== undefined ? { read: llmResult.cacheReadTokens } : {}),
|
|
959
|
+
...(llmResult.cacheCreationTokens !== undefined ? { write: llmResult.cacheCreationTokens } : {}),
|
|
960
|
+
});
|
|
912
961
|
evalTokens = {
|
|
913
962
|
input: llmResult.inputTokens,
|
|
914
963
|
output: llmResult.outputTokens,
|
|
@@ -977,12 +1026,22 @@ export async function runEvalRound(
|
|
|
977
1026
|
);
|
|
978
1027
|
|
|
979
1028
|
// ── Decide whether to continue the loop ──
|
|
980
|
-
// In fast mode: only fails trigger another iteration
|
|
1029
|
+
// In fast mode: only fails trigger another iteration, plus (ggui#1542) the visual JUDGE's major findings when
|
|
1030
|
+
// `visualEvaluation.actOn` is 'major'. They are taken from the visual outcome itself, never by category over the
|
|
1031
|
+
// merged list, because the text evaluator has a tier-2 `visual` criterion of its own. Only findings whose
|
|
1032
|
+
// `origin` is `judge` act (ggui#1545): the instruments' warn-level majors (fit's `canvas-overflow*` today) are
|
|
1033
|
+
// deterministic and run with or without a judge, so acting on them would switch a second feedback source on
|
|
1034
|
+
// with the judge's. Selecting by origin rather than excluding by name fails closed: a new instrument check, or
|
|
1035
|
+
// any issue with no origin, never joins the treatment unless it is marked a judge finding.
|
|
981
1036
|
// In auto-improve / high-quality: fails + warns trigger iterations.
|
|
982
1037
|
const blocking = evalResult.issues.filter((i) => i.result === "fail");
|
|
1038
|
+
const visualMajorsActed =
|
|
1039
|
+
visualEvaluation?.actOn === "major"
|
|
1040
|
+
? (visualIssues ?? []).filter((i) => i.origin === "judge" && i.result === "warn" && i.severity === "major")
|
|
1041
|
+
: [];
|
|
983
1042
|
const shouldContinue =
|
|
984
1043
|
qualityMode === "fast"
|
|
985
|
-
? blocking.length > 0
|
|
1044
|
+
? blocking.length > 0 || visualMajorsActed.length > 0
|
|
986
1045
|
: blocking.length > 0 || warns.length > 0;
|
|
987
1046
|
|
|
988
1047
|
// Compute current round's fail fingerprints — used by the stuck-loop
|
|
@@ -1027,6 +1086,7 @@ export async function runEvalRound(
|
|
|
1027
1086
|
prevFailFingerprints: enrichedFingerprints,
|
|
1028
1087
|
preWarmedContext,
|
|
1029
1088
|
evalTokens,
|
|
1089
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
1030
1090
|
evalLlmMs,
|
|
1031
1091
|
lastResultText: lines.join("\n\n"),
|
|
1032
1092
|
isEvalFeedback: true,
|
|
@@ -1060,6 +1120,7 @@ export async function runEvalRound(
|
|
|
1060
1120
|
prevFailFingerprints: new Set([...currFailFingerprints, ...contractFails.map(fingerprintFail)]),
|
|
1061
1121
|
preWarmedContext,
|
|
1062
1122
|
evalTokens,
|
|
1123
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
1063
1124
|
evalLlmMs,
|
|
1064
1125
|
lastResultText: lines.join("\n\n"),
|
|
1065
1126
|
isEvalFeedback: true,
|
|
@@ -1086,6 +1147,7 @@ export async function runEvalRound(
|
|
|
1086
1147
|
prevFailFingerprints: currFailFingerprints,
|
|
1087
1148
|
preWarmedContext,
|
|
1088
1149
|
evalTokens,
|
|
1150
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
1089
1151
|
evalLlmMs,
|
|
1090
1152
|
lastResultText: "",
|
|
1091
1153
|
isEvalFeedback: false,
|
|
@@ -1167,6 +1229,7 @@ export async function runEvalRound(
|
|
|
1167
1229
|
prevFailFingerprints: new Set([...currFailFingerprints, ...stuckContractFails.map(fingerprintFail)]),
|
|
1168
1230
|
preWarmedContext,
|
|
1169
1231
|
evalTokens,
|
|
1232
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
1170
1233
|
evalLlmMs,
|
|
1171
1234
|
lastResultText: lines.join("\n\n"),
|
|
1172
1235
|
isEvalFeedback: true,
|
|
@@ -1183,6 +1246,7 @@ export async function runEvalRound(
|
|
|
1183
1246
|
prevFailFingerprints: currFailFingerprints,
|
|
1184
1247
|
preWarmedContext,
|
|
1185
1248
|
evalTokens,
|
|
1249
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
1186
1250
|
evalLlmMs,
|
|
1187
1251
|
lastResultText: "",
|
|
1188
1252
|
isEvalFeedback: false,
|
|
@@ -1253,6 +1317,7 @@ export async function runEvalRound(
|
|
|
1253
1317
|
prevFailFingerprints: new Set([...currFailFingerprints, ...fitFails.map(fingerprintFail)]),
|
|
1254
1318
|
preWarmedContext,
|
|
1255
1319
|
evalTokens,
|
|
1320
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
1256
1321
|
evalLlmMs,
|
|
1257
1322
|
lastResultText: fitLines.join("\n\n"),
|
|
1258
1323
|
isEvalFeedback: true,
|
|
@@ -1282,6 +1347,7 @@ export async function runEvalRound(
|
|
|
1282
1347
|
prevFailFingerprints: new Set([...currFailFingerprints, ...capContractFails.map(fingerprintFail)]),
|
|
1283
1348
|
preWarmedContext,
|
|
1284
1349
|
evalTokens,
|
|
1350
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
1285
1351
|
evalLlmMs,
|
|
1286
1352
|
lastResultText: contractLines.join("\n\n"),
|
|
1287
1353
|
isEvalFeedback: true,
|
|
@@ -1308,6 +1374,7 @@ export async function runEvalRound(
|
|
|
1308
1374
|
prevFailFingerprints: currFailFingerprints,
|
|
1309
1375
|
preWarmedContext,
|
|
1310
1376
|
evalTokens,
|
|
1377
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
1311
1378
|
evalLlmMs,
|
|
1312
1379
|
lastResultText: "",
|
|
1313
1380
|
isEvalFeedback: false,
|
|
@@ -1340,6 +1407,7 @@ export async function runEvalRound(
|
|
|
1340
1407
|
]),
|
|
1341
1408
|
preWarmedContext,
|
|
1342
1409
|
evalTokens,
|
|
1410
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
1343
1411
|
evalLlmMs,
|
|
1344
1412
|
lastResultText: probeLines.join("\n\n"),
|
|
1345
1413
|
isEvalFeedback: true,
|
|
@@ -1361,6 +1429,7 @@ export async function runEvalRound(
|
|
|
1361
1429
|
prevFailFingerprints: currFailFingerprints,
|
|
1362
1430
|
preWarmedContext,
|
|
1363
1431
|
evalTokens,
|
|
1432
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
1364
1433
|
evalLlmMs,
|
|
1365
1434
|
lastResultText: "",
|
|
1366
1435
|
isEvalFeedback: false,
|
|
@@ -1393,6 +1462,14 @@ export async function runEvalRound(
|
|
|
1393
1462
|
console.log(
|
|
1394
1463
|
`[simple] eval round ${evalRoundsUsed}: ${allActionable.length} issues (${blocking.length} fail, ${warns.length} warn) → feeding back to coding loop`,
|
|
1395
1464
|
);
|
|
1465
|
+
// ggui#1542 — WHY this round continued, for a caller that set `actOn`: the manipulation check of an experiment on
|
|
1466
|
+
// the setting reads this line. `visual majors acted` counts the judge's majors only (fit's are never acted on).
|
|
1467
|
+
if (visualEvaluation?.actOn !== undefined) {
|
|
1468
|
+
console.log(
|
|
1469
|
+
`[simple] eval round ${evalRoundsUsed}: continuing: fails=${blocking.length}, visual majors acted=${visualMajorsActed.length} ` +
|
|
1470
|
+
`(actOn=${visualEvaluation.actOn})`,
|
|
1471
|
+
);
|
|
1472
|
+
}
|
|
1396
1473
|
|
|
1397
1474
|
return {
|
|
1398
1475
|
control: "feedback",
|
|
@@ -1412,6 +1489,7 @@ export async function runEvalRound(
|
|
|
1412
1489
|
),
|
|
1413
1490
|
preWarmedContext,
|
|
1414
1491
|
evalTokens,
|
|
1492
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
1415
1493
|
evalLlmMs,
|
|
1416
1494
|
lastResultText: issueLines.join("\n\n"),
|
|
1417
1495
|
isEvalFeedback: true,
|
|
@@ -1444,6 +1522,7 @@ export async function runEvalRound(
|
|
|
1444
1522
|
prevFailFingerprints,
|
|
1445
1523
|
preWarmedContext,
|
|
1446
1524
|
evalTokens,
|
|
1525
|
+
...(inLoopVisualTokens !== undefined ? { inLoopVisualTokens } : {}),
|
|
1447
1526
|
evalLlmMs,
|
|
1448
1527
|
lastResultText: "",
|
|
1449
1528
|
isEvalFeedback: false,
|