@ggui-ai/ui-gen 0.24.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{adapter-Du7P_5Fx.d.ts → adapter-BJwn74rx.d.ts} +49 -1
- package/dist/adapters/generation-dispatch.d.ts +32 -7
- package/dist/adapters/generation-dispatch.js +1997 -216
- package/dist/adapters/generation-dispatch.js.map +1 -1
- package/dist/adapters/index.js +29 -6
- package/dist/adapters/index.js.map +1 -1
- package/dist/advanced/index.d.ts +15 -12
- package/dist/advanced/index.js +2013 -270
- package/dist/advanced/index.js.map +1 -1
- package/dist/{axes-CzLEMDeB.d.ts → axes-CKVCzR_x.d.ts} +1 -1
- package/dist/bank-Ctdp3FUA.d.ts +118 -0
- package/dist/blueprint-validator.js +416 -92
- package/dist/blueprint-validator.js.map +1 -1
- package/dist/boilerplate.d.ts +2 -2
- package/dist/boilerplate.js +34 -15
- package/dist/boilerplate.js.map +1 -1
- package/dist/check/index.d.ts +13 -5
- package/dist/check/index.js +56 -3
- package/dist/check/index.js.map +1 -1
- package/dist/classifier/index.d.ts +2 -2
- package/dist/compose.d.ts +2 -2
- package/dist/{create-ui-generator-GIhzx08x.d.ts → create-ui-generator-N4gRXpcL.d.ts} +11 -4
- package/dist/{design-mode-DLufhrNp.d.ts → design-mode-DGWN_Gkm.d.ts} +1 -1
- package/dist/{dispatch-BnAH56p4.d.ts → dispatch-BUoXRj_r.d.ts} +6 -4
- package/dist/evaluation/axis-checks/index.d.ts +7 -6
- package/dist/evaluation/axis-checks/index.js +90 -2
- package/dist/evaluation/axis-checks/index.js.map +1 -1
- package/dist/evaluation/axis-checks/registry.d.ts +6 -5
- package/dist/evaluation/axis-checks/registry.js +88 -1
- package/dist/evaluation/axis-checks/registry.js.map +1 -1
- package/dist/evaluation/index.d.ts +189 -344
- package/dist/evaluation/index.js +1272 -59
- package/dist/evaluation/index.js.map +1 -1
- package/dist/evaluation/types.d.ts +26 -1
- package/dist/evaluation/types.js.map +1 -1
- package/dist/fragments/index.d.ts +3 -3
- package/dist/{generate-9IeNT4ZY.d.ts → generate-tSO_shQ0.d.ts} +1 -1
- package/dist/harness/check/runtime-render/index.d.ts +7 -7
- package/dist/harness/check/runtime-render/index.js +416 -92
- package/dist/harness/check/runtime-render/index.js.map +1 -1
- package/dist/harness/check/runtime-render/render-check-worker.js +393 -88
- package/dist/harness/check/runtime-render/render-check-worker.js.map +1 -1
- package/dist/harness/index.d.ts +5 -6
- package/dist/harness/index.js +69 -20
- package/dist/harness/index.js.map +1 -1
- package/dist/harness/prompts.d.ts +2 -2
- package/dist/harness/prompts.js +10 -1
- package/dist/harness/prompts.js.map +1 -1
- package/dist/harness/result-types.d.ts +19 -5
- package/dist/harness/result-types.js.map +1 -1
- package/dist/harness/runtime.d.ts +21 -7
- package/dist/harness/runtime.js +36 -17
- package/dist/harness/runtime.js.map +1 -1
- package/dist/harness/types-public.d.ts +6 -4
- package/dist/harness/types-public.js +43 -18
- package/dist/harness/types-public.js.map +1 -1
- package/dist/harness/validator-trace-sink.d.ts +6 -5
- package/dist/hash.d.ts +1 -1
- package/dist/index.d.ts +64 -13
- package/dist/index.js +2023 -270
- package/dist/index.js.map +1 -1
- package/dist/patch.d.ts +5 -5
- package/dist/policy.d.ts +1 -1
- package/dist/{prompts-YU9piOgs.d.ts → prompts-BhNUuRoZ.d.ts} +1 -1
- package/dist/tools/get-primitives-ts.d.ts +1 -1
- package/dist/tools/get-primitives-ts.js +1 -1
- package/dist/tools/get-primitives-ts.js.map +1 -1
- package/dist/tools/render-check-worker.js +8 -1
- package/dist/tools/render-check-worker.js.map +1 -1
- package/dist/{types-BOvHNG7K.d.ts → types-Dh1s4K9j.d.ts} +1 -1
- package/dist/types-public-B7wn4ce5.d.ts +959 -0
- package/dist/validation/index.d.ts +1 -1
- package/dist/validation/index.js +1 -1
- package/dist/validation/index.js.map +1 -1
- package/dist/visual-evaluator-C9L-o6ap.d.ts +639 -0
- package/dist/workflows.d.ts +5 -5
- package/package.json +12 -12
- package/src/adapters/generation-dispatch.ts +38 -2
- package/src/adapters/tools.ts +2 -1
- package/src/boilerplate/free-design-prompt.ts +4 -4
- package/src/boilerplate/generate.ts +26 -1
- package/src/boilerplate/hard-sections.ts +14 -14
- package/src/boilerplate/system-prompt.ts +1 -1
- package/src/boilerplate/templates/layouts/chat-mobile.tsx.tmpl +3 -1
- package/src/boilerplate/templates/layouts/chat-universal.tsx.tmpl +3 -1
- package/src/boilerplate/templates/layouts/fullscreen-desktop.tsx.tmpl +3 -1
- package/src/boilerplate/templates/layouts/fullscreen-universal.tsx.tmpl +3 -1
- package/src/check/index.ts +1 -0
- package/src/check/run-tier0.ts +6 -3
- package/src/check/sample-props.ts +63 -0
- package/src/coding-agent/tools.ts +4 -1
- package/src/contract-context.ts +1 -1
- package/src/create-harness.ts +3 -3
- package/src/create-ui-generator.ts +20 -1
- package/src/evaluation/__fixtures__/png.ts +100 -0
- package/src/evaluation/axis-checks/checks/state-merge.ts +1 -1
- package/src/evaluation/axis-checks/checks/universal.ts +111 -0
- package/src/evaluation/axis-checks/dispatch.ts +4 -1
- package/src/evaluation/cost-tracker.ts +54 -6
- package/src/evaluation/criteria/bank.ts +85 -0
- package/src/evaluation/criteria/context.ts +52 -0
- package/src/evaluation/criteria/edge-probe.ts +286 -0
- package/src/evaluation/criteria/edge.ts +202 -0
- package/src/evaluation/criteria/resolve.ts +207 -0
- package/src/evaluation/criteria/select.ts +79 -0
- package/src/evaluation/index.ts +27 -0
- package/src/evaluation/ink-extent.ts +200 -0
- package/src/evaluation/judge-spend.ts +48 -0
- package/src/evaluation/theme-css.ts +12 -5
- package/src/evaluation/types-public.ts +221 -3
- package/src/evaluation/types.ts +19 -0
- package/src/evaluation/visual-evaluator.ts +853 -51
- package/src/harness/check/runtime-render/adapter.ts +18 -0
- package/src/harness/check/runtime-render/probe-pending.ts +111 -0
- package/src/harness/check/runtime-render/render-check-host.ts +10 -6
- package/src/harness/check/runtime-render/render-check-worker.ts +6 -3
- package/src/harness/check/runtime-render/render-check.ts +417 -133
- package/src/harness/check/runtime-render/walk.ts +200 -0
- package/src/harness/coding/assemble-result.ts +2 -0
- package/src/harness/coding/generate-task-runner.ts +30 -0
- package/src/harness/coding/init-session.ts +5 -3
- package/src/harness/coding/run-coding-turn.ts +12 -4
- package/src/harness/coding/run-eval-round.ts +82 -3
- package/src/harness/in-loop-evaluation.ts +39 -0
- package/src/harness/llm-router.ts +58 -1
- package/src/harness/result-types.ts +9 -0
- package/src/harness/runtime.ts +14 -0
- package/src/harness/types-public.ts +10 -1
- package/src/harness-tool-names.ts +38 -0
- package/src/index.ts +17 -0
- package/src/internal/verdict-line.ts +33 -0
- package/src/run-check.ts +2 -0
- package/src/taught-tokens.ts +32 -0
- package/src/tools/get-primitives-ts.ts +1 -1
- package/src/tools/get-wire.ts +1 -1
- package/src/tools/render-check-worker.ts +2 -1
- package/src/tools/render-check.ts +8 -67
- package/src/tools.ts +11 -1
- package/src/validation/primitives.ts +1 -1
- package/dist/llm-router-DwwUK1bz.d.ts +0 -265
- package/dist/types-public-Bk6d1WLN.d.ts +0 -451
|
@@ -13,8 +13,16 @@
|
|
|
13
13
|
|
|
14
14
|
import { buildStylingProfileJudgeBlock } from '../boilerplate/styling-profile.js';
|
|
15
15
|
import type { GenerationProfileInput } from '../boilerplate/styling-profile.js';
|
|
16
|
+
import type { CriteriaAnswer } from './types.js';
|
|
17
|
+
import type { CriteriaBlock, CriteriaContext, CriteriaRollup } from './types-public.js';
|
|
18
|
+
import type { CriteriaBank } from './criteria/bank.js';
|
|
19
|
+
import type { JsonObject } from '@ggui-ai/protocol';
|
|
20
|
+
import type { CriteriaContextInput } from './criteria/context.js';
|
|
21
|
+
import { selectCriteria } from './criteria/select.js';
|
|
22
|
+
import { buildCriteriaJudgeBlock, resolveCriteriaBlock } from './criteria/resolve.js';
|
|
16
23
|
import { build } from 'esbuild';
|
|
17
|
-
import { fillFitRule, getCssTokens } from '@ggui-ai/design/rendering';
|
|
24
|
+
import { EXPANDED_FRAME, expandedFramePanelRule, expandedFrameScrimDecls, fillFitRule, getCssTokens } from '@ggui-ai/design/rendering';
|
|
25
|
+
import { readInkExtent, type InkExtent } from './ink-extent.js';
|
|
18
26
|
import { judgeDesignIdentity, type JudgeDesignIdentity } from './design-identity.js';
|
|
19
27
|
import { resolve, dirname } from 'path';
|
|
20
28
|
import { fileURLToPath } from 'url';
|
|
@@ -22,11 +30,21 @@ import { createRequire } from 'node:module';
|
|
|
22
30
|
import { existsSync, writeFileSync, mkdtempSync, rmSync } from 'fs';
|
|
23
31
|
import { createHash } from 'node:crypto';
|
|
24
32
|
import { tmpdir } from 'os';
|
|
25
|
-
import { createVisionAgent, type AgentConfig } from '../harness/llm-router';
|
|
33
|
+
import { createVisionAgent, type AgentConfig, type VisionFinishReason } from '../harness/llm-router';
|
|
26
34
|
import type { EvaluationResult, EvaluationIssue, DimensionScores } from './types';
|
|
27
|
-
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* An issue as the visual leg CREATES it (ggui#1545 follow-up): its `origin` is required here, so a new producer, an
|
|
38
|
+
* instrument check or a judge-side parse, that forgets to say whose finding it is fails to compile instead of
|
|
39
|
+
* quietly dropping out of `visualEvaluation.actOn`'s treatment. The public `EvaluationIssue` keeps `origin` optional
|
|
40
|
+
* for stored and external data.
|
|
41
|
+
*/
|
|
42
|
+
type VisualIssue = EvaluationIssue & { origin: 'judge' | 'instrument' };
|
|
43
|
+
import type { CanvasHostPresentation, CanvasJudgeRecord, CriteriaAnswerCounts, CanvasPresentationOutcome, CanvasVisualSummary, EvalIssue, HostInlineFrame, HostPresentationIgnoredReason, JudgeSalvageCause, JudgeSalvageCounts, VisualCoverage, VisualEvalSummary, VisualFitStamp } from './types-public.js';
|
|
28
44
|
import type { LaunchOptions } from 'puppeteer-core';
|
|
29
45
|
import { CANVAS_VIEWPORTS, displayModeForCanvas, type CanvasClass, type CanvasViewport } from '../design-mode.js';
|
|
46
|
+
import { judgeEdges, type EdgeVerdict } from './criteria/edge.js';
|
|
47
|
+
import { EDGE_BLOCKS_EXPRESSION, parseEdgeProbe } from './criteria/edge-probe.js';
|
|
30
48
|
|
|
31
49
|
// ---------------------------------------------------------------------------
|
|
32
50
|
// Types
|
|
@@ -69,6 +87,14 @@ export interface VisualEvalConfig {
|
|
|
69
87
|
* `viewport` all carry that same integer box.
|
|
70
88
|
*/
|
|
71
89
|
canvasViewports?: Partial<Readonly<Record<CanvasClass, CanvasViewport>>>;
|
|
90
|
+
/**
|
|
91
|
+
* ggui#1492 — how a host presents each canvas, as data the caller supplies (a lane that knows its host). Keyed by
|
|
92
|
+
* canvas; a canvas without an entry is judged exactly as before (the generic stand-in round the inline card, the
|
|
93
|
+
* scrim under a panel). The box stays in {@link canvasViewports}. An entry this judge cannot use (a colour that is
|
|
94
|
+
* not `#rrggbb`, a number out of range, a frame on a canvas that is not the inline card) is ignored for its canvas,
|
|
95
|
+
* said once in the log, never partly applied.
|
|
96
|
+
*/
|
|
97
|
+
hostPresentations?: Partial<Readonly<Record<CanvasClass, CanvasHostPresentation>>>;
|
|
72
98
|
/**
|
|
73
99
|
* Optional provider-routing override — see
|
|
74
100
|
* `AgentConfig.routeOverride`. Threaded onto the agent this
|
|
@@ -102,6 +128,15 @@ export interface VisualEvalConfig {
|
|
|
102
128
|
onRetry?: AgentConfig['onRetry'];
|
|
103
129
|
}
|
|
104
130
|
|
|
131
|
+
/**
|
|
132
|
+
* ggui#1436 — what the report-only criteria call spent. Kept APART from the scoring calls' `inputTokens` /
|
|
133
|
+
* `outputTokens`: those price the judgement a caller may meter, and the criteria call is a measurement beside it.
|
|
134
|
+
*/
|
|
135
|
+
export interface CriteriaCallTokens {
|
|
136
|
+
readonly inputTokens: number;
|
|
137
|
+
readonly outputTokens: number;
|
|
138
|
+
}
|
|
139
|
+
|
|
105
140
|
/** One canvas's verdict — screenshot + judge score at that class's viewport. */
|
|
106
141
|
export interface CanvasVisualResult {
|
|
107
142
|
canvas: CanvasClass;
|
|
@@ -121,35 +156,56 @@ export interface CanvasVisualResult {
|
|
|
121
156
|
/** `contentHeight > viewport.height` — a measurement; what it MEANS is {@link canvasFitPolicy}'s. */
|
|
122
157
|
overflow: boolean;
|
|
123
158
|
/** How `score` was reached — always present (ggui#1072). */
|
|
159
|
+
/**
|
|
160
|
+
* ggui#1120 — how far down the capture anything was painted (the last inked row over the measured
|
|
161
|
+
* region's height, read against the capture's dominant colour, inside a drawn panel's chrome); `0`
|
|
162
|
+
* is the blank the judge fails, `null` an unreadable capture.
|
|
163
|
+
*/
|
|
164
|
+
inkRatio: number | null;
|
|
124
165
|
judge: CanvasJudgeRecord;
|
|
125
166
|
/** How the judge composed the mount (ggui#1100): `'fill'` on every fullscreen canvas, absent on the inline card. */
|
|
126
167
|
fit?: 'fill';
|
|
127
168
|
/** ggui#1195 — `true` when the order DECLARED this canvas's box (`config.canvasViewports` carried an entry), whatever its value. */
|
|
128
169
|
declared?: true;
|
|
170
|
+
/** ggui#1436 — the typed criteria block, when a bank was configured. */
|
|
171
|
+
criteria?: CriteriaBlock;
|
|
172
|
+
/** ggui#1436 — the criteria call's own spend, when one was asked and answered; never inside the canvas's scoring tokens. */
|
|
173
|
+
criteriaTokens?: CriteriaCallTokens;
|
|
174
|
+
/** ggui#1492 — what the judge did with this canvas's host presentation (drawn, or ignored and why), when one was supplied. */
|
|
175
|
+
presentation?: CanvasPresentationOutcome;
|
|
129
176
|
}
|
|
130
177
|
|
|
131
178
|
/** How a canvas is captured for the judge and what an overflow means there (ggui#1027). */
|
|
132
|
-
export type CaptureMode = 'viewport' | 'full-page';
|
|
179
|
+
export type CaptureMode = 'viewport' | 'full-page' | 'natural';
|
|
133
180
|
export interface CanvasFitPolicy {
|
|
134
|
-
/**
|
|
181
|
+
/**
|
|
182
|
+
* `viewport` = the whole box; `full-page` = the whole scrolled document; `natural` (ggui#1475) = the
|
|
183
|
+
* card at its OWN height, clipped to its extent and capped at the box — the inline card a host sizes
|
|
184
|
+
* to its content, on the host's ground ({@link JUDGE_GROUND_MARGIN_PX}).
|
|
185
|
+
*/
|
|
135
186
|
readonly capture: CaptureMode;
|
|
136
187
|
/** The verdict an overflow earns on this canvas. */
|
|
137
188
|
readonly overflow: 'fail' | 'warn' | 'none';
|
|
138
189
|
}
|
|
139
190
|
|
|
140
191
|
/**
|
|
141
|
-
* The inline chat card
|
|
142
|
-
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
*
|
|
192
|
+
* The inline chat card takes its NATURAL height under a ceiling (ggui#1475):
|
|
193
|
+
* an MCP Apps host sizes the inline frame to the card's content
|
|
194
|
+
* (`ui/notifications/size-changed`, auto-resize) up to its maximum, so the
|
|
195
|
+
* visitor sees the card and nothing under it. The judge captures exactly
|
|
196
|
+
* that — the card's own extent on the host's ground, capped at the box — and
|
|
197
|
+
* never the box's remainder (a fixed 640 px capture scored every hello
|
|
198
|
+
* against a void no visitor sees). The box stays the ceiling: content taller
|
|
199
|
+
* than it is a critical layout issue that fails the canvas (a 1289 px hello
|
|
200
|
+
* on a 640 px card was scored 86 from a full-page capture that showed the
|
|
201
|
+
* judge what no user sees). A phone's first screen may scroll, so the
|
|
202
|
+
* overflow is reported as a major issue and the score stands. Pages
|
|
203
|
+
* (`md`/`lg`/`xl`) scroll by design: measured, never judged.
|
|
148
204
|
*/
|
|
149
205
|
export function canvasFitPolicy(canvas: CanvasClass): CanvasFitPolicy {
|
|
150
206
|
switch (canvas) {
|
|
151
207
|
case 'xs-chat-card':
|
|
152
|
-
return { capture: '
|
|
208
|
+
return { capture: 'natural', overflow: 'fail' };
|
|
153
209
|
case 'mobile-fullscreen-small':
|
|
154
210
|
return { capture: 'full-page', overflow: 'warn' };
|
|
155
211
|
default:
|
|
@@ -162,6 +218,15 @@ export function canvasFitPolicy(canvas: CanvasClass): CanvasFitPolicy {
|
|
|
162
218
|
* `fillFitRule` targets — the judge's counterpart of the runtime's mount list.
|
|
163
219
|
*/
|
|
164
220
|
export const JUDGE_SCOPE_CLASS = 'ggui-judge-scope';
|
|
221
|
+
/** The panel the judge draws round a fill canvas's scope on its stand-in host page (ggui#1083 cut 3) — see {@link canvasChrome}. */
|
|
222
|
+
export const JUDGE_PANEL_CLASS = 'ggui-judge-panel';
|
|
223
|
+
/**
|
|
224
|
+
* How far in from a panelled page's edges the ink is read (ggui#1120): past the panel's gap, its
|
|
225
|
+
* largest rounded corner (the `round` bucket, 24 px — the scrim shows through the corner) and its
|
|
226
|
+
* 1 px hairline, so the host's chrome never counts as paint. The card's own inset is 16 px, so the
|
|
227
|
+
* band skipped inside the card is 25 px of a canvas — reported ratios are over the region.
|
|
228
|
+
*/
|
|
229
|
+
export const JUDGE_INK_INSET_PX = EXPANDED_FRAME.insetPx + 24 + 1;
|
|
165
230
|
|
|
166
231
|
/**
|
|
167
232
|
* How the judge composes a canvas (ggui#1100): the served runtime stretches
|
|
@@ -173,6 +238,115 @@ export const JUDGE_SCOPE_CLASS = 'ggui-judge-scope';
|
|
|
173
238
|
export function canvasFit(canvas: CanvasClass): 'fill' | undefined {
|
|
174
239
|
return displayModeForCanvas(canvas) === 'fullscreen' ? 'fill' : undefined;
|
|
175
240
|
}
|
|
241
|
+
/**
|
|
242
|
+
* What the judge's stand-in host draws round a canvas: the embedding shell's floating panel on its
|
|
243
|
+
* scrim (a fill canvas, md and up), the host's ground round the inline card (`ground`, ggui#1475), or
|
|
244
|
+
* nothing (the phone, edge to edge).
|
|
245
|
+
*/
|
|
246
|
+
export type CanvasChrome = 'panel' | 'ground' | undefined;
|
|
247
|
+
/**
|
|
248
|
+
* The host ground round a naturally-captured inline card (ggui#1475): the card is mounted this far in
|
|
249
|
+
* from the page's edges and captured with this margin, so its edge and radius are read against the
|
|
250
|
+
* host surface — the page's scrim ground, never the card's own. The design's one gap (the panel's).
|
|
251
|
+
*/
|
|
252
|
+
export const JUDGE_GROUND_MARGIN_PX = EXPANDED_FRAME.insetPx;
|
|
253
|
+
/** The class the judge page wraps a naturally-captured card's mount in — its padding is the ground margin. */
|
|
254
|
+
export const JUDGE_GROUND_CLASS = 'ggui-judge-ground';
|
|
255
|
+
/**
|
|
256
|
+
* The host's frame round the inline card (ggui#1475). An inline card draws no chrome of its own (the
|
|
257
|
+
* chat layout tells it the host draws it), so without a frame it sits transparent on the page ground
|
|
258
|
+
* and has no edge to read. The judge draws a GENERIC stand-in for the host's frame, built from the
|
|
259
|
+
* card's own theme: its container surface, a 1 px ring of its ink at 8 %, a 16 px radius, and
|
|
260
|
+
* overflow hidden. It is a stand-in, and the frame line says so: a host's exact frame arrives as data.
|
|
261
|
+
*/
|
|
262
|
+
export const JUDGE_INLINE_FRAME_CLASS = 'ggui-judge-inline-frame';
|
|
263
|
+
export const JUDGE_INLINE_FRAME_BORDER_PX = 1;
|
|
264
|
+
export const JUDGE_INLINE_FRAME_RADIUS_PX = 16;
|
|
265
|
+
/** What the inline card's page adds on each side of the card: the ground margin plus the frame's border. */
|
|
266
|
+
export const JUDGE_INLINE_PAD_PX = JUDGE_GROUND_MARGIN_PX + JUDGE_INLINE_FRAME_BORDER_PX;
|
|
267
|
+
/** How far in the inline card's ink is read: past the margin, the frame's ring and its rounded corner, so the host's frame never counts as paint. */
|
|
268
|
+
export const JUDGE_INLINE_INK_INSET_PX = JUDGE_INLINE_PAD_PX + JUDGE_INLINE_FRAME_RADIUS_PX;
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* The bounds a host frame must meet to be drawn (ggui#1492), exported so a lane's reader applies exactly the frames
|
|
272
|
+
* this judge draws: an entry the reader accepts and the judge ignores would be echoed as applied while never drawn.
|
|
273
|
+
*/
|
|
274
|
+
export const HOST_COLOUR_PATTERN = /^#[0-9a-f]{6}$/;
|
|
275
|
+
export const HOST_FRAME_BOUNDS = {
|
|
276
|
+
ringWidthPx: { min: 0, max: 8 },
|
|
277
|
+
ringAlpha: { min: 0, max: 1 },
|
|
278
|
+
radiusPx: { min: 0, max: 64 },
|
|
279
|
+
minHeightPx: { min: 0, max: 2560 },
|
|
280
|
+
} as const;
|
|
281
|
+
/**
|
|
282
|
+
* ggui#1492 — what this judge will do with a canvas's host presentation: draw it, or ignore it with the lane
|
|
283
|
+
* reader's own reason code. `undefined` when none was supplied. Every string drawn reaches the page's CSS, so a
|
|
284
|
+
* colour must be `#rrggbb` and a number must sit inside {@link HOST_FRAME_BOUNDS}; an inline card's room lives on
|
|
285
|
+
* its frame, so a top-level `ground` there is refused rather than silently unused.
|
|
286
|
+
*/
|
|
287
|
+
function resolvePresentation(canvas: CanvasClass, p: CanvasHostPresentation | undefined): CanvasPresentationOutcome | undefined {
|
|
288
|
+
if (p === undefined) return undefined;
|
|
289
|
+
const ignored = (reason: HostPresentationIgnoredReason, why: string): CanvasPresentationOutcome => {
|
|
290
|
+
console.warn(`[visual-eval] host presentation for ${canvas} ignored (${reason}): ${why}`);
|
|
291
|
+
return { status: 'ignored', reason };
|
|
292
|
+
};
|
|
293
|
+
const inline = canvasFitPolicy(canvas).capture === 'natural';
|
|
294
|
+
if (typeof p.label !== 'string' || p.label.length === 0) return ignored('label_missing', 'no label');
|
|
295
|
+
if (p.ground !== undefined && !HOST_COLOUR_PATTERN.test(p.ground)) return ignored('color_not_hex', 'ground is not #rrggbb');
|
|
296
|
+
if (inline && p.ground !== undefined) return ignored('ground_on_inline', "the inline card's room is its frame's ground");
|
|
297
|
+
const f = p.frame;
|
|
298
|
+
if (f !== undefined) {
|
|
299
|
+
if (!inline) return ignored('frame_off_inline', 'a frame on a canvas that is not the inline card');
|
|
300
|
+
if (![f.surface, f.ground, f.ring.color].every((c) => HOST_COLOUR_PATTERN.test(c))) return ignored('color_not_hex', 'a frame colour is not #rrggbb');
|
|
301
|
+
const B = HOST_FRAME_BOUNDS;
|
|
302
|
+
const inRange = (n: number, b: { readonly min: number; readonly max: number }): boolean => Number.isFinite(n) && n >= b.min && n <= b.max;
|
|
303
|
+
if (!inRange(f.ring.widthPx, B.ringWidthPx) || !inRange(f.ring.alpha, B.ringAlpha) || !inRange(f.radiusPx, B.radiusPx)) return ignored('frame_malformed', 'a frame number out of range');
|
|
304
|
+
if (f.minHeightPx !== undefined && !inRange(f.minHeightPx, B.minHeightPx)) return ignored('frame_malformed', 'minHeightPx out of range');
|
|
305
|
+
}
|
|
306
|
+
return { status: 'applied', applied: p };
|
|
307
|
+
}
|
|
308
|
+
/** What the inline card's page adds per side, where its ink is read from, and the host's floor — from the host frame when given, else the stand-in's. */
|
|
309
|
+
interface InlineGeometry {
|
|
310
|
+
readonly padPx: number;
|
|
311
|
+
readonly inkInsetPx: number;
|
|
312
|
+
readonly floorPx: number | undefined;
|
|
313
|
+
}
|
|
314
|
+
function inlineGeometry(frame: HostInlineFrame | undefined): InlineGeometry {
|
|
315
|
+
if (frame === undefined) return { padPx: JUDGE_INLINE_PAD_PX, inkInsetPx: JUDGE_INLINE_INK_INSET_PX, floorPx: undefined };
|
|
316
|
+
const padPx = JUDGE_GROUND_MARGIN_PX + frame.ring.widthPx;
|
|
317
|
+
return { padPx, inkInsetPx: padPx + frame.radiusPx, floorPx: frame.minHeightPx };
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* The host stand-in round a fill canvas (ggui#1083 cut 3, ggui#1067 §5): the embedding shell floats
|
|
321
|
+
* the card in a panel on its scrim — a hairline, the theme's `xl` radius, `shadow-sm`, a 16 px gap —
|
|
322
|
+
* only when expanded at ≥ 601 px; below that the card owns the phone edge to edge. So the panel
|
|
323
|
+
* canvases are the fill canvases from `md` up (the shell's panel reaches `lg` / `xl` on wide windows)
|
|
324
|
+
* and `mobile-fullscreen-small` carries no chrome. The panel is the design's ONE definition
|
|
325
|
+
* (`expandedFramePanelRule`), never a second drawing, and the judge pads its window by the panel's
|
|
326
|
+
* gap so the CARD's box stays the canvas box ({@link judgeWindow}): the frame the founder approves
|
|
327
|
+
* carries the chrome the visitor sees, and the card is measured where it was before.
|
|
328
|
+
*/
|
|
329
|
+
export function canvasChrome(canvas: CanvasClass): CanvasChrome {
|
|
330
|
+
if (canvasFitPolicy(canvas).capture === 'natural') return 'ground';
|
|
331
|
+
return canvasFit(canvas) === 'fill' && canvas !== 'mobile-fullscreen-small' ? 'panel' : undefined;
|
|
332
|
+
}
|
|
333
|
+
/** The browser window for a canvas: the canvas box, plus the chrome's gap on every side when the judge draws one (the panel's, or the host ground's). */
|
|
334
|
+
export function judgeWindow(viewport: CanvasViewport, chrome: CanvasChrome, inlinePadPx: number = JUDGE_INLINE_PAD_PX): CanvasViewport {
|
|
335
|
+
if (chrome === undefined) return viewport;
|
|
336
|
+
const gap = 2 * (chrome === 'ground' ? inlinePadPx : EXPANDED_FRAME.insetPx);
|
|
337
|
+
return { width: viewport.width + gap, height: viewport.height + gap };
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* The card's height on a panelled page: the panel grows with the card and carries the gap above and
|
|
341
|
+
* below it. A document shorter than the gap laid out no panel (the panel alone is a viewport tall), so
|
|
342
|
+
* there is no reading — `null`, never a negative height.
|
|
343
|
+
*/
|
|
344
|
+
function cardHeight(documentHeight: number | null, chrome: CanvasChrome): number | null {
|
|
345
|
+
// A natural capture measures the card itself (the mount's extent), never the document.
|
|
346
|
+
if (documentHeight === null || chrome === undefined || chrome === 'ground') return documentHeight;
|
|
347
|
+
const gap = 2 * EXPANDED_FRAME.insetPx;
|
|
348
|
+
return documentHeight < gap ? null : documentHeight - gap;
|
|
349
|
+
}
|
|
176
350
|
|
|
177
351
|
/**
|
|
178
352
|
* The box a browser can be asked for: whole CSS px. CDP's
|
|
@@ -197,7 +371,7 @@ export function canvasOverflowIssue(
|
|
|
197
371
|
viewport: CanvasViewport,
|
|
198
372
|
contentHeight: number,
|
|
199
373
|
verdict: 'fail' | 'warn',
|
|
200
|
-
):
|
|
374
|
+
): VisualIssue {
|
|
201
375
|
const hidden = contentHeight - viewport.height;
|
|
202
376
|
// ggui#1195 — when the judge captured at a DECLARED box, the refusal names
|
|
203
377
|
// both boxes: the ceiling the content was measured against and the class
|
|
@@ -208,6 +382,7 @@ export function canvasOverflowIssue(
|
|
|
208
382
|
: `${viewport.width}×${viewport.height}`;
|
|
209
383
|
return {
|
|
210
384
|
dimension: 'canvas-overflow',
|
|
385
|
+
origin: 'instrument',
|
|
211
386
|
severity: verdict === 'fail' ? 'critical' : 'major',
|
|
212
387
|
description:
|
|
213
388
|
`Rendered content is ${contentHeight}px tall on the ${canvas} canvas (${box}) — ` +
|
|
@@ -218,6 +393,43 @@ export function canvasOverflowIssue(
|
|
|
218
393
|
};
|
|
219
394
|
}
|
|
220
395
|
|
|
396
|
+
/** ggui#1475 — the horizontal twin of {@link canvasOverflowIssue}: content wider than the card, which its frame clips out of the capture. */
|
|
397
|
+
export function canvasOverflowXIssue(
|
|
398
|
+
canvas: CanvasClass,
|
|
399
|
+
viewport: CanvasViewport,
|
|
400
|
+
hiddenPx: number,
|
|
401
|
+
verdict: 'fail' | 'warn',
|
|
402
|
+
): VisualIssue {
|
|
403
|
+
return {
|
|
404
|
+
dimension: 'canvas-overflow-x',
|
|
405
|
+
origin: 'instrument',
|
|
406
|
+
severity: verdict === 'fail' ? 'critical' : 'major',
|
|
407
|
+
description:
|
|
408
|
+
`Content runs ${hiddenPx}px past the ${canvas} card's ${viewport.width}px width — the card's frame clips it, ` +
|
|
409
|
+
'so a visitor sees a cut-off row or a sideways scroll.',
|
|
410
|
+
fix:
|
|
411
|
+
'Keep every row within the card: wrap long text (overflow-wrap: anywhere), let a wide table or code block scroll ' +
|
|
412
|
+
'inside its own container, and give no child a fixed width wider than the card.',
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/** ggui#1120 — the deterministic blank: the capture is one flat colour, so the component mounted and painted nothing. */
|
|
417
|
+
export function canvasBlankIssue(canvas: CanvasClass, viewport: CanvasViewport): VisualIssue {
|
|
418
|
+
const classBox = CANVAS_VIEWPORTS[canvas];
|
|
419
|
+
const box = isDeclaredViewport(canvas, viewport)
|
|
420
|
+
? `declared ${viewport.width}×${viewport.height}; class box ${classBox.width}×${classBox.height}`
|
|
421
|
+
: `${viewport.width}×${viewport.height}`;
|
|
422
|
+
return {
|
|
423
|
+
dimension: 'canvas-blank',
|
|
424
|
+
origin: 'instrument',
|
|
425
|
+
severity: 'critical',
|
|
426
|
+
description: `Nothing painted on the ${canvas} canvas (${box}): the capture is one flat colour — the component mounted and rendered no pixels.`,
|
|
427
|
+
fix:
|
|
428
|
+
'Return and paint the component tree: a root that renders nothing is a blank, not an empty state. Check that the ' +
|
|
429
|
+
'default export returns its JSX, that no early return yields null on the sample props, and that the first paint ' +
|
|
430
|
+
'shows visible content.',
|
|
431
|
+
};
|
|
432
|
+
}
|
|
221
433
|
/**
|
|
222
434
|
* `runVisualEvaluation`'s result. Identical to `EvaluationResult` when
|
|
223
435
|
* `config.canvases` is unset; with canvases set, `canvases` holds the
|
|
@@ -226,6 +438,16 @@ export function canvasOverflowIssue(
|
|
|
226
438
|
*/
|
|
227
439
|
export interface VisualEvaluationResult extends EvaluationResult {
|
|
228
440
|
canvases?: CanvasVisualResult[];
|
|
441
|
+
/**
|
|
442
|
+
* ggui#1436 — the criteria calls' spend across canvases, when any was asked and answered. `inputTokens` /
|
|
443
|
+
* `outputTokens` stay the SCORING calls' alone, so a cost read off them never includes the report-only block.
|
|
444
|
+
*/
|
|
445
|
+
criteriaTokens?: CriteriaCallTokens;
|
|
446
|
+
/**
|
|
447
|
+
* ggui#1127 — salvaged judge answers across the canvases, per leg, as N of M (scoring answers; criteria calls
|
|
448
|
+
* answered). Present on every canvas run, zeros included, so a reader can tell "none salvaged" from "not counted".
|
|
449
|
+
*/
|
|
450
|
+
judgeSalvage?: JudgeSalvageCounts;
|
|
229
451
|
/** The design tree the judge painted with (ggui#1042) — present on every per-canvas judgement. */
|
|
230
452
|
design?: JudgeDesignIdentity;
|
|
231
453
|
/** The mode `cssTokens` were composed in, when the caller said (ggui#1076). */
|
|
@@ -247,6 +469,8 @@ export interface VisualEvalContext {
|
|
|
247
469
|
cssTokens?: string;
|
|
248
470
|
/** The app's generation profile (#991) — judged relative to, never against. */
|
|
249
471
|
profile?: GenerationProfileInput;
|
|
472
|
+
/** ggui#1436 — a criteria bank and the card's context; the judge selects per canvas and answers report-only. */
|
|
473
|
+
criteria?: { readonly bank: CriteriaBank; readonly context: CriteriaContextInput };
|
|
250
474
|
}
|
|
251
475
|
|
|
252
476
|
// ---------------------------------------------------------------------------
|
|
@@ -447,11 +671,24 @@ try {
|
|
|
447
671
|
}
|
|
448
672
|
}
|
|
449
673
|
|
|
674
|
+
/** The inline card's frame: the host's, as data, when given; else the generic stand-in built from the card's own tokens. */
|
|
675
|
+
function inlineFrameDecls(frame: HostInlineFrame | undefined): string {
|
|
676
|
+
if (frame === undefined) {
|
|
677
|
+
return `background: var(--ggui-color-container); border: ${JUDGE_INLINE_FRAME_BORDER_PX}px solid color-mix(in srgb, var(--ggui-color-onContainer) 8%, transparent); border-radius: ${JUDGE_INLINE_FRAME_RADIUS_PX}px; overflow: hidden;`;
|
|
678
|
+
}
|
|
679
|
+
const alphaPct = Math.round(frame.ring.alpha * 1000) / 10;
|
|
680
|
+
return `background: ${frame.surface}; border: ${frame.ring.widthPx}px solid color-mix(in srgb, ${frame.ring.color} ${alphaPct}%, transparent); border-radius: ${frame.radiusPx}px; overflow: hidden;`;
|
|
681
|
+
}
|
|
682
|
+
|
|
450
683
|
function buildRenderHTML(
|
|
451
684
|
bundledCode: string,
|
|
452
685
|
cssTokens?: string,
|
|
453
686
|
fit?: 'fill',
|
|
687
|
+
chrome?: CanvasChrome,
|
|
688
|
+
presentation?: CanvasHostPresentation,
|
|
454
689
|
): string {
|
|
690
|
+
if (chrome === 'panel' && fit !== 'fill') throw new Error("the judge's panel frames a fill canvas only (ggui#1083 cut 3)");
|
|
691
|
+
if (chrome === 'ground' && fit === 'fill') throw new Error("the host ground frames an inline card only — a fill canvas has no natural height (ggui#1475)");
|
|
455
692
|
// Default to the design tokens production's no-theme branch injects
|
|
456
693
|
// (getCssTokens → default theme, light) — ggui#613: under the s4
|
|
457
694
|
// fallback ban the generated component's `var(--ggui-*)` references
|
|
@@ -473,15 +710,16 @@ function buildRenderHTML(
|
|
|
473
710
|
/* ggui#1083 — the host stand-in (neutral-50) under the theme's scrim (its tint at
|
|
474
711
|
its opacity): the ground a frosted host puts under the card, so the judge scores
|
|
475
712
|
the card where the visitor sees it. */
|
|
476
|
-
background:
|
|
713
|
+
${presentation?.frame !== undefined ? `background: ${presentation.frame.ground};` : presentation?.ground !== undefined ? `background: ${presentation.ground};` : expandedFrameScrimDecls()}
|
|
477
714
|
color: var(--ggui-color-neutral-900, #111827);
|
|
478
715
|
}
|
|
479
716
|
.error { color: #dc2626; padding: 16px; font-family: monospace; white-space: pre-wrap; }
|
|
480
|
-
${fit === 'fill' ? fillFitRule(JUDGE_SCOPE_CLASS) : ''}
|
|
717
|
+
${fit === 'fill' ? fillFitRule(JUDGE_SCOPE_CLASS) + (chrome === 'panel' ? expandedFramePanelRule(JUDGE_PANEL_CLASS, JUDGE_SCOPE_CLASS) : '') : ''}
|
|
718
|
+
${chrome === 'ground' ? `.${JUDGE_GROUND_CLASS} { padding: ${JUDGE_GROUND_MARGIN_PX}px; } .${JUDGE_INLINE_FRAME_CLASS} { ${inlineFrameDecls(presentation?.frame)} }` : ''}
|
|
481
719
|
</style>
|
|
482
720
|
</head>
|
|
483
721
|
<body>
|
|
484
|
-
<div id="root"${fit === 'fill' ? ` class="${JUDGE_SCOPE_CLASS}"` : ''}></div>
|
|
722
|
+
${chrome === 'panel' ? `<div class="${JUDGE_PANEL_CLASS}">` : chrome === 'ground' ? `<div class="${JUDGE_GROUND_CLASS}"><div class="${JUDGE_INLINE_FRAME_CLASS}">` : ''}<div id="root"${fit === 'fill' ? ` class="${JUDGE_SCOPE_CLASS}"` : ''}></div>${chrome === 'panel' ? '</div>' : chrome === 'ground' ? '</div></div>' : ''}
|
|
485
723
|
<script type="importmap">
|
|
486
724
|
{
|
|
487
725
|
"imports": {
|
|
@@ -503,6 +741,14 @@ ${bundledCode}
|
|
|
503
741
|
// Screenshot capture (optional puppeteer dependency)
|
|
504
742
|
// ---------------------------------------------------------------------------
|
|
505
743
|
|
|
744
|
+
/** A screenshot's region in page px (puppeteer's `clip`). */
|
|
745
|
+
export interface ScreenshotClip {
|
|
746
|
+
readonly x: number;
|
|
747
|
+
readonly y: number;
|
|
748
|
+
readonly width: number;
|
|
749
|
+
readonly height: number;
|
|
750
|
+
}
|
|
751
|
+
|
|
506
752
|
/** The page surface the capture uses — a structural subset of puppeteer's `Page`. */
|
|
507
753
|
export interface ScreenshotPage {
|
|
508
754
|
setContent(html: string, options: { waitUntil: 'load'; timeout: number }): Promise<void>;
|
|
@@ -510,7 +756,7 @@ export interface ScreenshotPage {
|
|
|
510
756
|
waitForSelector(selector: string, options: { timeout: number }): Promise<unknown>;
|
|
511
757
|
/** Evaluates a JS expression in the page and returns its serialised value — the fit measurement (ggui#1027). */
|
|
512
758
|
evaluate(expression: string): Promise<unknown>;
|
|
513
|
-
screenshot(options: { type: 'png'; fullPage: boolean }): Promise<Uint8Array>;
|
|
759
|
+
screenshot(options: { type: 'png'; fullPage: boolean; clip?: ScreenshotClip }): Promise<Uint8Array>;
|
|
514
760
|
}
|
|
515
761
|
|
|
516
762
|
/** The browser surface the capture uses — a structural subset of puppeteer's `Browser`. */
|
|
@@ -603,16 +849,63 @@ export interface ScreenshotAttempt {
|
|
|
603
849
|
readonly reason?: string;
|
|
604
850
|
/** The document's scroll height at the viewport (ggui#1027); `null` when unmeasurable. */
|
|
605
851
|
readonly contentHeight: number | null;
|
|
852
|
+
/** ggui#1475 — a natural capture's horizontal overflow: how many px the card's content runs past its width; absent on every other capture. */
|
|
853
|
+
readonly overflowX?: number | null;
|
|
854
|
+
/**
|
|
855
|
+
* ggui#1663 — the edge rule on this capture's DOM, read just before the screenshot (report-only); `null` when the
|
|
856
|
+
* probe failed. Absent on a failed capture.
|
|
857
|
+
*/
|
|
858
|
+
readonly edge?: EdgeVerdict | null;
|
|
606
859
|
}
|
|
607
860
|
|
|
608
861
|
/** The expression the fit measurement evaluates in the page — the taller of the two scroll heights. */
|
|
609
862
|
export const CONTENT_HEIGHT_EXPRESSION =
|
|
610
863
|
'Math.max(document.documentElement.scrollHeight, document.body ? document.body.scrollHeight : 0)';
|
|
611
864
|
|
|
612
|
-
/**
|
|
613
|
-
|
|
865
|
+
/**
|
|
866
|
+
* ggui#1475 — the natural capture's measurement: the card's height as the host is told it. The MCP
|
|
867
|
+
* Apps SDK's size-changed notification (the report an auto-resizing host sizes the inline frame by)
|
|
868
|
+
* reads the document at `height: max-content` and restores the style; the judge reads the same
|
|
869
|
+
* quantity, minus what its page adds on both sides (the ground margin and the host frame's border), so content outside the mount (a
|
|
870
|
+
* portal into body) counts here exactly as it counts for the host. The document's scroll height
|
|
871
|
+
* could not be used: it is floored at the viewport by definition, so it sees an overflow but never
|
|
872
|
+
* a card shorter than its box.
|
|
873
|
+
*/
|
|
874
|
+
export function cardHeightExpression(padPx: number): string {
|
|
875
|
+
return (
|
|
876
|
+
"(() => { const el = document.documentElement; const prev = el.style.height; el.style.height = 'max-content'; " +
|
|
877
|
+
`const h = Math.ceil(el.getBoundingClientRect().height); el.style.height = prev; return h - ${2 * padPx}; })()`
|
|
878
|
+
);
|
|
879
|
+
}
|
|
880
|
+
/** The stand-in page's card height ({@link cardHeightExpression} at the stand-in's pad). */
|
|
881
|
+
export const CARD_HEIGHT_EXPRESSION = cardHeightExpression(JUDGE_INLINE_PAD_PX);
|
|
882
|
+
|
|
883
|
+
/**
|
|
884
|
+
* ggui#1475 — the inline card's horizontal overflow: how far its content runs past the card's width. The host's
|
|
885
|
+
* frame (and the judge's stand-in) clips at the card's edge, so the capture cannot show it, while a visitor's frame
|
|
886
|
+
* scrolls it sideways: only a measurement carries it.
|
|
887
|
+
*/
|
|
888
|
+
/** The sideways reading layout rounding alone can produce (ggui#1475): `scrollWidth` and `clientWidth` are integers. */
|
|
889
|
+
export const OVERFLOW_X_ROUNDING_PX = 1;
|
|
890
|
+
export const CARD_OVERFLOW_X_EXPRESSION =
|
|
891
|
+
"(() => { const r = document.getElementById('root'); return r ? Math.max(0, Math.ceil(r.scrollWidth - r.clientWidth)) : null; })()";
|
|
892
|
+
|
|
893
|
+
/**
|
|
894
|
+
* The natural capture's region: the card on the host ground, the ground margin on every side, the
|
|
895
|
+
* card's height capped at the box (content past it is the overflow the fit verdict reads). At least
|
|
896
|
+
* one row of card is kept, so a card that painted nothing is still read — as a blank, never as an
|
|
897
|
+
* unreadable capture.
|
|
898
|
+
*/
|
|
899
|
+
export function naturalClip(window: { width: number; height: number }, cardHeightPx: number, padPx: number = JUDGE_INLINE_PAD_PX): ScreenshotClip {
|
|
900
|
+
const m = padPx;
|
|
901
|
+
const box = window.height - 2 * m;
|
|
902
|
+
return { x: 0, y: 0, width: window.width, height: Math.max(1, Math.min(Math.ceil(cardHeightPx), box)) + 2 * m };
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
/** Measure the rendered document's height (or the card's, for a natural capture); a failed measurement is reported, never a failed capture. */
|
|
906
|
+
async function measureContentHeight(page: ScreenshotPage, expression: string = CONTENT_HEIGHT_EXPRESSION): Promise<number | null> {
|
|
614
907
|
try {
|
|
615
|
-
const h = await page.evaluate(
|
|
908
|
+
const h = await page.evaluate(expression);
|
|
616
909
|
return typeof h === 'number' && Number.isFinite(h) ? Math.round(h) : null;
|
|
617
910
|
} catch (e) {
|
|
618
911
|
console.warn(`[visual-eval] content height unavailable: ${e instanceof Error ? e.message : String(e)}`);
|
|
@@ -620,11 +913,36 @@ async function measureContentHeight(page: ScreenshotPage): Promise<number | null
|
|
|
620
913
|
}
|
|
621
914
|
}
|
|
622
915
|
|
|
916
|
+
/**
|
|
917
|
+
* ggui#1663 — read the edge rule off the page the screenshot is about to take. The probe only reads layout (it never
|
|
918
|
+
* writes to the DOM), so the frame is unchanged; a failed read is reported and returns `null`, never a failed capture.
|
|
919
|
+
*/
|
|
920
|
+
async function measureEdges(page: ScreenshotPage): Promise<EdgeVerdict | null> {
|
|
921
|
+
try {
|
|
922
|
+
const probe = parseEdgeProbe(await page.evaluate(EDGE_BLOCKS_EXPRESSION));
|
|
923
|
+
if (probe === null) {
|
|
924
|
+
console.warn('[visual-eval] edge probe unavailable: the page returned no readable blocks');
|
|
925
|
+
return null;
|
|
926
|
+
}
|
|
927
|
+
return judgeEdges(probe.blocks, { placeholdersOnly: probe.placeholdersOnly });
|
|
928
|
+
} catch (e) {
|
|
929
|
+
console.warn(`[visual-eval] edge probe unavailable: ${e instanceof Error ? e.message : String(e)}`);
|
|
930
|
+
return null;
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
/** ggui#1492 — a natural capture's geometry: the page's pad per side, and the host's floor when it has one. */
|
|
935
|
+
export interface NaturalCaptureOptions {
|
|
936
|
+
readonly padPx: number;
|
|
937
|
+
readonly floorPx?: number;
|
|
938
|
+
}
|
|
939
|
+
|
|
623
940
|
export async function captureScreenshotDetailed(
|
|
624
941
|
html: string,
|
|
625
942
|
viewport: { width: number; height: number } = DEFAULT_SCREENSHOT_VIEWPORT,
|
|
626
943
|
deps: ScreenshotDeps = {},
|
|
627
944
|
capture: CaptureMode = 'full-page',
|
|
945
|
+
natural: NaturalCaptureOptions = { padPx: JUDGE_INLINE_PAD_PX },
|
|
628
946
|
): Promise<ScreenshotAttempt> {
|
|
629
947
|
try {
|
|
630
948
|
const launchOptions = await resolveLaunchOptions(viewport, deps);
|
|
@@ -639,9 +957,26 @@ export async function captureScreenshotDetailed(
|
|
|
639
957
|
await page.waitForSelector('#root > *', { timeout: 10000 }).catch(() => {});
|
|
640
958
|
// Wait a bit for CSS/fonts to settle
|
|
641
959
|
await new Promise((r) => setTimeout(r, deps.settleMs ?? 1000));
|
|
960
|
+
if (capture === 'natural') {
|
|
961
|
+
// ggui#1475 — the inline card at its natural height: the capture is the card's extent on the host ground.
|
|
962
|
+
const cardHeightPx = await measureContentHeight(page, natural.padPx === JUDGE_INLINE_PAD_PX ? CARD_HEIGHT_EXPRESSION : cardHeightExpression(natural.padPx));
|
|
963
|
+
const overflowX = await measureContentHeight(page, CARD_OVERFLOW_X_EXPRESSION);
|
|
964
|
+
// ggui#1492 — the host's floor: measured AFTER the card (its height stays the card's own), then applied to the
|
|
965
|
+
// frame so the capture shows what a visitor sees under a short card — the frame's surface, as void.
|
|
966
|
+
const floorPx = natural.floorPx;
|
|
967
|
+
if (floorPx !== undefined && cardHeightPx !== null && cardHeightPx < floorPx) {
|
|
968
|
+
await page.evaluate(`(() => { const f = document.querySelector('.${JUDGE_INLINE_FRAME_CLASS}'); if (f) f.style.minHeight = '${floorPx}px'; })()`);
|
|
969
|
+
}
|
|
970
|
+
const framed = cardHeightPx === null ? viewport.height - 2 * natural.padPx : floorPx !== undefined ? Math.max(cardHeightPx, floorPx) : cardHeightPx;
|
|
971
|
+
const clip = naturalClip(viewport, framed, natural.padPx);
|
|
972
|
+
const edge = await measureEdges(page);
|
|
973
|
+
const screenshot = await page.screenshot({ type: 'png', fullPage: false, clip });
|
|
974
|
+
return { png: Buffer.from(screenshot), contentHeight: cardHeightPx, overflowX, edge };
|
|
975
|
+
}
|
|
642
976
|
const contentHeight = await measureContentHeight(page);
|
|
977
|
+
const edge = await measureEdges(page);
|
|
643
978
|
const screenshot = await page.screenshot({ type: 'png', fullPage: capture === 'full-page' });
|
|
644
|
-
return { png: Buffer.from(screenshot), contentHeight };
|
|
979
|
+
return { png: Buffer.from(screenshot), contentHeight, edge };
|
|
645
980
|
} finally {
|
|
646
981
|
await browser.close();
|
|
647
982
|
}
|
|
@@ -741,10 +1076,14 @@ async function callMultimodalLLM(
|
|
|
741
1076
|
screenshot: Buffer,
|
|
742
1077
|
originalPrompt: string,
|
|
743
1078
|
profileBlock = '',
|
|
1079
|
+
criteriaBlock = '',
|
|
744
1080
|
) {
|
|
1081
|
+
// ggui#1436 — the criteria block rides in the USER turn after the profile block, so the
|
|
1082
|
+
// system prompt (and its digest pin) never moves; the block's digest is the set id.
|
|
745
1083
|
const userPrompt =
|
|
746
1084
|
`## Original Request\n${originalPrompt}\n\n` +
|
|
747
1085
|
(profileBlock.length > 0 ? `${profileBlock}\n\n` : '') +
|
|
1086
|
+
(criteriaBlock.length > 0 ? `${criteriaBlock}\n\n` : '') +
|
|
748
1087
|
'Evaluate the screenshot of the generated component.';
|
|
749
1088
|
const agent = createVisionAgent({
|
|
750
1089
|
provider: config.provider === 'claude' ? 'anthropic' : config.provider,
|
|
@@ -802,9 +1141,34 @@ interface CanvasFrame {
|
|
|
802
1141
|
readonly viewport: CanvasViewport;
|
|
803
1142
|
readonly policy: CanvasFitPolicy;
|
|
804
1143
|
readonly fit: 'fill' | undefined;
|
|
1144
|
+
/** The host stand-in drawn round the scope (ggui#1083 cut 3) — the window is padded by its gap; `viewport` stays the card's box. */
|
|
1145
|
+
readonly chrome: CanvasChrome;
|
|
805
1146
|
/** `true` when `canvasViewports` carried an entry for this canvas, whatever its value. */
|
|
806
1147
|
readonly declared: boolean;
|
|
807
1148
|
readonly attempt: ScreenshotAttempt;
|
|
1149
|
+
/** The capture's ink extent (ggui#1120); `null` when there is no capture or it could not be read. */
|
|
1150
|
+
readonly ink: InkExtent | null;
|
|
1151
|
+
/** ggui#1492 — what the judge did with this canvas's host presentation, when the caller supplied one. */
|
|
1152
|
+
readonly presentation?: CanvasPresentationOutcome;
|
|
1153
|
+
}
|
|
1154
|
+
/** The capture's ink extent (ggui#1120), read inside a drawn panel's chrome; an unreadable capture is reported, never blank. */
|
|
1155
|
+
function readInk(png: Buffer | null, chrome: CanvasChrome, canvas: CanvasClass, inlineInkInsetPx: number = JUDGE_INLINE_INK_INSET_PX): InkExtent | null {
|
|
1156
|
+
if (png === null) return null;
|
|
1157
|
+
const ink = readInkExtent(png, chrome === 'panel' ? JUDGE_INK_INSET_PX : chrome === 'ground' ? inlineInkInsetPx : 0);
|
|
1158
|
+
if ('reason' in ink) {
|
|
1159
|
+
console.warn(`[visual-eval] ink extent unreadable at canvas ${canvas}: ${ink.reason}`);
|
|
1160
|
+
return null;
|
|
1161
|
+
}
|
|
1162
|
+
return ink;
|
|
1163
|
+
}
|
|
1164
|
+
/** The deterministic blank verdict (ggui#1120): an issue when the frame read no ink at all; `null` otherwise, an unreadable capture included. */
|
|
1165
|
+
function blankVerdict(canvas: CanvasClass, frame: CanvasFrame): VisualIssue | null {
|
|
1166
|
+
// ggui#1475 — a naturally-captured card that measured no height painted nothing: its capture is only the host's frame,
|
|
1167
|
+
// which the ink reading skips by design, so the blank is read from the measurement, never mistaken for unreadable.
|
|
1168
|
+
if (frame.chrome === 'ground' && frame.attempt.contentHeight !== null && frame.attempt.contentHeight <= 0) {
|
|
1169
|
+
return canvasBlankIssue(canvas, frame.viewport);
|
|
1170
|
+
}
|
|
1171
|
+
return frame.ink !== null && frame.ink.lastInkRow === null ? canvasBlankIssue(canvas, frame.viewport) : null;
|
|
808
1172
|
}
|
|
809
1173
|
|
|
810
1174
|
async function frameCanvas(
|
|
@@ -814,14 +1178,29 @@ async function frameCanvas(
|
|
|
814
1178
|
canvas: CanvasClass,
|
|
815
1179
|
canvasViewports: VisualEvalConfig['canvasViewports'],
|
|
816
1180
|
deps: ScreenshotDeps,
|
|
1181
|
+
hostPresentations?: VisualEvalConfig['hostPresentations'],
|
|
817
1182
|
): Promise<CanvasFrame> {
|
|
818
1183
|
const declaredBox = canvasViewports?.[canvas];
|
|
819
1184
|
const viewport = declaredBox !== undefined ? integerBox(declaredBox) : CANVAS_VIEWPORTS[canvas];
|
|
820
1185
|
const policy = canvasFitPolicy(canvas);
|
|
821
1186
|
const fit = canvasFit(canvas);
|
|
822
|
-
const
|
|
823
|
-
const
|
|
824
|
-
|
|
1187
|
+
const chrome = canvasChrome(canvas);
|
|
1188
|
+
const outcome = resolvePresentation(canvas, hostPresentations?.[canvas]);
|
|
1189
|
+
const presentation = outcome?.status === 'applied' ? outcome.applied : undefined;
|
|
1190
|
+
const geometry = inlineGeometry(presentation?.frame);
|
|
1191
|
+
const canvasHtml =
|
|
1192
|
+
fit !== undefined || chrome !== undefined ? buildRenderHTML(bundledCode, context.cssTokens, fit, chrome, presentation) : html;
|
|
1193
|
+
const captured = await captureScreenshotDetailed(canvasHtml, judgeWindow(viewport, chrome, geometry.padPx), deps, policy.capture, {
|
|
1194
|
+
padPx: geometry.padPx,
|
|
1195
|
+
...(geometry.floorPx !== undefined ? { floorPx: geometry.floorPx } : {}),
|
|
1196
|
+
});
|
|
1197
|
+
// ggui#1083 cut 3 — on a panelled page the document is the panel plus its gap; every verdict reads the CARD's height.
|
|
1198
|
+
const attempt: ScreenshotAttempt = { ...captured, contentHeight: cardHeight(captured.contentHeight, chrome) };
|
|
1199
|
+
return {
|
|
1200
|
+
viewport, policy, fit, chrome, declared: declaredBox !== undefined, attempt,
|
|
1201
|
+
ink: readInk(captured.png, chrome, canvas, geometry.inkInsetPx),
|
|
1202
|
+
...(outcome !== undefined ? { presentation: outcome } : {}),
|
|
1203
|
+
};
|
|
825
1204
|
}
|
|
826
1205
|
|
|
827
1206
|
/**
|
|
@@ -829,17 +1208,161 @@ async function frameCanvas(
|
|
|
829
1208
|
* canvas's policy gives an overflow, or `null` when the content fits, could
|
|
830
1209
|
* not be measured, or the canvas scrolls by design.
|
|
831
1210
|
*/
|
|
832
|
-
function fitVerdict(canvas: CanvasClass, frame: CanvasFrame):
|
|
1211
|
+
function fitVerdict(canvas: CanvasClass, frame: CanvasFrame): VisualIssue | null {
|
|
833
1212
|
const contentHeight = frame.attempt.contentHeight;
|
|
834
1213
|
if (contentHeight === null || contentHeight <= frame.viewport.height || frame.policy.overflow === 'none') return null;
|
|
835
1214
|
return canvasOverflowIssue(canvas, frame.viewport, contentHeight, frame.policy.overflow);
|
|
836
1215
|
}
|
|
837
1216
|
|
|
1217
|
+
/**
|
|
1218
|
+
* ggui#1475 — the deterministic horizontal verdict on a natural capture: the card's content runs past its width. It
|
|
1219
|
+
* is judged by the canvas's own fit policy (the inline card: a critical that fails the canvas), in its own dimension,
|
|
1220
|
+
* so the in-loop `[fit]` extension (vertical overflow) is unchanged.
|
|
1221
|
+
*/
|
|
1222
|
+
function overflowXVerdict(canvas: CanvasClass, frame: CanvasFrame): VisualIssue | null {
|
|
1223
|
+
const px = frame.attempt.overflowX;
|
|
1224
|
+
// `scrollWidth` / `clientWidth` are rounded integers: a fractional layout width reads as 1 px past the card with
|
|
1225
|
+
// nothing a visitor could see, so 1 px is rounding and an overflow starts at 2.
|
|
1226
|
+
if (px === undefined || px === null || px <= OVERFLOW_X_ROUNDING_PX || frame.policy.overflow === 'none') return null;
|
|
1227
|
+
return canvasOverflowXIssue(canvas, frame.viewport, px, frame.policy.overflow);
|
|
1228
|
+
}
|
|
1229
|
+
|
|
838
1230
|
/** One judge call parsed, retried ONCE on a malformed answer (#1017); the FIRST reason is kept verbatim. */
|
|
839
1231
|
type JudgedAnswer =
|
|
840
|
-
| {
|
|
1232
|
+
| {
|
|
1233
|
+
readonly kind: 'ok';
|
|
1234
|
+
readonly result: EvaluationResult;
|
|
1235
|
+
readonly inputTokens: number;
|
|
1236
|
+
readonly outputTokens: number;
|
|
1237
|
+
/** ggui#1127 — present when the answer did not parse and its score was salvaged. */
|
|
1238
|
+
readonly salvage?: JudgeSalvage;
|
|
1239
|
+
/** ggui#1687 — the provider's stop reason and the model it says answered; each absent when unreported. */
|
|
1240
|
+
readonly finishReason?: VisionFinishReason;
|
|
1241
|
+
readonly servedModel?: string;
|
|
1242
|
+
}
|
|
841
1243
|
| { readonly kind: 'unparsable'; readonly reason: string };
|
|
842
1244
|
|
|
1245
|
+
/** ggui#1127 — what a salvaged answer leaves on the record: why, and where the parse broke. Never the answer's text. */
|
|
1246
|
+
interface JudgeSalvage {
|
|
1247
|
+
readonly cause: JudgeSalvageCause;
|
|
1248
|
+
/** The provider's normalized stop reason; absent when it reported none. */
|
|
1249
|
+
readonly finishReason?: VisionFinishReason;
|
|
1250
|
+
/** The parser's error, with every quoted span of the answer removed. */
|
|
1251
|
+
readonly parseError: string;
|
|
1252
|
+
/** Where the parse broke, as a character offset into the answer, when the parser said. */
|
|
1253
|
+
readonly offset?: number;
|
|
1254
|
+
/** The answer's length in characters: an offset at the end reads as a cut, one mid-answer as malformed. */
|
|
1255
|
+
readonly chars: number;
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
/**
|
|
1259
|
+
* ggui#1127 — why an unparsable answer was salvaged. The cause comes from the provider's stop reason, never from the
|
|
1260
|
+
* answer's length: a short answer can still be cut (thinking spends the cap), and a long one can still be malformed.
|
|
1261
|
+
*/
|
|
1262
|
+
function judgeSalvageOf(error: unknown, text: string, finishReason: VisionFinishReason | undefined): JudgeSalvage {
|
|
1263
|
+
const cause: JudgeSalvageCause =
|
|
1264
|
+
finishReason === undefined ? 'unknown' : finishReason === 'length' ? 'output-cap' : finishReason === 'stop' ? 'malformed' : 'stopped';
|
|
1265
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1266
|
+
// The JSON is parsed from the first `{` (parseVisualResponse), so the parser's position is shifted by that start.
|
|
1267
|
+
const position = /position (\d+)/.exec(message);
|
|
1268
|
+
const start = text.indexOf('{');
|
|
1269
|
+
const offset = position !== null ? (start > 0 ? start : 0) + Number(position[1]) : undefined;
|
|
1270
|
+
const parseError = message.startsWith('Visual evaluator did not return valid JSON')
|
|
1271
|
+
? 'no complete JSON object in the answer'
|
|
1272
|
+
: message.replace(/"(?:[^"\\]|\\.)*"/g, '"…"').slice(0, 160);
|
|
1273
|
+
return { cause, ...(finishReason !== undefined ? { finishReason } : {}), parseError, ...(offset !== undefined ? { offset } : {}), chars: text.length };
|
|
1274
|
+
}
|
|
1275
|
+
|
|
1276
|
+
/** ggui#1127 — a canvas's judge record: the K samples, and which of them (and the criteria call) were salvaged. */
|
|
1277
|
+
function judgeRecordOf(
|
|
1278
|
+
k: number,
|
|
1279
|
+
parsed: readonly Extract<JudgedAnswer, { kind: 'ok' }>[],
|
|
1280
|
+
samples: number[],
|
|
1281
|
+
criteria: JudgedAnswer | undefined,
|
|
1282
|
+
requestedModel: string,
|
|
1283
|
+
): CanvasJudgeRecord {
|
|
1284
|
+
const salvaged = parsed.flatMap((a) => (a.salvage !== undefined ? [a.salvage.cause] : []));
|
|
1285
|
+
// ggui#1687 — every model the provider said answered, scoring and criteria, deduplicated in call order.
|
|
1286
|
+
const answered = [...parsed, ...(criteria?.kind === 'ok' ? [criteria] : [])];
|
|
1287
|
+
const served = [...new Set(answered.flatMap((a) => (a.servedModel !== undefined ? [a.servedModel] : [])))];
|
|
1288
|
+
const criteriaSalvaged = criteria?.kind === 'ok' ? criteria.salvage?.cause : undefined;
|
|
1289
|
+
return {
|
|
1290
|
+
k,
|
|
1291
|
+
rule: 'median',
|
|
1292
|
+
samples,
|
|
1293
|
+
sigma: populationSigma(samples),
|
|
1294
|
+
notes: parsed.map((a) => a.result.critique ?? ''),
|
|
1295
|
+
...(salvaged.length > 0 ? { salvaged } : {}),
|
|
1296
|
+
...(criteriaSalvaged !== undefined ? { criteriaSalvaged } : {}),
|
|
1297
|
+
models: { requested: requestedModel, served },
|
|
1298
|
+
};
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
/** ggui#1687 — what one call's response says about itself: its stop reason and the model that answered. */
|
|
1302
|
+
function callFacts(response: { readonly finishReason?: VisionFinishReason; readonly servedModel?: string }): {
|
|
1303
|
+
finishReason?: VisionFinishReason;
|
|
1304
|
+
servedModel?: string;
|
|
1305
|
+
} {
|
|
1306
|
+
return {
|
|
1307
|
+
...(response.finishReason !== undefined ? { finishReason: response.finishReason } : {}),
|
|
1308
|
+
...(response.servedModel !== undefined ? { servedModel: response.servedModel } : {}),
|
|
1309
|
+
};
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
const NO_CRITERIA_ARRAY: CriteriaAnswerCounts = { array: false, received: 0, kept: 0, dropped: { notObject: 0, badId: 0, badVerdict: 0 } };
|
|
1313
|
+
|
|
1314
|
+
/** ggui#1687 — the criteria call as it came back, for the block: its answer's counts and its stop reason. */
|
|
1315
|
+
function criteriaCallOf(criteria: JudgedAnswer | undefined): { call?: { answer: CriteriaAnswerCounts; finishReason?: VisionFinishReason } } {
|
|
1316
|
+
if (criteria?.kind !== 'ok') return {};
|
|
1317
|
+
const answer = criteria.result.criteriaCounts ?? NO_CRITERIA_ARRAY;
|
|
1318
|
+
return { call: { answer, ...(criteria.finishReason !== undefined ? { finishReason: criteria.finishReason } : {}) } };
|
|
1319
|
+
}
|
|
1320
|
+
|
|
1321
|
+
/**
|
|
1322
|
+
* ggui#1436 — the criteria are REPORT-ONLY, so they are never asked inside a call whose score binds. The K
|
|
1323
|
+
* scoring calls go out exactly as they did before the block existed (the same user turn, byte for byte); when a
|
|
1324
|
+
* block was selected, the criteria get ONE call of their own on the same frame, run beside them, and that call's
|
|
1325
|
+
* score is discarded. Its spend is reported apart (`criteriaTokens`), never inside the scoring calls' tokens, so a
|
|
1326
|
+
* cost read off the judgement is the judgement's alone. Asked inside the scoring call, the block moved the score it
|
|
1327
|
+
* sat beside — the first release that carried it into a served judge lowered first-attempt scores on the same kind
|
|
1328
|
+
* of card (ggui#1436). One read, not K: the block binds on nothing, and a second K would double its calls.
|
|
1329
|
+
*/
|
|
1330
|
+
async function judgeFrame(
|
|
1331
|
+
judge: typeof callMultimodalLLM,
|
|
1332
|
+
config: VisualEvalConfig,
|
|
1333
|
+
model: string,
|
|
1334
|
+
png: Buffer,
|
|
1335
|
+
originalPrompt: string,
|
|
1336
|
+
profileBlock: string,
|
|
1337
|
+
canvas: CanvasClass,
|
|
1338
|
+
k: number,
|
|
1339
|
+
criteriaBlock: string,
|
|
1340
|
+
): Promise<{ readonly scoring: readonly JudgedAnswer[]; readonly criteria: JudgedAnswer | undefined }> {
|
|
1341
|
+
const [scoring, criteria] = await Promise.all([
|
|
1342
|
+
Promise.all(Array.from({ length: k }, () => judgeAndParse(judge, config, model, png, originalPrompt, profileBlock, canvas))),
|
|
1343
|
+
criteriaBlock.length > 0 ? judgeAndParse(judge, config, model, png, originalPrompt, profileBlock, canvas, criteriaBlock) : undefined,
|
|
1344
|
+
]);
|
|
1345
|
+
return { scoring, criteria };
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
/** What the frame's criteria call spent, when it was asked and answered — reported apart from the scoring calls. */
|
|
1349
|
+
function criteriaCallTokens(criteria: JudgedAnswer | undefined): CriteriaCallTokens | undefined {
|
|
1350
|
+
return criteria?.kind === 'ok' ? { inputTokens: criteria.inputTokens, outputTokens: criteria.outputTokens } : undefined;
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
/** The criteria call's answers, as the one read the block resolves; none when it was not asked or did not parse. */
|
|
1354
|
+
function criteriaReads(criteria: JudgedAnswer | undefined): CriteriaAnswer[][] {
|
|
1355
|
+
// ggui#1127 — a salvaged answer keeps no criteria: it is no read, and its rows say why (criteriaUnansweredReason).
|
|
1356
|
+
return criteria?.kind === 'ok' && criteria.salvage === undefined ? [criteria.result.criteriaAnswers ?? []] : [];
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
/** ggui#1127 — the reason a salvaged criteria answer's judge rows give, so none reads as a clean `n/a`. */
|
|
1360
|
+
function criteriaUnansweredReason(criteria: JudgedAnswer | undefined): { unansweredReason?: string } {
|
|
1361
|
+
return criteria?.kind === 'ok' && criteria.salvage !== undefined
|
|
1362
|
+
? { unansweredReason: `not answered: the criteria answer was salvaged (${criteria.salvage.cause})` }
|
|
1363
|
+
: {};
|
|
1364
|
+
}
|
|
1365
|
+
|
|
843
1366
|
/** How many vision calls this canvas gets (ggui#1072): `judgeK` when the canvas is sampled, else 1. */
|
|
844
1367
|
function judgeCountFor(config: VisualEvalConfig, canvas: CanvasClass): number {
|
|
845
1368
|
const k = Math.max(1, Math.floor(config.judgeK ?? 1));
|
|
@@ -869,21 +1392,28 @@ async function judgeAndParse(
|
|
|
869
1392
|
originalPrompt: string,
|
|
870
1393
|
profileBlock: string,
|
|
871
1394
|
canvas?: CanvasClass,
|
|
1395
|
+
criteriaBlock = '',
|
|
872
1396
|
): Promise<JudgedAnswer> {
|
|
873
1397
|
let firstReason: string | undefined;
|
|
874
1398
|
for (let attempt = 1; attempt <= 2; attempt++) {
|
|
875
|
-
const response = await judge(config, model, VISUAL_EVAL_PROMPT, screenshot, originalPrompt, profileBlock);
|
|
1399
|
+
const response = await judge(config, model, VISUAL_EVAL_PROMPT, screenshot, originalPrompt, profileBlock, criteriaBlock);
|
|
876
1400
|
try {
|
|
877
1401
|
const result = parseVisualResponse(response.text, config.passThreshold);
|
|
878
|
-
return { kind: 'ok', result, inputTokens: response.inputTokens, outputTokens: response.outputTokens };
|
|
1402
|
+
return { kind: 'ok', result, inputTokens: response.inputTokens, outputTokens: response.outputTokens, ...callFacts(response) };
|
|
879
1403
|
} catch (e) {
|
|
880
|
-
//
|
|
881
|
-
// the
|
|
882
|
-
//
|
|
1404
|
+
// An answer that fails to parse but whose four dimensions closed is SALVAGED (ggui#1127): they are emitted
|
|
1405
|
+
// before `issues`, so the score is recoverable from the closed prefix. Whether it was cut at the output cap or
|
|
1406
|
+
// ended malformed is the provider's stop reason to say, not the answer's length; the record and the warn
|
|
1407
|
+
// line carry the cause, the parse error and where it broke, never the answer's text. One token per leg.
|
|
883
1408
|
const salvaged = salvageTruncatedVisualAnswer(response.text, config.passThreshold);
|
|
884
1409
|
if (salvaged !== null) {
|
|
885
|
-
|
|
886
|
-
|
|
1410
|
+
const salvage = judgeSalvageOf(e, response.text, response.finishReason);
|
|
1411
|
+
const token = criteriaBlock.length > 0 ? 'visual_criteria_salvaged' : 'visual_judge_salvaged';
|
|
1412
|
+
console.warn(
|
|
1413
|
+
`[visual-eval] ${token}${canvas ? ` canvas=${canvas}` : ''} cause=${salvage.cause} finish=${salvage.finishReason ?? 'unreported'} ` +
|
|
1414
|
+
`at=${salvage.offset ?? '?'}/${salvage.chars} (${salvage.parseError}): score recovered from the closed prefix, issues list partial`,
|
|
1415
|
+
);
|
|
1416
|
+
return { kind: 'ok', result: salvaged, inputTokens: response.inputTokens, outputTokens: response.outputTokens, salvage, ...callFacts(response) };
|
|
887
1417
|
}
|
|
888
1418
|
const reason = `judge answer unparsable: ${e instanceof Error ? e.message : String(e)}`;
|
|
889
1419
|
if (attempt === 1) {
|
|
@@ -941,7 +1471,7 @@ export async function runVisualEvaluationDetailed(
|
|
|
941
1471
|
const perCanvas: CanvasVisualResult[] = [];
|
|
942
1472
|
const perCanvasResults: EvaluationResult[] = [];
|
|
943
1473
|
for (const canvas of config.canvases) {
|
|
944
|
-
const frame = await frameCanvas(bundledCode, context, html, canvas, config.canvasViewports, deps);
|
|
1474
|
+
const frame = await frameCanvas(bundledCode, context, html, canvas, config.canvasViewports, deps, config.hostPresentations);
|
|
945
1475
|
const { viewport, policy, fit, attempt } = frame;
|
|
946
1476
|
const screenshot = attempt.png;
|
|
947
1477
|
if (!screenshot) {
|
|
@@ -952,8 +1482,20 @@ export async function runVisualEvaluationDetailed(
|
|
|
952
1482
|
// ggui#1072: k vision calls on the SAME frame; the decision value is the median.
|
|
953
1483
|
const k = judgeCountFor(config, canvas);
|
|
954
1484
|
const profileBlock = buildStylingProfileJudgeBlock(context.profile);
|
|
955
|
-
|
|
956
|
-
|
|
1485
|
+
// ggui#1436 — the selection is decided BEFORE the judge runs and recorded with the verdict.
|
|
1486
|
+
const criteriaContext: CriteriaContext | undefined =
|
|
1487
|
+
context.criteria !== undefined ? { ...context.criteria.context, canvas } : undefined;
|
|
1488
|
+
const criteriaSelected =
|
|
1489
|
+
context.criteria !== undefined && criteriaContext !== undefined ? selectCriteria(context.criteria.bank, criteriaContext) : undefined;
|
|
1490
|
+
const criteriaBlock =
|
|
1491
|
+
context.criteria !== undefined && criteriaSelected !== undefined
|
|
1492
|
+
? buildCriteriaJudgeBlock(context.criteria.bank, criteriaSelected, {
|
|
1493
|
+
frame: { canvas, width: viewport.width, height: viewport.height, ...(frame.presentation?.status === 'applied' ? { hostLabel: frame.presentation.applied.label } : {}) },
|
|
1494
|
+
...(config.sampleProps !== undefined ? { propsJson: JSON.stringify(config.sampleProps) } : {}),
|
|
1495
|
+
})
|
|
1496
|
+
: '';
|
|
1497
|
+
const { scoring: answers, criteria: criteriaAnswer } = await judgeFrame(
|
|
1498
|
+
judge, config, model, screenshot, context.originalPrompt, profileBlock, canvas, k, criteriaBlock,
|
|
957
1499
|
);
|
|
958
1500
|
const parsed = answers.filter((a): a is Extract<JudgedAnswer, { kind: 'ok' }> => a.kind === 'ok');
|
|
959
1501
|
if (parsed.length === 0) {
|
|
@@ -969,13 +1511,8 @@ export async function runVisualEvaluationDetailed(
|
|
|
969
1511
|
result.inputTokens = parsed.reduce((sum, a) => sum + a.inputTokens, 0);
|
|
970
1512
|
result.outputTokens = parsed.reduce((sum, a) => sum + a.outputTokens, 0);
|
|
971
1513
|
const response = { inputTokens: result.inputTokens, outputTokens: result.outputTokens };
|
|
972
|
-
const
|
|
973
|
-
|
|
974
|
-
rule: 'median',
|
|
975
|
-
samples,
|
|
976
|
-
sigma: populationSigma(samples),
|
|
977
|
-
notes: parsed.map((a) => a.result.critique ?? ''),
|
|
978
|
-
};
|
|
1514
|
+
const criteriaTokens = criteriaCallTokens(criteriaAnswer);
|
|
1515
|
+
const judgeRecord = judgeRecordOf(k, parsed, samples, criteriaAnswer, model);
|
|
979
1516
|
// The fit verdict (ggui#1027): deterministic, in the judge's issue channel.
|
|
980
1517
|
const contentHeight = attempt.contentHeight;
|
|
981
1518
|
const overflow = contentHeight !== null && contentHeight > viewport.height;
|
|
@@ -984,6 +1521,31 @@ export async function runVisualEvaluationDetailed(
|
|
|
984
1521
|
result.issues.push(fitIssue);
|
|
985
1522
|
if (policy.overflow === 'fail') result.passed = false;
|
|
986
1523
|
}
|
|
1524
|
+
const overflowXIssue = overflowXVerdict(canvas, frame);
|
|
1525
|
+
if (overflowXIssue !== null) {
|
|
1526
|
+
result.issues.push(overflowXIssue);
|
|
1527
|
+
if (policy.overflow === 'fail') result.passed = false;
|
|
1528
|
+
}
|
|
1529
|
+
// The blank verdict (ggui#1120): deterministic, critical on every canvas — a render that painted nothing fails.
|
|
1530
|
+
const blankIssue = blankVerdict(canvas, frame);
|
|
1531
|
+
if (blankIssue !== null) {
|
|
1532
|
+
result.issues.push(blankIssue);
|
|
1533
|
+
result.passed = false;
|
|
1534
|
+
}
|
|
1535
|
+
const inkRatio = frame.ink?.ratio ?? null;
|
|
1536
|
+
// ggui#1436 — the block: the criteria call's read, the instruments' measurements, report-only.
|
|
1537
|
+
const criteria: CriteriaBlock | undefined =
|
|
1538
|
+
context.criteria !== undefined && criteriaContext !== undefined && criteriaSelected !== undefined
|
|
1539
|
+
? resolveCriteriaBlock({
|
|
1540
|
+
bank: context.criteria.bank,
|
|
1541
|
+
context: criteriaContext,
|
|
1542
|
+
selected: criteriaSelected,
|
|
1543
|
+
answers: criteriaReads(criteriaAnswer),
|
|
1544
|
+
measurements: { overflow, contentHeight, viewportHeight: viewport.height, inkRatio, edge: attempt.edge ?? null },
|
|
1545
|
+
...criteriaUnansweredReason(criteriaAnswer),
|
|
1546
|
+
...criteriaCallOf(criteriaAnswer),
|
|
1547
|
+
})
|
|
1548
|
+
: undefined;
|
|
987
1549
|
perCanvasResults.push(result);
|
|
988
1550
|
perCanvas.push({
|
|
989
1551
|
canvas,
|
|
@@ -993,14 +1555,20 @@ export async function runVisualEvaluationDetailed(
|
|
|
993
1555
|
screenshotPng: screenshot,
|
|
994
1556
|
contentHeight,
|
|
995
1557
|
overflow,
|
|
1558
|
+
inkRatio,
|
|
996
1559
|
judge: judgeRecord,
|
|
997
1560
|
...(fit !== undefined ? { fit } : {}),
|
|
998
1561
|
...(frame.declared ? { declared: true as const } : {}),
|
|
1562
|
+
...(criteria !== undefined ? { criteria } : {}),
|
|
1563
|
+
...(criteriaTokens !== undefined ? { criteriaTokens } : {}),
|
|
1564
|
+
...(frame.presentation !== undefined ? { presentation: frame.presentation } : {}),
|
|
999
1565
|
});
|
|
1000
1566
|
console.log(
|
|
1001
1567
|
`[visual-eval] canvas=${canvas} ${viewport.width}×${viewport.height} score=${result.finalScore}${k > 1 ? ` (median of ${judgeRecord.samples.length}/${k}: ${judgeRecord.samples.join(',')} σ=${judgeRecord.sigma})` : ''} ` +
|
|
1002
1568
|
`${result.passed ? 'pass' : 'FAIL'} | content=${contentHeight ?? '?'}px${overflow ? ` OVERFLOW (${policy.overflow})` : ''} ` +
|
|
1003
|
-
|
|
1569
|
+
`ink=${inkRatio ?? '?'}${blankIssue !== null ? ' BLANK' : ''} ` +
|
|
1570
|
+
`| in=${response.inputTokens} out=${response.outputTokens}` +
|
|
1571
|
+
(criteriaTokens !== undefined ? ` | criteria in=${criteriaTokens.inputTokens} out=${criteriaTokens.outputTokens}` : ''),
|
|
1004
1572
|
);
|
|
1005
1573
|
}
|
|
1006
1574
|
const aggregate = aggregateCanvasResults(perCanvas, perCanvasResults);
|
|
@@ -1081,7 +1649,21 @@ function aggregateCanvasResults(
|
|
|
1081
1649
|
inputTokens += r.inputTokens ?? 0;
|
|
1082
1650
|
outputTokens += r.outputTokens ?? 0;
|
|
1083
1651
|
});
|
|
1652
|
+
const criteriaSpent = perCanvas.flatMap((c) => (c.criteriaTokens !== undefined ? [c.criteriaTokens] : []));
|
|
1653
|
+
// ggui#1127 — N of M per leg. A criteria call that parsed or was salvaged reports its spend, so `criteriaTokens`
|
|
1654
|
+
// marks an answered call.
|
|
1655
|
+
const judgeSalvage: JudgeSalvageCounts = {
|
|
1656
|
+
scoring: {
|
|
1657
|
+
salvaged: perCanvas.reduce((sum, c) => sum + (c.judge.salvaged?.length ?? 0), 0),
|
|
1658
|
+
answers: perCanvas.reduce((sum, c) => sum + c.judge.samples.length, 0),
|
|
1659
|
+
},
|
|
1660
|
+
criteria: {
|
|
1661
|
+
salvaged: perCanvas.filter((c) => c.judge.criteriaSalvaged !== undefined).length,
|
|
1662
|
+
answered: perCanvas.filter((c) => c.criteriaTokens !== undefined).length,
|
|
1663
|
+
},
|
|
1664
|
+
};
|
|
1084
1665
|
return {
|
|
1666
|
+
judgeSalvage,
|
|
1085
1667
|
passed: perCanvas.every((c) => c.passed),
|
|
1086
1668
|
finalScore: mean((r) => r.finalScore),
|
|
1087
1669
|
dimensions,
|
|
@@ -1089,6 +1671,14 @@ function aggregateCanvasResults(
|
|
|
1089
1671
|
...(critiques.length > 0 ? { critique: critiques.join('\n') } : {}),
|
|
1090
1672
|
inputTokens,
|
|
1091
1673
|
outputTokens,
|
|
1674
|
+
...(criteriaSpent.length > 0
|
|
1675
|
+
? {
|
|
1676
|
+
criteriaTokens: {
|
|
1677
|
+
inputTokens: criteriaSpent.reduce((sum, t) => sum + t.inputTokens, 0),
|
|
1678
|
+
outputTokens: criteriaSpent.reduce((sum, t) => sum + t.outputTokens, 0),
|
|
1679
|
+
},
|
|
1680
|
+
}
|
|
1681
|
+
: {}),
|
|
1092
1682
|
canvases: [...perCanvas],
|
|
1093
1683
|
};
|
|
1094
1684
|
}
|
|
@@ -1103,9 +1693,19 @@ export function summarizeVisualResult(result: VisualEvaluationResult): VisualEva
|
|
|
1103
1693
|
passed: c.passed,
|
|
1104
1694
|
contentHeight: c.contentHeight,
|
|
1105
1695
|
overflow: c.overflow,
|
|
1696
|
+
inkRatio: c.inkRatio,
|
|
1106
1697
|
judge: c.judge,
|
|
1107
1698
|
...(c.fit !== undefined ? { fit: c.fit } : {}),
|
|
1699
|
+
...(c.criteria !== undefined ? { criteria: c.criteria } : {}),
|
|
1700
|
+
...(c.presentation !== undefined ? { presentation: c.presentation } : {}),
|
|
1108
1701
|
}));
|
|
1702
|
+
// ggui#1436 — the roll-up: ids failed / unreadable across canvases, deduped; a must at n/a is listed, never passed.
|
|
1703
|
+
const blocks = result.canvases.map((c) => c.criteria).filter((b): b is CriteriaBlock => b !== undefined);
|
|
1704
|
+
const idsWhere = (severity: 'must' | 'should', verdict: 'fail' | 'n/a'): string[] => [
|
|
1705
|
+
...new Set(blocks.flatMap((b) => b.verdicts.filter((v) => v.severity === severity && v.verdict === verdict).map((v) => v.id))),
|
|
1706
|
+
];
|
|
1707
|
+
const rollup: CriteriaRollup | undefined =
|
|
1708
|
+
blocks.length > 0 ? { mustFailed: idsWhere('must', 'fail'), shouldFailed: idsWhere('should', 'fail'), mustNa: idsWhere('must', 'n/a') } : undefined;
|
|
1109
1709
|
// ggui#1195 — the fit stamp: the first canvas whose policy FAILS an
|
|
1110
1710
|
// overflow (the inline card), judged with a measurable height.
|
|
1111
1711
|
const fitCanvas = result.canvases.find((c) => canvasFitPolicy(c.canvas).overflow === 'fail' && c.contentHeight !== null);
|
|
@@ -1126,6 +1726,136 @@ export function summarizeVisualResult(result: VisualEvaluationResult): VisualEva
|
|
|
1126
1726
|
...(fit !== undefined ? { fit } : {}),
|
|
1127
1727
|
...(result.design !== undefined ? { design: result.design } : {}),
|
|
1128
1728
|
...(result.themeMode !== undefined ? { themeMode: result.themeMode } : {}),
|
|
1729
|
+
...(rollup !== undefined ? { criteria: rollup } : {}),
|
|
1730
|
+
};
|
|
1731
|
+
}
|
|
1732
|
+
|
|
1733
|
+
// ---------------------------------------------------------------------------
|
|
1734
|
+
// A stored capture, judged (ggui#1438)
|
|
1735
|
+
// ---------------------------------------------------------------------------
|
|
1736
|
+
|
|
1737
|
+
/** One stored capture to judge: the frame the judge saw at capture time, plus what the capture measured. */
|
|
1738
|
+
export interface StoredCaptureInput {
|
|
1739
|
+
readonly canvas: CanvasClass;
|
|
1740
|
+
/** The capture as saved (`canvas-<class>.png`). */
|
|
1741
|
+
readonly png: Buffer;
|
|
1742
|
+
/** The box the capture was taken at — the class box, or the declared one. */
|
|
1743
|
+
readonly viewport: CanvasViewport;
|
|
1744
|
+
/** The document's scroll height the capture recorded; `null` when it did not. */
|
|
1745
|
+
readonly contentHeight: number | null;
|
|
1746
|
+
readonly originalPrompt: string;
|
|
1747
|
+
readonly profile?: GenerationProfileInput;
|
|
1748
|
+
/** ggui#1436 — the bank and the card's context; absent = today's judge, no block. */
|
|
1749
|
+
readonly criteria?: { readonly bank: CriteriaBank; readonly context: CriteriaContextInput };
|
|
1750
|
+
/** The props the capture was rendered with — handed to the criteria block for the rows whose evidence names props. */
|
|
1751
|
+
readonly sampleProps?: JsonObject;
|
|
1752
|
+
}
|
|
1753
|
+
|
|
1754
|
+
export interface StoredCaptureVerdict {
|
|
1755
|
+
readonly canvas: CanvasClass;
|
|
1756
|
+
readonly viewport: CanvasViewport;
|
|
1757
|
+
readonly score: number;
|
|
1758
|
+
readonly passed: boolean;
|
|
1759
|
+
readonly issues: EvaluationIssue[];
|
|
1760
|
+
readonly judge: CanvasJudgeRecord;
|
|
1761
|
+
readonly contentHeight: number | null;
|
|
1762
|
+
readonly overflow: boolean;
|
|
1763
|
+
readonly inkRatio: number | null;
|
|
1764
|
+
readonly criteria?: CriteriaBlock;
|
|
1765
|
+
/** The scoring calls' spend. */
|
|
1766
|
+
readonly inputTokens: number;
|
|
1767
|
+
readonly outputTokens: number;
|
|
1768
|
+
/** ggui#1436 — the report-only criteria call's spend, apart from the scoring calls'; absent when none was asked or answered. */
|
|
1769
|
+
readonly criteriaTokens?: CriteriaCallTokens;
|
|
1770
|
+
}
|
|
1771
|
+
|
|
1772
|
+
export type StoredCaptureOutcome =
|
|
1773
|
+
| { readonly kind: 'ok'; readonly verdict: StoredCaptureVerdict }
|
|
1774
|
+
| { readonly kind: 'unavailable'; readonly reason: string };
|
|
1775
|
+
|
|
1776
|
+
/**
|
|
1777
|
+
* Judge a STORED capture exactly as the per-canvas leg judges a fresh one — the same K scoring calls on the
|
|
1778
|
+
* same frame, the same median, the same fit and blank verdicts, the same report-only criteria call — without
|
|
1779
|
+
* rendering.
|
|
1780
|
+
* This is the calibration's path (ggui#1438): the frame his eye marks is the frame the judge saw, and a
|
|
1781
|
+
* re-judge costs judge tokens only. Fit reads the capture's recorded `contentHeight`; ink is read off the
|
|
1782
|
+
* PNG inside the canvas's chrome, as at capture.
|
|
1783
|
+
*/
|
|
1784
|
+
export async function judgeStoredCapture(
|
|
1785
|
+
input: StoredCaptureInput,
|
|
1786
|
+
config: VisualEvalConfig,
|
|
1787
|
+
deps: Pick<VisualEvalDeps, 'judge'> = {},
|
|
1788
|
+
): Promise<StoredCaptureOutcome> {
|
|
1789
|
+
const judge = deps.judge ?? callMultimodalLLM;
|
|
1790
|
+
const model = config.model ?? getDefaultVisualModel(config.provider);
|
|
1791
|
+
const { canvas, viewport, png } = input;
|
|
1792
|
+
const chrome = canvasChrome(canvas);
|
|
1793
|
+
const ink = readInk(png, chrome, canvas);
|
|
1794
|
+
const inkRatio = ink?.ratio ?? null;
|
|
1795
|
+
const k = judgeCountFor(config, canvas);
|
|
1796
|
+
const profileBlock = buildStylingProfileJudgeBlock(input.profile);
|
|
1797
|
+
const criteriaContext: CriteriaContext | undefined = input.criteria !== undefined ? { ...input.criteria.context, canvas } : undefined;
|
|
1798
|
+
const criteriaSelected =
|
|
1799
|
+
input.criteria !== undefined && criteriaContext !== undefined ? selectCriteria(input.criteria.bank, criteriaContext) : undefined;
|
|
1800
|
+
const criteriaBlock =
|
|
1801
|
+
input.criteria !== undefined && criteriaSelected !== undefined
|
|
1802
|
+
? buildCriteriaJudgeBlock(input.criteria.bank, criteriaSelected, {
|
|
1803
|
+
frame: { canvas, width: viewport.width, height: viewport.height },
|
|
1804
|
+
...(input.sampleProps !== undefined ? { propsJson: JSON.stringify(input.sampleProps) } : {}),
|
|
1805
|
+
})
|
|
1806
|
+
: '';
|
|
1807
|
+
const { scoring: answers, criteria: criteriaAnswer } = await judgeFrame(judge, config, model, png, input.originalPrompt, profileBlock, canvas, k, criteriaBlock);
|
|
1808
|
+
const parsed = answers.filter((a): a is Extract<JudgedAnswer, { kind: 'ok' }> => a.kind === 'ok');
|
|
1809
|
+
if (parsed.length === 0) {
|
|
1810
|
+
const first = answers.find((a): a is Extract<JudgedAnswer, { kind: 'unparsable' }> => a.kind === 'unparsable');
|
|
1811
|
+
return { kind: 'unavailable', reason: first?.reason ?? 'judge answer unparsable' };
|
|
1812
|
+
}
|
|
1813
|
+
const samples = parsed.map((a) => a.result.finalScore);
|
|
1814
|
+
const median = medianOf(samples);
|
|
1815
|
+
const result = parsed[samples.indexOf(median)]!.result;
|
|
1816
|
+
const issues = [...result.issues];
|
|
1817
|
+
let passed = median >= config.passThreshold;
|
|
1818
|
+
const contentHeight = input.contentHeight;
|
|
1819
|
+
const overflow = contentHeight !== null && contentHeight > viewport.height;
|
|
1820
|
+
const policy = canvasFitPolicy(canvas);
|
|
1821
|
+
if (overflow && contentHeight !== null && policy.overflow !== 'none') {
|
|
1822
|
+
issues.push(canvasOverflowIssue(canvas, viewport, contentHeight, policy.overflow));
|
|
1823
|
+
if (policy.overflow === 'fail') passed = false;
|
|
1824
|
+
}
|
|
1825
|
+
if (ink !== null && ink.lastInkRow === null) {
|
|
1826
|
+
issues.push(canvasBlankIssue(canvas, viewport));
|
|
1827
|
+
passed = false;
|
|
1828
|
+
}
|
|
1829
|
+
const criteria: CriteriaBlock | undefined =
|
|
1830
|
+
input.criteria !== undefined && criteriaContext !== undefined && criteriaSelected !== undefined
|
|
1831
|
+
? resolveCriteriaBlock({
|
|
1832
|
+
bank: input.criteria.bank,
|
|
1833
|
+
context: criteriaContext,
|
|
1834
|
+
selected: criteriaSelected,
|
|
1835
|
+
answers: criteriaReads(criteriaAnswer),
|
|
1836
|
+
measurements: { overflow, contentHeight, viewportHeight: viewport.height, inkRatio },
|
|
1837
|
+
...criteriaUnansweredReason(criteriaAnswer),
|
|
1838
|
+
...criteriaCallOf(criteriaAnswer),
|
|
1839
|
+
})
|
|
1840
|
+
: undefined;
|
|
1841
|
+
const criteriaTokens = criteriaCallTokens(criteriaAnswer);
|
|
1842
|
+
return {
|
|
1843
|
+
kind: 'ok',
|
|
1844
|
+
verdict: {
|
|
1845
|
+
canvas,
|
|
1846
|
+
viewport,
|
|
1847
|
+
score: median,
|
|
1848
|
+
passed,
|
|
1849
|
+
issues,
|
|
1850
|
+
judge: judgeRecordOf(k, parsed, samples, criteriaAnswer, model),
|
|
1851
|
+
contentHeight,
|
|
1852
|
+
overflow,
|
|
1853
|
+
inkRatio,
|
|
1854
|
+
...(criteria !== undefined ? { criteria } : {}),
|
|
1855
|
+
inputTokens: parsed.reduce((sum, a) => sum + a.inputTokens, 0),
|
|
1856
|
+
outputTokens: parsed.reduce((sum, a) => sum + a.outputTokens, 0),
|
|
1857
|
+
...(criteriaTokens !== undefined ? { criteriaTokens } : {}),
|
|
1858
|
+
},
|
|
1129
1859
|
};
|
|
1130
1860
|
}
|
|
1131
1861
|
|
|
@@ -1181,7 +1911,7 @@ export function salvageTruncatedVisualAnswer(text: string, passThreshold: number
|
|
|
1181
1911
|
hierarchy,
|
|
1182
1912
|
aesthetics,
|
|
1183
1913
|
issues,
|
|
1184
|
-
critique: `[
|
|
1914
|
+
critique: `[salvaged judge answer — it did not parse; score recovered from the closed prefix; ${issues.length} issue(s) kept, the rest and the critique lost]`,
|
|
1185
1915
|
});
|
|
1186
1916
|
return parseVisualResponse(rebuilt, passThreshold);
|
|
1187
1917
|
}
|
|
@@ -1217,22 +1947,57 @@ function parseVisualResponse(text: string, passThreshold: number): EvaluationRes
|
|
|
1217
1947
|
codeQuality: 0, // can't assess code from screenshot
|
|
1218
1948
|
};
|
|
1219
1949
|
|
|
1220
|
-
const issues:
|
|
1950
|
+
const issues: VisualIssue[] = (raw.issues || []).map((i: Record<string, string>) => ({
|
|
1221
1951
|
dimension: i.dimension || 'visual',
|
|
1952
|
+
origin: 'judge',
|
|
1222
1953
|
severity: (i.severity as 'critical' | 'major' | 'minor') || 'minor',
|
|
1223
1954
|
description: i.description || '',
|
|
1224
1955
|
fix: i.fix || '',
|
|
1225
1956
|
}));
|
|
1226
1957
|
|
|
1958
|
+
// ggui#1436 — the criteria answers, tolerant: absent block → no field (an old answer stays
|
|
1959
|
+
// valid); an entry without a string id or with a verdict outside the vocabulary is dropped.
|
|
1960
|
+
const criteria = parseCriteriaAnswers(raw.criteria);
|
|
1961
|
+
|
|
1227
1962
|
return {
|
|
1228
1963
|
passed: finalScore >= passThreshold,
|
|
1229
1964
|
finalScore,
|
|
1230
1965
|
dimensions,
|
|
1231
1966
|
issues,
|
|
1232
1967
|
critique: raw.critique,
|
|
1968
|
+
...(criteria.answers !== undefined ? { criteriaAnswers: criteria.answers } : {}),
|
|
1969
|
+
criteriaCounts: criteria.counts,
|
|
1233
1970
|
};
|
|
1234
1971
|
}
|
|
1235
1972
|
|
|
1973
|
+
/**
|
|
1974
|
+
* The criteria answers, tolerant (ggui#1436), and what the tolerance dropped (ggui#1687): an absent or non-array block
|
|
1975
|
+
* reads no row, and each dropped entry is counted by why, so an empty or cut call is visible rather than a row of
|
|
1976
|
+
* clean `n/a`s.
|
|
1977
|
+
*/
|
|
1978
|
+
function parseCriteriaAnswers(raw: unknown): { answers?: CriteriaAnswer[]; counts: CriteriaAnswerCounts } {
|
|
1979
|
+
if (!Array.isArray(raw)) return { counts: NO_CRITERIA_ARRAY };
|
|
1980
|
+
const answers: CriteriaAnswer[] = [];
|
|
1981
|
+
const dropped = { notObject: 0, badId: 0, badVerdict: 0 };
|
|
1982
|
+
for (const entry of raw) {
|
|
1983
|
+
if (typeof entry !== 'object' || entry === null) {
|
|
1984
|
+
dropped.notObject += 1;
|
|
1985
|
+
continue;
|
|
1986
|
+
}
|
|
1987
|
+
const o = entry as { id?: unknown; verdict?: unknown; evidence?: unknown };
|
|
1988
|
+
if (typeof o.id !== 'string') {
|
|
1989
|
+
dropped.badId += 1;
|
|
1990
|
+
continue;
|
|
1991
|
+
}
|
|
1992
|
+
if (o.verdict !== 'pass' && o.verdict !== 'fail' && o.verdict !== 'n/a') {
|
|
1993
|
+
dropped.badVerdict += 1;
|
|
1994
|
+
continue;
|
|
1995
|
+
}
|
|
1996
|
+
answers.push({ id: o.id, verdict: o.verdict, evidence: typeof o.evidence === 'string' ? o.evidence : '' });
|
|
1997
|
+
}
|
|
1998
|
+
return { answers, counts: { array: true, received: raw.length, kept: answers.length, dropped } };
|
|
1999
|
+
}
|
|
2000
|
+
|
|
1236
2001
|
// ---------------------------------------------------------------------------
|
|
1237
2002
|
// Default models for visual evaluation
|
|
1238
2003
|
// ---------------------------------------------------------------------------
|
|
@@ -1248,6 +2013,16 @@ function getDefaultVisualModel(provider: 'claude' | 'google'): string {
|
|
|
1248
2013
|
// Tier 2 adapter — returns EvalIssue[]
|
|
1249
2014
|
// ---------------------------------------------------------------------------
|
|
1250
2015
|
|
|
2016
|
+
/**
|
|
2017
|
+
* ggui#1522 — what a visual leg's judge calls spent: the scoring calls, and the report-only criteria calls apart
|
|
2018
|
+
* (ggui#1436). Priced at the VISUAL agent's model, never folded into a text evaluator's or a coding turn's tokens.
|
|
2019
|
+
*/
|
|
2020
|
+
export interface VisualLegTokens {
|
|
2021
|
+
readonly inputTokens: number;
|
|
2022
|
+
readonly outputTokens: number;
|
|
2023
|
+
readonly criteria?: CriteriaCallTokens;
|
|
2024
|
+
}
|
|
2025
|
+
|
|
1251
2026
|
/** What the harness's eval round consumes from the visual leg. */
|
|
1252
2027
|
export interface VisualEvalOutcome {
|
|
1253
2028
|
/** Tier-2 issues (empty when no browser is available). */
|
|
@@ -1264,6 +2039,11 @@ export interface VisualEvalOutcome {
|
|
|
1264
2039
|
* summary used to be indistinguishable from a clean single-shot run.
|
|
1265
2040
|
*/
|
|
1266
2041
|
coverage: VisualCoverage;
|
|
2042
|
+
/**
|
|
2043
|
+
* ggui#1522 — the leg's own spend, when its judge ran and reported tokens. The round records it in the cost
|
|
2044
|
+
* tracker at the visual agent's model and carries it apart; absent on a skipped leg.
|
|
2045
|
+
*/
|
|
2046
|
+
tokens?: VisualLegTokens;
|
|
1267
2047
|
}
|
|
1268
2048
|
|
|
1269
2049
|
/**
|
|
@@ -1287,7 +2067,16 @@ export async function runVisualEval(
|
|
|
1287
2067
|
const issues: EvalIssue[] = (result.issues || []).map(toEvalIssue);
|
|
1288
2068
|
const summary = summarizeVisualResult(result);
|
|
1289
2069
|
const coverage: VisualCoverage = { status: 'ran' };
|
|
1290
|
-
|
|
2070
|
+
// ggui#1522 — the judge's spend rides the outcome; until now it was dropped here, before the harness saw it.
|
|
2071
|
+
const tokens: VisualLegTokens | undefined =
|
|
2072
|
+
result.inputTokens !== undefined && result.outputTokens !== undefined
|
|
2073
|
+
? {
|
|
2074
|
+
inputTokens: result.inputTokens,
|
|
2075
|
+
outputTokens: result.outputTokens,
|
|
2076
|
+
...(result.criteriaTokens !== undefined ? { criteria: result.criteriaTokens } : {}),
|
|
2077
|
+
}
|
|
2078
|
+
: undefined;
|
|
2079
|
+
return { issues, ...(summary !== undefined ? { summary } : {}), coverage, ...(tokens !== undefined ? { tokens } : {}) };
|
|
1291
2080
|
}
|
|
1292
2081
|
|
|
1293
2082
|
/** A judge-shaped issue as the harness's tier-2 issue: critical ⇒ a blocking fail, anything else ⇒ a warn. */
|
|
@@ -1300,6 +2089,7 @@ function toEvalIssue(issue: EvaluationIssue): EvalIssue {
|
|
|
1300
2089
|
severity: issue.severity === 'critical' ? 'critical' : 'major',
|
|
1301
2090
|
description: issue.description,
|
|
1302
2091
|
fix: issue.fix || '',
|
|
2092
|
+
...(issue.origin !== undefined ? { origin: issue.origin } : {}),
|
|
1303
2093
|
};
|
|
1304
2094
|
}
|
|
1305
2095
|
|
|
@@ -1308,7 +2098,7 @@ function toEvalIssue(issue: EvaluationIssue): EvalIssue {
|
|
|
1308
2098
|
// ---------------------------------------------------------------------------
|
|
1309
2099
|
|
|
1310
2100
|
/** What the fit half reads — the frame's inputs; nothing about a judge. */
|
|
1311
|
-
export type VisualFitConfig = Pick<VisualEvalConfig, 'sampleProps' | 'canvases' | 'canvasViewports' | 'designSrcDir'>;
|
|
2101
|
+
export type VisualFitConfig = Pick<VisualEvalConfig, 'sampleProps' | 'canvases' | 'canvasViewports' | 'hostPresentations' | 'designSrcDir'>;
|
|
1312
2102
|
|
|
1313
2103
|
/** One canvas's fit reading: the box, the measure, whether it overflowed — no score. */
|
|
1314
2104
|
export interface CanvasFitReading {
|
|
@@ -1316,8 +2106,12 @@ export interface CanvasFitReading {
|
|
|
1316
2106
|
readonly viewport: CanvasViewport;
|
|
1317
2107
|
readonly contentHeight: number | null;
|
|
1318
2108
|
readonly overflow: boolean;
|
|
2109
|
+
/** ggui#1120 — the capture's ink ratio; `0` is the blank, `null` an unreadable capture. */
|
|
2110
|
+
readonly inkRatio: number | null;
|
|
1319
2111
|
/** `true` when the order declared this canvas's box. */
|
|
1320
2112
|
readonly declared?: true;
|
|
2113
|
+
/** ggui#1492 — what the judge did with this canvas's host presentation, when one was supplied. */
|
|
2114
|
+
readonly presentation?: CanvasPresentationOutcome;
|
|
1321
2115
|
}
|
|
1322
2116
|
|
|
1323
2117
|
/** `measured` = every canvas was framed (its fit issues, possibly none); `unavailable` = no verdict, with why. */
|
|
@@ -1361,7 +2155,7 @@ export async function runVisualFit(
|
|
|
1361
2155
|
const issues: EvalIssue[] = [];
|
|
1362
2156
|
const readings: CanvasFitReading[] = [];
|
|
1363
2157
|
for (const canvas of config.canvases) {
|
|
1364
|
-
const frame = await frameCanvas(bundledCode, context, html, canvas, config.canvasViewports, deps);
|
|
2158
|
+
const frame = await frameCanvas(bundledCode, context, html, canvas, config.canvasViewports, deps, config.hostPresentations);
|
|
1365
2159
|
if (!frame.attempt.png) {
|
|
1366
2160
|
return { status: 'unavailable', reason: `${frame.attempt.reason ?? 'no browser available'} at canvas ${canvas}` };
|
|
1367
2161
|
}
|
|
@@ -1369,16 +2163,24 @@ export async function runVisualFit(
|
|
|
1369
2163
|
const overflow = contentHeight !== null && contentHeight > frame.viewport.height;
|
|
1370
2164
|
const verdict = fitVerdict(canvas, frame);
|
|
1371
2165
|
if (verdict !== null) issues.push(toEvalIssue(canvasScopedIssue(canvas, verdict)));
|
|
2166
|
+
const overflowXIssue = overflowXVerdict(canvas, frame);
|
|
2167
|
+
if (overflowXIssue !== null) issues.push(toEvalIssue(canvasScopedIssue(canvas, overflowXIssue)));
|
|
2168
|
+
const blank = blankVerdict(canvas, frame);
|
|
2169
|
+
if (blank !== null) issues.push(toEvalIssue(canvasScopedIssue(canvas, blank)));
|
|
2170
|
+
const inkRatio = frame.ink?.ratio ?? null;
|
|
1372
2171
|
readings.push({
|
|
1373
2172
|
canvas,
|
|
1374
2173
|
viewport: frame.viewport,
|
|
1375
2174
|
contentHeight,
|
|
1376
2175
|
overflow,
|
|
2176
|
+
inkRatio,
|
|
1377
2177
|
...(frame.declared ? { declared: true as const } : {}),
|
|
2178
|
+
...(frame.presentation !== undefined ? { presentation: frame.presentation } : {}),
|
|
1378
2179
|
});
|
|
1379
2180
|
console.log(
|
|
1380
2181
|
`[visual-fit] canvas=${canvas} ${frame.viewport.width}×${frame.viewport.height} ` +
|
|
1381
|
-
`content=${contentHeight ?? '?'}px${overflow ? ` OVERFLOW (${frame.policy.overflow})` : ''}
|
|
2182
|
+
`content=${contentHeight ?? '?'}px${overflow ? ` OVERFLOW (${frame.policy.overflow})` : ''} ` +
|
|
2183
|
+
`ink=${inkRatio ?? '?'}${blank !== null ? ' BLANK' : ''} | no judge`,
|
|
1382
2184
|
);
|
|
1383
2185
|
}
|
|
1384
2186
|
return { status: 'measured', issues, readings };
|