@ggui-ai/ui-gen 0.24.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{adapter-Du7P_5Fx.d.ts → adapter-Cg_rkpGq.d.ts} +1 -1
- package/dist/adapters/generation-dispatch.d.ts +27 -5
- package/dist/adapters/generation-dispatch.js +909 -53
- package/dist/adapters/generation-dispatch.js.map +1 -1
- package/dist/adapters/index.js +3 -1
- package/dist/adapters/index.js.map +1 -1
- package/dist/advanced/index.d.ts +12 -10
- package/dist/advanced/index.js +917 -98
- package/dist/advanced/index.js.map +1 -1
- package/dist/{axes-CzLEMDeB.d.ts → axes-CKVCzR_x.d.ts} +1 -1
- package/dist/bank-CT4eEkta.d.ts +118 -0
- package/dist/boilerplate.js +24 -12
- package/dist/boilerplate.js.map +1 -1
- package/dist/check/index.d.ts +11 -4
- package/dist/check/index.js +47 -1
- package/dist/check/index.js.map +1 -1
- package/dist/classifier/index.d.ts +2 -2
- package/dist/compose.d.ts +2 -2
- package/dist/{create-ui-generator-GIhzx08x.d.ts → create-ui-generator-DFnnArrv.d.ts} +11 -3
- package/dist/{dispatch-BnAH56p4.d.ts → dispatch-CStl9eAw.d.ts} +5 -3
- package/dist/evaluation/axis-checks/index.d.ts +5 -5
- package/dist/evaluation/axis-checks/index.js +90 -2
- package/dist/evaluation/axis-checks/index.js.map +1 -1
- package/dist/evaluation/axis-checks/registry.d.ts +3 -3
- package/dist/evaluation/axis-checks/registry.js +88 -1
- package/dist/evaluation/axis-checks/registry.js.map +1 -1
- package/dist/evaluation/index.d.ts +300 -23
- package/dist/evaluation/index.js +746 -30
- package/dist/evaluation/index.js.map +1 -1
- package/dist/evaluation/types.d.ts +9 -1
- package/dist/evaluation/types.js.map +1 -1
- package/dist/fragments/index.d.ts +3 -3
- package/dist/harness/check/runtime-render/index.d.ts +5 -5
- package/dist/harness/index.d.ts +4 -4
- package/dist/harness/index.js +33 -15
- package/dist/harness/index.js.map +1 -1
- package/dist/harness/result-types.d.ts +3 -3
- package/dist/harness/runtime.d.ts +10 -4
- package/dist/harness/runtime.js +26 -14
- package/dist/harness/runtime.js.map +1 -1
- package/dist/harness/types-public.d.ts +3 -3
- package/dist/harness/types-public.js +33 -15
- package/dist/harness/types-public.js.map +1 -1
- package/dist/harness/validator-trace-sink.d.ts +3 -3
- package/dist/hash.d.ts +1 -1
- package/dist/index.d.ts +25 -9
- package/dist/index.js +926 -101
- package/dist/index.js.map +1 -1
- package/dist/{llm-router-DwwUK1bz.d.ts → llm-router-nfO7wPec.d.ts} +3 -1
- package/dist/patch.d.ts +3 -3
- package/dist/policy.d.ts +1 -1
- package/dist/tools/get-primitives-ts.d.ts +1 -1
- package/dist/tools/get-primitives-ts.js +1 -1
- package/dist/tools/get-primitives-ts.js.map +1 -1
- package/dist/{types-BOvHNG7K.d.ts → types-Dh1s4K9j.d.ts} +1 -1
- package/dist/{types-public-Bk6d1WLN.d.ts → types-public-BJTLpzWn.d.ts} +134 -7
- package/dist/validation/index.d.ts +1 -1
- package/dist/validation/index.js +1 -1
- package/dist/validation/index.js.map +1 -1
- package/dist/workflows.d.ts +3 -3
- package/package.json +12 -12
- package/src/adapters/generation-dispatch.ts +36 -2
- package/src/adapters/tools.ts +2 -1
- package/src/boilerplate/free-design-prompt.ts +1 -1
- package/src/boilerplate/generate.ts +26 -1
- package/src/boilerplate/hard-sections.ts +13 -13
- package/src/boilerplate/system-prompt.ts +1 -1
- package/src/check/index.ts +1 -0
- package/src/check/sample-props.ts +63 -0
- package/src/coding-agent/tools.ts +4 -1
- package/src/create-harness.ts +3 -3
- package/src/create-ui-generator.ts +20 -1
- package/src/evaluation/__fixtures__/png.ts +100 -0
- package/src/evaluation/axis-checks/checks/state-merge.ts +1 -1
- package/src/evaluation/axis-checks/checks/universal.ts +111 -0
- package/src/evaluation/axis-checks/dispatch.ts +4 -1
- package/src/evaluation/criteria/bank.ts +85 -0
- package/src/evaluation/criteria/context.ts +52 -0
- package/src/evaluation/criteria/resolve.ts +159 -0
- package/src/evaluation/criteria/select.ts +79 -0
- package/src/evaluation/index.ts +23 -0
- package/src/evaluation/ink-extent.ts +200 -0
- package/src/evaluation/theme-css.ts +12 -5
- package/src/evaluation/types-public.ts +130 -3
- package/src/evaluation/types.ts +9 -0
- package/src/evaluation/visual-evaluator.ts +636 -30
- package/src/harness/coding/generate-task-runner.ts +3 -0
- package/src/harness/coding/run-coding-turn.ts +7 -3
- package/src/harness/coding/run-eval-round.ts +19 -0
- package/src/harness/runtime.ts +5 -0
- package/src/harness-tool-names.ts +38 -0
- package/src/index.ts +12 -0
- package/src/run-check.ts +2 -0
- package/src/tools/get-primitives-ts.ts +1 -1
- package/src/tools/get-wire.ts +1 -1
- package/src/tools/render-check.ts +0 -60
- package/src/tools.ts +11 -1
- package/src/validation/primitives.ts +1 -1
|
@@ -13,8 +13,16 @@
|
|
|
13
13
|
|
|
14
14
|
import { buildStylingProfileJudgeBlock } from '../boilerplate/styling-profile.js';
|
|
15
15
|
import type { GenerationProfileInput } from '../boilerplate/styling-profile.js';
|
|
16
|
+
import type { CriteriaAnswer } from './types.js';
|
|
17
|
+
import type { CriteriaBlock, CriteriaContext, CriteriaRollup } from './types-public.js';
|
|
18
|
+
import type { CriteriaBank } from './criteria/bank.js';
|
|
19
|
+
import type { JsonObject } from '@ggui-ai/protocol';
|
|
20
|
+
import type { CriteriaContextInput } from './criteria/context.js';
|
|
21
|
+
import { selectCriteria } from './criteria/select.js';
|
|
22
|
+
import { buildCriteriaJudgeBlock, resolveCriteriaBlock } from './criteria/resolve.js';
|
|
16
23
|
import { build } from 'esbuild';
|
|
17
|
-
import { fillFitRule, getCssTokens } from '@ggui-ai/design/rendering';
|
|
24
|
+
import { EXPANDED_FRAME, expandedFramePanelRule, expandedFrameScrimDecls, fillFitRule, getCssTokens } from '@ggui-ai/design/rendering';
|
|
25
|
+
import { readInkExtent, type InkExtent } from './ink-extent.js';
|
|
18
26
|
import { judgeDesignIdentity, type JudgeDesignIdentity } from './design-identity.js';
|
|
19
27
|
import { resolve, dirname } from 'path';
|
|
20
28
|
import { fileURLToPath } from 'url';
|
|
@@ -24,7 +32,7 @@ import { createHash } from 'node:crypto';
|
|
|
24
32
|
import { tmpdir } from 'os';
|
|
25
33
|
import { createVisionAgent, type AgentConfig } from '../harness/llm-router';
|
|
26
34
|
import type { EvaluationResult, EvaluationIssue, DimensionScores } from './types';
|
|
27
|
-
import type { CanvasJudgeRecord, CanvasVisualSummary, EvalIssue, VisualCoverage, VisualEvalSummary, VisualFitStamp } from './types-public.js';
|
|
35
|
+
import type { CanvasHostPresentation, CanvasJudgeRecord, CanvasPresentationOutcome, CanvasVisualSummary, EvalIssue, HostInlineFrame, HostPresentationIgnoredReason, VisualCoverage, VisualEvalSummary, VisualFitStamp } from './types-public.js';
|
|
28
36
|
import type { LaunchOptions } from 'puppeteer-core';
|
|
29
37
|
import { CANVAS_VIEWPORTS, displayModeForCanvas, type CanvasClass, type CanvasViewport } from '../design-mode.js';
|
|
30
38
|
|
|
@@ -69,6 +77,14 @@ export interface VisualEvalConfig {
|
|
|
69
77
|
* `viewport` all carry that same integer box.
|
|
70
78
|
*/
|
|
71
79
|
canvasViewports?: Partial<Readonly<Record<CanvasClass, CanvasViewport>>>;
|
|
80
|
+
/**
|
|
81
|
+
* ggui#1492 — how a host presents each canvas, as data the caller supplies (a lane that knows its host). Keyed by
|
|
82
|
+
* canvas; a canvas without an entry is judged exactly as before (the generic stand-in round the inline card, the
|
|
83
|
+
* scrim under a panel). The box stays in {@link canvasViewports}. An entry this judge cannot use (a colour that is
|
|
84
|
+
* not `#rrggbb`, a number out of range, a frame on a canvas that is not the inline card) is ignored for its canvas,
|
|
85
|
+
* said once in the log, never partly applied.
|
|
86
|
+
*/
|
|
87
|
+
hostPresentations?: Partial<Readonly<Record<CanvasClass, CanvasHostPresentation>>>;
|
|
72
88
|
/**
|
|
73
89
|
* Optional provider-routing override — see
|
|
74
90
|
* `AgentConfig.routeOverride`. Threaded onto the agent this
|
|
@@ -102,6 +118,15 @@ export interface VisualEvalConfig {
|
|
|
102
118
|
onRetry?: AgentConfig['onRetry'];
|
|
103
119
|
}
|
|
104
120
|
|
|
121
|
+
/**
|
|
122
|
+
* ggui#1436 — what the report-only criteria call spent. Kept APART from the scoring calls' `inputTokens` /
|
|
123
|
+
* `outputTokens`: those price the judgement a caller may meter, and the criteria call is a measurement beside it.
|
|
124
|
+
*/
|
|
125
|
+
export interface CriteriaCallTokens {
|
|
126
|
+
readonly inputTokens: number;
|
|
127
|
+
readonly outputTokens: number;
|
|
128
|
+
}
|
|
129
|
+
|
|
105
130
|
/** One canvas's verdict — screenshot + judge score at that class's viewport. */
|
|
106
131
|
export interface CanvasVisualResult {
|
|
107
132
|
canvas: CanvasClass;
|
|
@@ -121,35 +146,56 @@ export interface CanvasVisualResult {
|
|
|
121
146
|
/** `contentHeight > viewport.height` — a measurement; what it MEANS is {@link canvasFitPolicy}'s. */
|
|
122
147
|
overflow: boolean;
|
|
123
148
|
/** How `score` was reached — always present (ggui#1072). */
|
|
149
|
+
/**
|
|
150
|
+
* ggui#1120 — how far down the capture anything was painted (the last inked row over the measured
|
|
151
|
+
* region's height, read against the capture's dominant colour, inside a drawn panel's chrome); `0`
|
|
152
|
+
* is the blank the judge fails, `null` an unreadable capture.
|
|
153
|
+
*/
|
|
154
|
+
inkRatio: number | null;
|
|
124
155
|
judge: CanvasJudgeRecord;
|
|
125
156
|
/** How the judge composed the mount (ggui#1100): `'fill'` on every fullscreen canvas, absent on the inline card. */
|
|
126
157
|
fit?: 'fill';
|
|
127
158
|
/** ggui#1195 — `true` when the order DECLARED this canvas's box (`config.canvasViewports` carried an entry), whatever its value. */
|
|
128
159
|
declared?: true;
|
|
160
|
+
/** ggui#1436 — the typed criteria block, when a bank was configured. */
|
|
161
|
+
criteria?: CriteriaBlock;
|
|
162
|
+
/** ggui#1436 — the criteria call's own spend, when one was asked and answered; never inside the canvas's scoring tokens. */
|
|
163
|
+
criteriaTokens?: CriteriaCallTokens;
|
|
164
|
+
/** ggui#1492 — what the judge did with this canvas's host presentation (drawn, or ignored and why), when one was supplied. */
|
|
165
|
+
presentation?: CanvasPresentationOutcome;
|
|
129
166
|
}
|
|
130
167
|
|
|
131
168
|
/** How a canvas is captured for the judge and what an overflow means there (ggui#1027). */
|
|
132
|
-
export type CaptureMode = 'viewport' | 'full-page';
|
|
169
|
+
export type CaptureMode = 'viewport' | 'full-page' | 'natural';
|
|
133
170
|
export interface CanvasFitPolicy {
|
|
134
|
-
/**
|
|
171
|
+
/**
|
|
172
|
+
* `viewport` = the whole box; `full-page` = the whole scrolled document; `natural` (ggui#1475) = the
|
|
173
|
+
* card at its OWN height, clipped to its extent and capped at the box — the inline card a host sizes
|
|
174
|
+
* to its content, on the host's ground ({@link JUDGE_GROUND_MARGIN_PX}).
|
|
175
|
+
*/
|
|
135
176
|
readonly capture: CaptureMode;
|
|
136
177
|
/** The verdict an overflow earns on this canvas. */
|
|
137
178
|
readonly overflow: 'fail' | 'warn' | 'none';
|
|
138
179
|
}
|
|
139
180
|
|
|
140
181
|
/**
|
|
141
|
-
* The inline chat card
|
|
142
|
-
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
*
|
|
182
|
+
* The inline chat card takes its NATURAL height under a ceiling (ggui#1475):
|
|
183
|
+
* an MCP Apps host sizes the inline frame to the card's content
|
|
184
|
+
* (`ui/notifications/size-changed`, auto-resize) up to its maximum, so the
|
|
185
|
+
* visitor sees the card and nothing under it. The judge captures exactly
|
|
186
|
+
* that — the card's own extent on the host's ground, capped at the box — and
|
|
187
|
+
* never the box's remainder (a fixed 640 px capture scored every hello
|
|
188
|
+
* against a void no visitor sees). The box stays the ceiling: content taller
|
|
189
|
+
* than it is a critical layout issue that fails the canvas (a 1289 px hello
|
|
190
|
+
* on a 640 px card was scored 86 from a full-page capture that showed the
|
|
191
|
+
* judge what no user sees). A phone's first screen may scroll, so the
|
|
192
|
+
* overflow is reported as a major issue and the score stands. Pages
|
|
193
|
+
* (`md`/`lg`/`xl`) scroll by design: measured, never judged.
|
|
148
194
|
*/
|
|
149
195
|
export function canvasFitPolicy(canvas: CanvasClass): CanvasFitPolicy {
|
|
150
196
|
switch (canvas) {
|
|
151
197
|
case 'xs-chat-card':
|
|
152
|
-
return { capture: '
|
|
198
|
+
return { capture: 'natural', overflow: 'fail' };
|
|
153
199
|
case 'mobile-fullscreen-small':
|
|
154
200
|
return { capture: 'full-page', overflow: 'warn' };
|
|
155
201
|
default:
|
|
@@ -162,6 +208,15 @@ export function canvasFitPolicy(canvas: CanvasClass): CanvasFitPolicy {
|
|
|
162
208
|
* `fillFitRule` targets — the judge's counterpart of the runtime's mount list.
|
|
163
209
|
*/
|
|
164
210
|
export const JUDGE_SCOPE_CLASS = 'ggui-judge-scope';
|
|
211
|
+
/** The panel the judge draws round a fill canvas's scope on its stand-in host page (ggui#1083 cut 3) — see {@link canvasChrome}. */
|
|
212
|
+
export const JUDGE_PANEL_CLASS = 'ggui-judge-panel';
|
|
213
|
+
/**
|
|
214
|
+
* How far in from a panelled page's edges the ink is read (ggui#1120): past the panel's gap, its
|
|
215
|
+
* largest rounded corner (the `round` bucket, 24 px — the scrim shows through the corner) and its
|
|
216
|
+
* 1 px hairline, so the host's chrome never counts as paint. The card's own inset is 16 px, so the
|
|
217
|
+
* band skipped inside the card is 25 px of a canvas — reported ratios are over the region.
|
|
218
|
+
*/
|
|
219
|
+
export const JUDGE_INK_INSET_PX = EXPANDED_FRAME.insetPx + 24 + 1;
|
|
165
220
|
|
|
166
221
|
/**
|
|
167
222
|
* How the judge composes a canvas (ggui#1100): the served runtime stretches
|
|
@@ -173,6 +228,115 @@ export const JUDGE_SCOPE_CLASS = 'ggui-judge-scope';
|
|
|
173
228
|
export function canvasFit(canvas: CanvasClass): 'fill' | undefined {
|
|
174
229
|
return displayModeForCanvas(canvas) === 'fullscreen' ? 'fill' : undefined;
|
|
175
230
|
}
|
|
231
|
+
/**
|
|
232
|
+
* What the judge's stand-in host draws round a canvas: the embedding shell's floating panel on its
|
|
233
|
+
* scrim (a fill canvas, md and up), the host's ground round the inline card (`ground`, ggui#1475), or
|
|
234
|
+
* nothing (the phone, edge to edge).
|
|
235
|
+
*/
|
|
236
|
+
export type CanvasChrome = 'panel' | 'ground' | undefined;
|
|
237
|
+
/**
|
|
238
|
+
* The host ground round a naturally-captured inline card (ggui#1475): the card is mounted this far in
|
|
239
|
+
* from the page's edges and captured with this margin, so its edge and radius are read against the
|
|
240
|
+
* host surface — the page's scrim ground, never the card's own. The design's one gap (the panel's).
|
|
241
|
+
*/
|
|
242
|
+
export const JUDGE_GROUND_MARGIN_PX = EXPANDED_FRAME.insetPx;
|
|
243
|
+
/** The class the judge page wraps a naturally-captured card's mount in — its padding is the ground margin. */
|
|
244
|
+
export const JUDGE_GROUND_CLASS = 'ggui-judge-ground';
|
|
245
|
+
/**
|
|
246
|
+
* The host's frame round the inline card (ggui#1475). An inline card draws no chrome of its own (the
|
|
247
|
+
* chat layout tells it the host draws it), so without a frame it sits transparent on the page ground
|
|
248
|
+
* and has no edge to read. The judge draws a GENERIC stand-in for the host's frame, built from the
|
|
249
|
+
* card's own theme: its container surface, a 1 px ring of its ink at 8 %, a 16 px radius, and
|
|
250
|
+
* overflow hidden. It is a stand-in, and the frame line says so: a host's exact frame arrives as data.
|
|
251
|
+
*/
|
|
252
|
+
export const JUDGE_INLINE_FRAME_CLASS = 'ggui-judge-inline-frame';
|
|
253
|
+
export const JUDGE_INLINE_FRAME_BORDER_PX = 1;
|
|
254
|
+
export const JUDGE_INLINE_FRAME_RADIUS_PX = 16;
|
|
255
|
+
/** What the inline card's page adds on each side of the card: the ground margin plus the frame's border. */
|
|
256
|
+
export const JUDGE_INLINE_PAD_PX = JUDGE_GROUND_MARGIN_PX + JUDGE_INLINE_FRAME_BORDER_PX;
|
|
257
|
+
/** How far in the inline card's ink is read: past the margin, the frame's ring and its rounded corner, so the host's frame never counts as paint. */
|
|
258
|
+
export const JUDGE_INLINE_INK_INSET_PX = JUDGE_INLINE_PAD_PX + JUDGE_INLINE_FRAME_RADIUS_PX;
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* The bounds a host frame must meet to be drawn (ggui#1492), exported so a lane's reader applies exactly the frames
|
|
262
|
+
* this judge draws: an entry the reader accepts and the judge ignores would be echoed as applied while never drawn.
|
|
263
|
+
*/
|
|
264
|
+
export const HOST_COLOUR_PATTERN = /^#[0-9a-f]{6}$/;
|
|
265
|
+
export const HOST_FRAME_BOUNDS = {
|
|
266
|
+
ringWidthPx: { min: 0, max: 8 },
|
|
267
|
+
ringAlpha: { min: 0, max: 1 },
|
|
268
|
+
radiusPx: { min: 0, max: 64 },
|
|
269
|
+
minHeightPx: { min: 0, max: 2560 },
|
|
270
|
+
} as const;
|
|
271
|
+
/**
|
|
272
|
+
* ggui#1492 — what this judge will do with a canvas's host presentation: draw it, or ignore it with the lane
|
|
273
|
+
* reader's own reason code. `undefined` when none was supplied. Every string drawn reaches the page's CSS, so a
|
|
274
|
+
* colour must be `#rrggbb` and a number must sit inside {@link HOST_FRAME_BOUNDS}; an inline card's room lives on
|
|
275
|
+
* its frame, so a top-level `ground` there is refused rather than silently unused.
|
|
276
|
+
*/
|
|
277
|
+
function resolvePresentation(canvas: CanvasClass, p: CanvasHostPresentation | undefined): CanvasPresentationOutcome | undefined {
|
|
278
|
+
if (p === undefined) return undefined;
|
|
279
|
+
const ignored = (reason: HostPresentationIgnoredReason, why: string): CanvasPresentationOutcome => {
|
|
280
|
+
console.warn(`[visual-eval] host presentation for ${canvas} ignored (${reason}): ${why}`);
|
|
281
|
+
return { status: 'ignored', reason };
|
|
282
|
+
};
|
|
283
|
+
const inline = canvasFitPolicy(canvas).capture === 'natural';
|
|
284
|
+
if (typeof p.label !== 'string' || p.label.length === 0) return ignored('label_missing', 'no label');
|
|
285
|
+
if (p.ground !== undefined && !HOST_COLOUR_PATTERN.test(p.ground)) return ignored('color_not_hex', 'ground is not #rrggbb');
|
|
286
|
+
if (inline && p.ground !== undefined) return ignored('ground_on_inline', "the inline card's room is its frame's ground");
|
|
287
|
+
const f = p.frame;
|
|
288
|
+
if (f !== undefined) {
|
|
289
|
+
if (!inline) return ignored('frame_off_inline', 'a frame on a canvas that is not the inline card');
|
|
290
|
+
if (![f.surface, f.ground, f.ring.color].every((c) => HOST_COLOUR_PATTERN.test(c))) return ignored('color_not_hex', 'a frame colour is not #rrggbb');
|
|
291
|
+
const B = HOST_FRAME_BOUNDS;
|
|
292
|
+
const inRange = (n: number, b: { readonly min: number; readonly max: number }): boolean => Number.isFinite(n) && n >= b.min && n <= b.max;
|
|
293
|
+
if (!inRange(f.ring.widthPx, B.ringWidthPx) || !inRange(f.ring.alpha, B.ringAlpha) || !inRange(f.radiusPx, B.radiusPx)) return ignored('frame_malformed', 'a frame number out of range');
|
|
294
|
+
if (f.minHeightPx !== undefined && !inRange(f.minHeightPx, B.minHeightPx)) return ignored('frame_malformed', 'minHeightPx out of range');
|
|
295
|
+
}
|
|
296
|
+
return { status: 'applied', applied: p };
|
|
297
|
+
}
|
|
298
|
+
/** What the inline card's page adds per side, where its ink is read from, and the host's floor — from the host frame when given, else the stand-in's. */
|
|
299
|
+
interface InlineGeometry {
|
|
300
|
+
readonly padPx: number;
|
|
301
|
+
readonly inkInsetPx: number;
|
|
302
|
+
readonly floorPx: number | undefined;
|
|
303
|
+
}
|
|
304
|
+
function inlineGeometry(frame: HostInlineFrame | undefined): InlineGeometry {
|
|
305
|
+
if (frame === undefined) return { padPx: JUDGE_INLINE_PAD_PX, inkInsetPx: JUDGE_INLINE_INK_INSET_PX, floorPx: undefined };
|
|
306
|
+
const padPx = JUDGE_GROUND_MARGIN_PX + frame.ring.widthPx;
|
|
307
|
+
return { padPx, inkInsetPx: padPx + frame.radiusPx, floorPx: frame.minHeightPx };
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* The host stand-in round a fill canvas (ggui#1083 cut 3, ggui#1067 §5): the embedding shell floats
|
|
311
|
+
* the card in a panel on its scrim — a hairline, the theme's `xl` radius, `shadow-sm`, a 16 px gap —
|
|
312
|
+
* only when expanded at ≥ 601 px; below that the card owns the phone edge to edge. So the panel
|
|
313
|
+
* canvases are the fill canvases from `md` up (the shell's panel reaches `lg` / `xl` on wide windows)
|
|
314
|
+
* and `mobile-fullscreen-small` carries no chrome. The panel is the design's ONE definition
|
|
315
|
+
* (`expandedFramePanelRule`), never a second drawing, and the judge pads its window by the panel's
|
|
316
|
+
* gap so the CARD's box stays the canvas box ({@link judgeWindow}): the frame the founder approves
|
|
317
|
+
* carries the chrome the visitor sees, and the card is measured where it was before.
|
|
318
|
+
*/
|
|
319
|
+
export function canvasChrome(canvas: CanvasClass): CanvasChrome {
|
|
320
|
+
if (canvasFitPolicy(canvas).capture === 'natural') return 'ground';
|
|
321
|
+
return canvasFit(canvas) === 'fill' && canvas !== 'mobile-fullscreen-small' ? 'panel' : undefined;
|
|
322
|
+
}
|
|
323
|
+
/** The browser window for a canvas: the canvas box, plus the chrome's gap on every side when the judge draws one (the panel's, or the host ground's). */
|
|
324
|
+
export function judgeWindow(viewport: CanvasViewport, chrome: CanvasChrome, inlinePadPx: number = JUDGE_INLINE_PAD_PX): CanvasViewport {
|
|
325
|
+
if (chrome === undefined) return viewport;
|
|
326
|
+
const gap = 2 * (chrome === 'ground' ? inlinePadPx : EXPANDED_FRAME.insetPx);
|
|
327
|
+
return { width: viewport.width + gap, height: viewport.height + gap };
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* The card's height on a panelled page: the panel grows with the card and carries the gap above and
|
|
331
|
+
* below it. A document shorter than the gap laid out no panel (the panel alone is a viewport tall), so
|
|
332
|
+
* there is no reading — `null`, never a negative height.
|
|
333
|
+
*/
|
|
334
|
+
function cardHeight(documentHeight: number | null, chrome: CanvasChrome): number | null {
|
|
335
|
+
// A natural capture measures the card itself (the mount's extent), never the document.
|
|
336
|
+
if (documentHeight === null || chrome === undefined || chrome === 'ground') return documentHeight;
|
|
337
|
+
const gap = 2 * EXPANDED_FRAME.insetPx;
|
|
338
|
+
return documentHeight < gap ? null : documentHeight - gap;
|
|
339
|
+
}
|
|
176
340
|
|
|
177
341
|
/**
|
|
178
342
|
* The box a browser can be asked for: whole CSS px. CDP's
|
|
@@ -218,6 +382,41 @@ export function canvasOverflowIssue(
|
|
|
218
382
|
};
|
|
219
383
|
}
|
|
220
384
|
|
|
385
|
+
/** ggui#1475 — the horizontal twin of {@link canvasOverflowIssue}: content wider than the card, which its frame clips out of the capture. */
|
|
386
|
+
export function canvasOverflowXIssue(
|
|
387
|
+
canvas: CanvasClass,
|
|
388
|
+
viewport: CanvasViewport,
|
|
389
|
+
hiddenPx: number,
|
|
390
|
+
verdict: 'fail' | 'warn',
|
|
391
|
+
): EvaluationIssue {
|
|
392
|
+
return {
|
|
393
|
+
dimension: 'canvas-overflow-x',
|
|
394
|
+
severity: verdict === 'fail' ? 'critical' : 'major',
|
|
395
|
+
description:
|
|
396
|
+
`Content runs ${hiddenPx}px past the ${canvas} card's ${viewport.width}px width — the card's frame clips it, ` +
|
|
397
|
+
'so a visitor sees a cut-off row or a sideways scroll.',
|
|
398
|
+
fix:
|
|
399
|
+
'Keep every row within the card: wrap long text (overflow-wrap: anywhere), let a wide table or code block scroll ' +
|
|
400
|
+
'inside its own container, and give no child a fixed width wider than the card.',
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/** ggui#1120 — the deterministic blank: the capture is one flat colour, so the component mounted and painted nothing. */
|
|
405
|
+
export function canvasBlankIssue(canvas: CanvasClass, viewport: CanvasViewport): EvaluationIssue {
|
|
406
|
+
const classBox = CANVAS_VIEWPORTS[canvas];
|
|
407
|
+
const box = isDeclaredViewport(canvas, viewport)
|
|
408
|
+
? `declared ${viewport.width}×${viewport.height}; class box ${classBox.width}×${classBox.height}`
|
|
409
|
+
: `${viewport.width}×${viewport.height}`;
|
|
410
|
+
return {
|
|
411
|
+
dimension: 'canvas-blank',
|
|
412
|
+
severity: 'critical',
|
|
413
|
+
description: `Nothing painted on the ${canvas} canvas (${box}): the capture is one flat colour — the component mounted and rendered no pixels.`,
|
|
414
|
+
fix:
|
|
415
|
+
'Return and paint the component tree: a root that renders nothing is a blank, not an empty state. Check that the ' +
|
|
416
|
+
'default export returns its JSX, that no early return yields null on the sample props, and that the first paint ' +
|
|
417
|
+
'shows visible content.',
|
|
418
|
+
};
|
|
419
|
+
}
|
|
221
420
|
/**
|
|
222
421
|
* `runVisualEvaluation`'s result. Identical to `EvaluationResult` when
|
|
223
422
|
* `config.canvases` is unset; with canvases set, `canvases` holds the
|
|
@@ -226,6 +425,11 @@ export function canvasOverflowIssue(
|
|
|
226
425
|
*/
|
|
227
426
|
export interface VisualEvaluationResult extends EvaluationResult {
|
|
228
427
|
canvases?: CanvasVisualResult[];
|
|
428
|
+
/**
|
|
429
|
+
* ggui#1436 — the criteria calls' spend across canvases, when any was asked and answered. `inputTokens` /
|
|
430
|
+
* `outputTokens` stay the SCORING calls' alone, so a cost read off them never includes the report-only block.
|
|
431
|
+
*/
|
|
432
|
+
criteriaTokens?: CriteriaCallTokens;
|
|
229
433
|
/** The design tree the judge painted with (ggui#1042) — present on every per-canvas judgement. */
|
|
230
434
|
design?: JudgeDesignIdentity;
|
|
231
435
|
/** The mode `cssTokens` were composed in, when the caller said (ggui#1076). */
|
|
@@ -247,6 +451,8 @@ export interface VisualEvalContext {
|
|
|
247
451
|
cssTokens?: string;
|
|
248
452
|
/** The app's generation profile (#991) — judged relative to, never against. */
|
|
249
453
|
profile?: GenerationProfileInput;
|
|
454
|
+
/** ggui#1436 — a criteria bank and the card's context; the judge selects per canvas and answers report-only. */
|
|
455
|
+
criteria?: { readonly bank: CriteriaBank; readonly context: CriteriaContextInput };
|
|
250
456
|
}
|
|
251
457
|
|
|
252
458
|
// ---------------------------------------------------------------------------
|
|
@@ -447,11 +653,24 @@ try {
|
|
|
447
653
|
}
|
|
448
654
|
}
|
|
449
655
|
|
|
656
|
+
/** The inline card's frame: the host's, as data, when given; else the generic stand-in built from the card's own tokens. */
|
|
657
|
+
function inlineFrameDecls(frame: HostInlineFrame | undefined): string {
|
|
658
|
+
if (frame === undefined) {
|
|
659
|
+
return `background: var(--ggui-color-container); border: ${JUDGE_INLINE_FRAME_BORDER_PX}px solid color-mix(in srgb, var(--ggui-color-onContainer) 8%, transparent); border-radius: ${JUDGE_INLINE_FRAME_RADIUS_PX}px; overflow: hidden;`;
|
|
660
|
+
}
|
|
661
|
+
const alphaPct = Math.round(frame.ring.alpha * 1000) / 10;
|
|
662
|
+
return `background: ${frame.surface}; border: ${frame.ring.widthPx}px solid color-mix(in srgb, ${frame.ring.color} ${alphaPct}%, transparent); border-radius: ${frame.radiusPx}px; overflow: hidden;`;
|
|
663
|
+
}
|
|
664
|
+
|
|
450
665
|
function buildRenderHTML(
|
|
451
666
|
bundledCode: string,
|
|
452
667
|
cssTokens?: string,
|
|
453
668
|
fit?: 'fill',
|
|
669
|
+
chrome?: CanvasChrome,
|
|
670
|
+
presentation?: CanvasHostPresentation,
|
|
454
671
|
): string {
|
|
672
|
+
if (chrome === 'panel' && fit !== 'fill') throw new Error("the judge's panel frames a fill canvas only (ggui#1083 cut 3)");
|
|
673
|
+
if (chrome === 'ground' && fit === 'fill') throw new Error("the host ground frames an inline card only — a fill canvas has no natural height (ggui#1475)");
|
|
455
674
|
// Default to the design tokens production's no-theme branch injects
|
|
456
675
|
// (getCssTokens → default theme, light) — ggui#613: under the s4
|
|
457
676
|
// fallback ban the generated component's `var(--ggui-*)` references
|
|
@@ -473,15 +692,16 @@ function buildRenderHTML(
|
|
|
473
692
|
/* ggui#1083 — the host stand-in (neutral-50) under the theme's scrim (its tint at
|
|
474
693
|
its opacity): the ground a frosted host puts under the card, so the judge scores
|
|
475
694
|
the card where the visitor sees it. */
|
|
476
|
-
background:
|
|
695
|
+
${presentation?.frame !== undefined ? `background: ${presentation.frame.ground};` : presentation?.ground !== undefined ? `background: ${presentation.ground};` : expandedFrameScrimDecls()}
|
|
477
696
|
color: var(--ggui-color-neutral-900, #111827);
|
|
478
697
|
}
|
|
479
698
|
.error { color: #dc2626; padding: 16px; font-family: monospace; white-space: pre-wrap; }
|
|
480
|
-
${fit === 'fill' ? fillFitRule(JUDGE_SCOPE_CLASS) : ''}
|
|
699
|
+
${fit === 'fill' ? fillFitRule(JUDGE_SCOPE_CLASS) + (chrome === 'panel' ? expandedFramePanelRule(JUDGE_PANEL_CLASS, JUDGE_SCOPE_CLASS) : '') : ''}
|
|
700
|
+
${chrome === 'ground' ? `.${JUDGE_GROUND_CLASS} { padding: ${JUDGE_GROUND_MARGIN_PX}px; } .${JUDGE_INLINE_FRAME_CLASS} { ${inlineFrameDecls(presentation?.frame)} }` : ''}
|
|
481
701
|
</style>
|
|
482
702
|
</head>
|
|
483
703
|
<body>
|
|
484
|
-
<div id="root"${fit === 'fill' ? ` class="${JUDGE_SCOPE_CLASS}"` : ''}></div>
|
|
704
|
+
${chrome === 'panel' ? `<div class="${JUDGE_PANEL_CLASS}">` : chrome === 'ground' ? `<div class="${JUDGE_GROUND_CLASS}"><div class="${JUDGE_INLINE_FRAME_CLASS}">` : ''}<div id="root"${fit === 'fill' ? ` class="${JUDGE_SCOPE_CLASS}"` : ''}></div>${chrome === 'panel' ? '</div>' : chrome === 'ground' ? '</div></div>' : ''}
|
|
485
705
|
<script type="importmap">
|
|
486
706
|
{
|
|
487
707
|
"imports": {
|
|
@@ -503,6 +723,14 @@ ${bundledCode}
|
|
|
503
723
|
// Screenshot capture (optional puppeteer dependency)
|
|
504
724
|
// ---------------------------------------------------------------------------
|
|
505
725
|
|
|
726
|
+
/** A screenshot's region in page px (puppeteer's `clip`). */
|
|
727
|
+
export interface ScreenshotClip {
|
|
728
|
+
readonly x: number;
|
|
729
|
+
readonly y: number;
|
|
730
|
+
readonly width: number;
|
|
731
|
+
readonly height: number;
|
|
732
|
+
}
|
|
733
|
+
|
|
506
734
|
/** The page surface the capture uses — a structural subset of puppeteer's `Page`. */
|
|
507
735
|
export interface ScreenshotPage {
|
|
508
736
|
setContent(html: string, options: { waitUntil: 'load'; timeout: number }): Promise<void>;
|
|
@@ -510,7 +738,7 @@ export interface ScreenshotPage {
|
|
|
510
738
|
waitForSelector(selector: string, options: { timeout: number }): Promise<unknown>;
|
|
511
739
|
/** Evaluates a JS expression in the page and returns its serialised value — the fit measurement (ggui#1027). */
|
|
512
740
|
evaluate(expression: string): Promise<unknown>;
|
|
513
|
-
screenshot(options: { type: 'png'; fullPage: boolean }): Promise<Uint8Array>;
|
|
741
|
+
screenshot(options: { type: 'png'; fullPage: boolean; clip?: ScreenshotClip }): Promise<Uint8Array>;
|
|
514
742
|
}
|
|
515
743
|
|
|
516
744
|
/** The browser surface the capture uses — a structural subset of puppeteer's `Browser`. */
|
|
@@ -603,16 +831,58 @@ export interface ScreenshotAttempt {
|
|
|
603
831
|
readonly reason?: string;
|
|
604
832
|
/** The document's scroll height at the viewport (ggui#1027); `null` when unmeasurable. */
|
|
605
833
|
readonly contentHeight: number | null;
|
|
834
|
+
/** ggui#1475 — a natural capture's horizontal overflow: how many px the card's content runs past its width; absent on every other capture. */
|
|
835
|
+
readonly overflowX?: number | null;
|
|
606
836
|
}
|
|
607
837
|
|
|
608
838
|
/** The expression the fit measurement evaluates in the page — the taller of the two scroll heights. */
|
|
609
839
|
export const CONTENT_HEIGHT_EXPRESSION =
|
|
610
840
|
'Math.max(document.documentElement.scrollHeight, document.body ? document.body.scrollHeight : 0)';
|
|
611
841
|
|
|
612
|
-
/**
|
|
613
|
-
|
|
842
|
+
/**
|
|
843
|
+
* ggui#1475 — the natural capture's measurement: the card's height as the host is told it. The MCP
|
|
844
|
+
* Apps SDK's size-changed notification (the report an auto-resizing host sizes the inline frame by)
|
|
845
|
+
* reads the document at `height: max-content` and restores the style; the judge reads the same
|
|
846
|
+
* quantity, minus what its page adds on both sides (the ground margin and the host frame's border), so content outside the mount (a
|
|
847
|
+
* portal into body) counts here exactly as it counts for the host. The document's scroll height
|
|
848
|
+
* could not be used: it is floored at the viewport by definition, so it sees an overflow but never
|
|
849
|
+
* a card shorter than its box.
|
|
850
|
+
*/
|
|
851
|
+
export function cardHeightExpression(padPx: number): string {
|
|
852
|
+
return (
|
|
853
|
+
"(() => { const el = document.documentElement; const prev = el.style.height; el.style.height = 'max-content'; " +
|
|
854
|
+
`const h = Math.ceil(el.getBoundingClientRect().height); el.style.height = prev; return h - ${2 * padPx}; })()`
|
|
855
|
+
);
|
|
856
|
+
}
|
|
857
|
+
/** The stand-in page's card height ({@link cardHeightExpression} at the stand-in's pad). */
|
|
858
|
+
export const CARD_HEIGHT_EXPRESSION = cardHeightExpression(JUDGE_INLINE_PAD_PX);
|
|
859
|
+
|
|
860
|
+
/**
|
|
861
|
+
* ggui#1475 — the inline card's horizontal overflow: how far its content runs past the card's width. The host's
|
|
862
|
+
* frame (and the judge's stand-in) clips at the card's edge, so the capture cannot show it, while a visitor's frame
|
|
863
|
+
* scrolls it sideways: only a measurement carries it.
|
|
864
|
+
*/
|
|
865
|
+
/** The sideways reading layout rounding alone can produce (ggui#1475): `scrollWidth` and `clientWidth` are integers. */
|
|
866
|
+
export const OVERFLOW_X_ROUNDING_PX = 1;
|
|
867
|
+
export const CARD_OVERFLOW_X_EXPRESSION =
|
|
868
|
+
"(() => { const r = document.getElementById('root'); return r ? Math.max(0, Math.ceil(r.scrollWidth - r.clientWidth)) : null; })()";
|
|
869
|
+
|
|
870
|
+
/**
|
|
871
|
+
* The natural capture's region: the card on the host ground, the ground margin on every side, the
|
|
872
|
+
* card's height capped at the box (content past it is the overflow the fit verdict reads). At least
|
|
873
|
+
* one row of card is kept, so a card that painted nothing is still read — as a blank, never as an
|
|
874
|
+
* unreadable capture.
|
|
875
|
+
*/
|
|
876
|
+
export function naturalClip(window: { width: number; height: number }, cardHeightPx: number, padPx: number = JUDGE_INLINE_PAD_PX): ScreenshotClip {
|
|
877
|
+
const m = padPx;
|
|
878
|
+
const box = window.height - 2 * m;
|
|
879
|
+
return { x: 0, y: 0, width: window.width, height: Math.max(1, Math.min(Math.ceil(cardHeightPx), box)) + 2 * m };
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
/** Measure the rendered document's height (or the card's, for a natural capture); a failed measurement is reported, never a failed capture. */
|
|
883
|
+
async function measureContentHeight(page: ScreenshotPage, expression: string = CONTENT_HEIGHT_EXPRESSION): Promise<number | null> {
|
|
614
884
|
try {
|
|
615
|
-
const h = await page.evaluate(
|
|
885
|
+
const h = await page.evaluate(expression);
|
|
616
886
|
return typeof h === 'number' && Number.isFinite(h) ? Math.round(h) : null;
|
|
617
887
|
} catch (e) {
|
|
618
888
|
console.warn(`[visual-eval] content height unavailable: ${e instanceof Error ? e.message : String(e)}`);
|
|
@@ -620,11 +890,18 @@ async function measureContentHeight(page: ScreenshotPage): Promise<number | null
|
|
|
620
890
|
}
|
|
621
891
|
}
|
|
622
892
|
|
|
893
|
+
/** ggui#1492 — a natural capture's geometry: the page's pad per side, and the host's floor when it has one. */
|
|
894
|
+
export interface NaturalCaptureOptions {
|
|
895
|
+
readonly padPx: number;
|
|
896
|
+
readonly floorPx?: number;
|
|
897
|
+
}
|
|
898
|
+
|
|
623
899
|
export async function captureScreenshotDetailed(
|
|
624
900
|
html: string,
|
|
625
901
|
viewport: { width: number; height: number } = DEFAULT_SCREENSHOT_VIEWPORT,
|
|
626
902
|
deps: ScreenshotDeps = {},
|
|
627
903
|
capture: CaptureMode = 'full-page',
|
|
904
|
+
natural: NaturalCaptureOptions = { padPx: JUDGE_INLINE_PAD_PX },
|
|
628
905
|
): Promise<ScreenshotAttempt> {
|
|
629
906
|
try {
|
|
630
907
|
const launchOptions = await resolveLaunchOptions(viewport, deps);
|
|
@@ -639,6 +916,21 @@ export async function captureScreenshotDetailed(
|
|
|
639
916
|
await page.waitForSelector('#root > *', { timeout: 10000 }).catch(() => {});
|
|
640
917
|
// Wait a bit for CSS/fonts to settle
|
|
641
918
|
await new Promise((r) => setTimeout(r, deps.settleMs ?? 1000));
|
|
919
|
+
if (capture === 'natural') {
|
|
920
|
+
// ggui#1475 — the inline card at its natural height: the capture is the card's extent on the host ground.
|
|
921
|
+
const cardHeightPx = await measureContentHeight(page, natural.padPx === JUDGE_INLINE_PAD_PX ? CARD_HEIGHT_EXPRESSION : cardHeightExpression(natural.padPx));
|
|
922
|
+
const overflowX = await measureContentHeight(page, CARD_OVERFLOW_X_EXPRESSION);
|
|
923
|
+
// ggui#1492 — the host's floor: measured AFTER the card (its height stays the card's own), then applied to the
|
|
924
|
+
// frame so the capture shows what a visitor sees under a short card — the frame's surface, as void.
|
|
925
|
+
const floorPx = natural.floorPx;
|
|
926
|
+
if (floorPx !== undefined && cardHeightPx !== null && cardHeightPx < floorPx) {
|
|
927
|
+
await page.evaluate(`(() => { const f = document.querySelector('.${JUDGE_INLINE_FRAME_CLASS}'); if (f) f.style.minHeight = '${floorPx}px'; })()`);
|
|
928
|
+
}
|
|
929
|
+
const framed = cardHeightPx === null ? viewport.height - 2 * natural.padPx : floorPx !== undefined ? Math.max(cardHeightPx, floorPx) : cardHeightPx;
|
|
930
|
+
const clip = naturalClip(viewport, framed, natural.padPx);
|
|
931
|
+
const screenshot = await page.screenshot({ type: 'png', fullPage: false, clip });
|
|
932
|
+
return { png: Buffer.from(screenshot), contentHeight: cardHeightPx, overflowX };
|
|
933
|
+
}
|
|
642
934
|
const contentHeight = await measureContentHeight(page);
|
|
643
935
|
const screenshot = await page.screenshot({ type: 'png', fullPage: capture === 'full-page' });
|
|
644
936
|
return { png: Buffer.from(screenshot), contentHeight };
|
|
@@ -741,10 +1033,14 @@ async function callMultimodalLLM(
|
|
|
741
1033
|
screenshot: Buffer,
|
|
742
1034
|
originalPrompt: string,
|
|
743
1035
|
profileBlock = '',
|
|
1036
|
+
criteriaBlock = '',
|
|
744
1037
|
) {
|
|
1038
|
+
// ggui#1436 — the criteria block rides in the USER turn after the profile block, so the
|
|
1039
|
+
// system prompt (and its digest pin) never moves; the block's digest is the set id.
|
|
745
1040
|
const userPrompt =
|
|
746
1041
|
`## Original Request\n${originalPrompt}\n\n` +
|
|
747
1042
|
(profileBlock.length > 0 ? `${profileBlock}\n\n` : '') +
|
|
1043
|
+
(criteriaBlock.length > 0 ? `${criteriaBlock}\n\n` : '') +
|
|
748
1044
|
'Evaluate the screenshot of the generated component.';
|
|
749
1045
|
const agent = createVisionAgent({
|
|
750
1046
|
provider: config.provider === 'claude' ? 'anthropic' : config.provider,
|
|
@@ -802,9 +1098,34 @@ interface CanvasFrame {
|
|
|
802
1098
|
readonly viewport: CanvasViewport;
|
|
803
1099
|
readonly policy: CanvasFitPolicy;
|
|
804
1100
|
readonly fit: 'fill' | undefined;
|
|
1101
|
+
/** The host stand-in drawn round the scope (ggui#1083 cut 3) — the window is padded by its gap; `viewport` stays the card's box. */
|
|
1102
|
+
readonly chrome: CanvasChrome;
|
|
805
1103
|
/** `true` when `canvasViewports` carried an entry for this canvas, whatever its value. */
|
|
806
1104
|
readonly declared: boolean;
|
|
807
1105
|
readonly attempt: ScreenshotAttempt;
|
|
1106
|
+
/** The capture's ink extent (ggui#1120); `null` when there is no capture or it could not be read. */
|
|
1107
|
+
readonly ink: InkExtent | null;
|
|
1108
|
+
/** ggui#1492 — what the judge did with this canvas's host presentation, when the caller supplied one. */
|
|
1109
|
+
readonly presentation?: CanvasPresentationOutcome;
|
|
1110
|
+
}
|
|
1111
|
+
/** The capture's ink extent (ggui#1120), read inside a drawn panel's chrome; an unreadable capture is reported, never blank. */
|
|
1112
|
+
function readInk(png: Buffer | null, chrome: CanvasChrome, canvas: CanvasClass, inlineInkInsetPx: number = JUDGE_INLINE_INK_INSET_PX): InkExtent | null {
|
|
1113
|
+
if (png === null) return null;
|
|
1114
|
+
const ink = readInkExtent(png, chrome === 'panel' ? JUDGE_INK_INSET_PX : chrome === 'ground' ? inlineInkInsetPx : 0);
|
|
1115
|
+
if ('reason' in ink) {
|
|
1116
|
+
console.warn(`[visual-eval] ink extent unreadable at canvas ${canvas}: ${ink.reason}`);
|
|
1117
|
+
return null;
|
|
1118
|
+
}
|
|
1119
|
+
return ink;
|
|
1120
|
+
}
|
|
1121
|
+
/** The deterministic blank verdict (ggui#1120): an issue when the frame read no ink at all; `null` otherwise, an unreadable capture included. */
|
|
1122
|
+
function blankVerdict(canvas: CanvasClass, frame: CanvasFrame): EvaluationIssue | null {
|
|
1123
|
+
// ggui#1475 — a naturally-captured card that measured no height painted nothing: its capture is only the host's frame,
|
|
1124
|
+
// which the ink reading skips by design, so the blank is read from the measurement, never mistaken for unreadable.
|
|
1125
|
+
if (frame.chrome === 'ground' && frame.attempt.contentHeight !== null && frame.attempt.contentHeight <= 0) {
|
|
1126
|
+
return canvasBlankIssue(canvas, frame.viewport);
|
|
1127
|
+
}
|
|
1128
|
+
return frame.ink !== null && frame.ink.lastInkRow === null ? canvasBlankIssue(canvas, frame.viewport) : null;
|
|
808
1129
|
}
|
|
809
1130
|
|
|
810
1131
|
async function frameCanvas(
|
|
@@ -814,14 +1135,29 @@ async function frameCanvas(
|
|
|
814
1135
|
canvas: CanvasClass,
|
|
815
1136
|
canvasViewports: VisualEvalConfig['canvasViewports'],
|
|
816
1137
|
deps: ScreenshotDeps,
|
|
1138
|
+
hostPresentations?: VisualEvalConfig['hostPresentations'],
|
|
817
1139
|
): Promise<CanvasFrame> {
|
|
818
1140
|
const declaredBox = canvasViewports?.[canvas];
|
|
819
1141
|
const viewport = declaredBox !== undefined ? integerBox(declaredBox) : CANVAS_VIEWPORTS[canvas];
|
|
820
1142
|
const policy = canvasFitPolicy(canvas);
|
|
821
1143
|
const fit = canvasFit(canvas);
|
|
822
|
-
const
|
|
823
|
-
const
|
|
824
|
-
|
|
1144
|
+
const chrome = canvasChrome(canvas);
|
|
1145
|
+
const outcome = resolvePresentation(canvas, hostPresentations?.[canvas]);
|
|
1146
|
+
const presentation = outcome?.status === 'applied' ? outcome.applied : undefined;
|
|
1147
|
+
const geometry = inlineGeometry(presentation?.frame);
|
|
1148
|
+
const canvasHtml =
|
|
1149
|
+
fit !== undefined || chrome !== undefined ? buildRenderHTML(bundledCode, context.cssTokens, fit, chrome, presentation) : html;
|
|
1150
|
+
const captured = await captureScreenshotDetailed(canvasHtml, judgeWindow(viewport, chrome, geometry.padPx), deps, policy.capture, {
|
|
1151
|
+
padPx: geometry.padPx,
|
|
1152
|
+
...(geometry.floorPx !== undefined ? { floorPx: geometry.floorPx } : {}),
|
|
1153
|
+
});
|
|
1154
|
+
// ggui#1083 cut 3 — on a panelled page the document is the panel plus its gap; every verdict reads the CARD's height.
|
|
1155
|
+
const attempt: ScreenshotAttempt = { ...captured, contentHeight: cardHeight(captured.contentHeight, chrome) };
|
|
1156
|
+
return {
|
|
1157
|
+
viewport, policy, fit, chrome, declared: declaredBox !== undefined, attempt,
|
|
1158
|
+
ink: readInk(captured.png, chrome, canvas, geometry.inkInsetPx),
|
|
1159
|
+
...(outcome !== undefined ? { presentation: outcome } : {}),
|
|
1160
|
+
};
|
|
825
1161
|
}
|
|
826
1162
|
|
|
827
1163
|
/**
|
|
@@ -835,11 +1171,61 @@ function fitVerdict(canvas: CanvasClass, frame: CanvasFrame): EvaluationIssue |
|
|
|
835
1171
|
return canvasOverflowIssue(canvas, frame.viewport, contentHeight, frame.policy.overflow);
|
|
836
1172
|
}
|
|
837
1173
|
|
|
1174
|
+
/**
|
|
1175
|
+
* ggui#1475 — the deterministic horizontal verdict on a natural capture: the card's content runs past its width. It
|
|
1176
|
+
* is judged by the canvas's own fit policy (the inline card: a critical that fails the canvas), in its own dimension,
|
|
1177
|
+
* so the in-loop `[fit]` extension (vertical overflow) is unchanged.
|
|
1178
|
+
*/
|
|
1179
|
+
function overflowXVerdict(canvas: CanvasClass, frame: CanvasFrame): EvaluationIssue | null {
|
|
1180
|
+
const px = frame.attempt.overflowX;
|
|
1181
|
+
// `scrollWidth` / `clientWidth` are rounded integers: a fractional layout width reads as 1 px past the card with
|
|
1182
|
+
// nothing a visitor could see, so 1 px is rounding and an overflow starts at 2.
|
|
1183
|
+
if (px === undefined || px === null || px <= OVERFLOW_X_ROUNDING_PX || frame.policy.overflow === 'none') return null;
|
|
1184
|
+
return canvasOverflowXIssue(canvas, frame.viewport, px, frame.policy.overflow);
|
|
1185
|
+
}
|
|
1186
|
+
|
|
838
1187
|
/** One judge call parsed, retried ONCE on a malformed answer (#1017); the FIRST reason is kept verbatim. */
|
|
839
1188
|
type JudgedAnswer =
|
|
840
1189
|
| { readonly kind: 'ok'; readonly result: EvaluationResult; readonly inputTokens: number; readonly outputTokens: number }
|
|
841
1190
|
| { readonly kind: 'unparsable'; readonly reason: string };
|
|
842
1191
|
|
|
1192
|
+
/**
|
|
1193
|
+
* ggui#1436 — the criteria are REPORT-ONLY, so they are never asked inside a call whose score binds. The K
|
|
1194
|
+
* scoring calls go out exactly as they did before the block existed (the same user turn, byte for byte); when a
|
|
1195
|
+
* block was selected, the criteria get ONE call of their own on the same frame, run beside them, and that call's
|
|
1196
|
+
* score is discarded. Its spend is reported apart (`criteriaTokens`), never inside the scoring calls' tokens, so a
|
|
1197
|
+
* cost read off the judgement is the judgement's alone. Asked inside the scoring call, the block moved the score it
|
|
1198
|
+
* sat beside — the first release that carried it into a served judge lowered first-attempt scores on the same kind
|
|
1199
|
+
* of card (ggui#1436). One read, not K: the block binds on nothing, and a second K would double its calls.
|
|
1200
|
+
*/
|
|
1201
|
+
async function judgeFrame(
|
|
1202
|
+
judge: typeof callMultimodalLLM,
|
|
1203
|
+
config: VisualEvalConfig,
|
|
1204
|
+
model: string,
|
|
1205
|
+
png: Buffer,
|
|
1206
|
+
originalPrompt: string,
|
|
1207
|
+
profileBlock: string,
|
|
1208
|
+
canvas: CanvasClass,
|
|
1209
|
+
k: number,
|
|
1210
|
+
criteriaBlock: string,
|
|
1211
|
+
): Promise<{ readonly scoring: readonly JudgedAnswer[]; readonly criteria: JudgedAnswer | undefined }> {
|
|
1212
|
+
const [scoring, criteria] = await Promise.all([
|
|
1213
|
+
Promise.all(Array.from({ length: k }, () => judgeAndParse(judge, config, model, png, originalPrompt, profileBlock, canvas))),
|
|
1214
|
+
criteriaBlock.length > 0 ? judgeAndParse(judge, config, model, png, originalPrompt, profileBlock, canvas, criteriaBlock) : undefined,
|
|
1215
|
+
]);
|
|
1216
|
+
return { scoring, criteria };
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
/** What the frame's criteria call spent, when it was asked and answered — reported apart from the scoring calls. */
|
|
1220
|
+
function criteriaCallTokens(criteria: JudgedAnswer | undefined): CriteriaCallTokens | undefined {
|
|
1221
|
+
return criteria?.kind === 'ok' ? { inputTokens: criteria.inputTokens, outputTokens: criteria.outputTokens } : undefined;
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
/** The criteria call's answers, as the one read the block resolves; none when it was not asked or did not parse. */
|
|
1225
|
+
function criteriaReads(criteria: JudgedAnswer | undefined): CriteriaAnswer[][] {
|
|
1226
|
+
return criteria?.kind === 'ok' ? [criteria.result.criteriaAnswers ?? []] : [];
|
|
1227
|
+
}
|
|
1228
|
+
|
|
843
1229
|
/** How many vision calls this canvas gets (ggui#1072): `judgeK` when the canvas is sampled, else 1. */
|
|
844
1230
|
function judgeCountFor(config: VisualEvalConfig, canvas: CanvasClass): number {
|
|
845
1231
|
const k = Math.max(1, Math.floor(config.judgeK ?? 1));
|
|
@@ -869,10 +1255,11 @@ async function judgeAndParse(
|
|
|
869
1255
|
originalPrompt: string,
|
|
870
1256
|
profileBlock: string,
|
|
871
1257
|
canvas?: CanvasClass,
|
|
1258
|
+
criteriaBlock = '',
|
|
872
1259
|
): Promise<JudgedAnswer> {
|
|
873
1260
|
let firstReason: string | undefined;
|
|
874
1261
|
for (let attempt = 1; attempt <= 2; attempt++) {
|
|
875
|
-
const response = await judge(config, model, VISUAL_EVAL_PROMPT, screenshot, originalPrompt, profileBlock);
|
|
1262
|
+
const response = await judge(config, model, VISUAL_EVAL_PROMPT, screenshot, originalPrompt, profileBlock, criteriaBlock);
|
|
876
1263
|
try {
|
|
877
1264
|
const result = parseVisualResponse(response.text, config.passThreshold);
|
|
878
1265
|
return { kind: 'ok', result, inputTokens: response.inputTokens, outputTokens: response.outputTokens };
|
|
@@ -941,7 +1328,7 @@ export async function runVisualEvaluationDetailed(
|
|
|
941
1328
|
const perCanvas: CanvasVisualResult[] = [];
|
|
942
1329
|
const perCanvasResults: EvaluationResult[] = [];
|
|
943
1330
|
for (const canvas of config.canvases) {
|
|
944
|
-
const frame = await frameCanvas(bundledCode, context, html, canvas, config.canvasViewports, deps);
|
|
1331
|
+
const frame = await frameCanvas(bundledCode, context, html, canvas, config.canvasViewports, deps, config.hostPresentations);
|
|
945
1332
|
const { viewport, policy, fit, attempt } = frame;
|
|
946
1333
|
const screenshot = attempt.png;
|
|
947
1334
|
if (!screenshot) {
|
|
@@ -952,8 +1339,20 @@ export async function runVisualEvaluationDetailed(
|
|
|
952
1339
|
// ggui#1072: k vision calls on the SAME frame; the decision value is the median.
|
|
953
1340
|
const k = judgeCountFor(config, canvas);
|
|
954
1341
|
const profileBlock = buildStylingProfileJudgeBlock(context.profile);
|
|
955
|
-
|
|
956
|
-
|
|
1342
|
+
// ggui#1436 — the selection is decided BEFORE the judge runs and recorded with the verdict.
|
|
1343
|
+
const criteriaContext: CriteriaContext | undefined =
|
|
1344
|
+
context.criteria !== undefined ? { ...context.criteria.context, canvas } : undefined;
|
|
1345
|
+
const criteriaSelected =
|
|
1346
|
+
context.criteria !== undefined && criteriaContext !== undefined ? selectCriteria(context.criteria.bank, criteriaContext) : undefined;
|
|
1347
|
+
const criteriaBlock =
|
|
1348
|
+
context.criteria !== undefined && criteriaSelected !== undefined
|
|
1349
|
+
? buildCriteriaJudgeBlock(context.criteria.bank, criteriaSelected, {
|
|
1350
|
+
frame: { canvas, width: viewport.width, height: viewport.height, ...(frame.presentation?.status === 'applied' ? { hostLabel: frame.presentation.applied.label } : {}) },
|
|
1351
|
+
...(config.sampleProps !== undefined ? { propsJson: JSON.stringify(config.sampleProps) } : {}),
|
|
1352
|
+
})
|
|
1353
|
+
: '';
|
|
1354
|
+
const { scoring: answers, criteria: criteriaAnswer } = await judgeFrame(
|
|
1355
|
+
judge, config, model, screenshot, context.originalPrompt, profileBlock, canvas, k, criteriaBlock,
|
|
957
1356
|
);
|
|
958
1357
|
const parsed = answers.filter((a): a is Extract<JudgedAnswer, { kind: 'ok' }> => a.kind === 'ok');
|
|
959
1358
|
if (parsed.length === 0) {
|
|
@@ -969,6 +1368,7 @@ export async function runVisualEvaluationDetailed(
|
|
|
969
1368
|
result.inputTokens = parsed.reduce((sum, a) => sum + a.inputTokens, 0);
|
|
970
1369
|
result.outputTokens = parsed.reduce((sum, a) => sum + a.outputTokens, 0);
|
|
971
1370
|
const response = { inputTokens: result.inputTokens, outputTokens: result.outputTokens };
|
|
1371
|
+
const criteriaTokens = criteriaCallTokens(criteriaAnswer);
|
|
972
1372
|
const judgeRecord: CanvasJudgeRecord = {
|
|
973
1373
|
k,
|
|
974
1374
|
rule: 'median',
|
|
@@ -984,6 +1384,29 @@ export async function runVisualEvaluationDetailed(
|
|
|
984
1384
|
result.issues.push(fitIssue);
|
|
985
1385
|
if (policy.overflow === 'fail') result.passed = false;
|
|
986
1386
|
}
|
|
1387
|
+
const overflowXIssue = overflowXVerdict(canvas, frame);
|
|
1388
|
+
if (overflowXIssue !== null) {
|
|
1389
|
+
result.issues.push(overflowXIssue);
|
|
1390
|
+
if (policy.overflow === 'fail') result.passed = false;
|
|
1391
|
+
}
|
|
1392
|
+
// The blank verdict (ggui#1120): deterministic, critical on every canvas — a render that painted nothing fails.
|
|
1393
|
+
const blankIssue = blankVerdict(canvas, frame);
|
|
1394
|
+
if (blankIssue !== null) {
|
|
1395
|
+
result.issues.push(blankIssue);
|
|
1396
|
+
result.passed = false;
|
|
1397
|
+
}
|
|
1398
|
+
const inkRatio = frame.ink?.ratio ?? null;
|
|
1399
|
+
// ggui#1436 — the block: the criteria call's read, the instruments' measurements, report-only.
|
|
1400
|
+
const criteria: CriteriaBlock | undefined =
|
|
1401
|
+
context.criteria !== undefined && criteriaContext !== undefined && criteriaSelected !== undefined
|
|
1402
|
+
? resolveCriteriaBlock({
|
|
1403
|
+
bank: context.criteria.bank,
|
|
1404
|
+
context: criteriaContext,
|
|
1405
|
+
selected: criteriaSelected,
|
|
1406
|
+
answers: criteriaReads(criteriaAnswer),
|
|
1407
|
+
measurements: { overflow, contentHeight, viewportHeight: viewport.height, inkRatio },
|
|
1408
|
+
})
|
|
1409
|
+
: undefined;
|
|
987
1410
|
perCanvasResults.push(result);
|
|
988
1411
|
perCanvas.push({
|
|
989
1412
|
canvas,
|
|
@@ -993,14 +1416,20 @@ export async function runVisualEvaluationDetailed(
|
|
|
993
1416
|
screenshotPng: screenshot,
|
|
994
1417
|
contentHeight,
|
|
995
1418
|
overflow,
|
|
1419
|
+
inkRatio,
|
|
996
1420
|
judge: judgeRecord,
|
|
997
1421
|
...(fit !== undefined ? { fit } : {}),
|
|
998
1422
|
...(frame.declared ? { declared: true as const } : {}),
|
|
1423
|
+
...(criteria !== undefined ? { criteria } : {}),
|
|
1424
|
+
...(criteriaTokens !== undefined ? { criteriaTokens } : {}),
|
|
1425
|
+
...(frame.presentation !== undefined ? { presentation: frame.presentation } : {}),
|
|
999
1426
|
});
|
|
1000
1427
|
console.log(
|
|
1001
1428
|
`[visual-eval] canvas=${canvas} ${viewport.width}×${viewport.height} score=${result.finalScore}${k > 1 ? ` (median of ${judgeRecord.samples.length}/${k}: ${judgeRecord.samples.join(',')} σ=${judgeRecord.sigma})` : ''} ` +
|
|
1002
1429
|
`${result.passed ? 'pass' : 'FAIL'} | content=${contentHeight ?? '?'}px${overflow ? ` OVERFLOW (${policy.overflow})` : ''} ` +
|
|
1003
|
-
|
|
1430
|
+
`ink=${inkRatio ?? '?'}${blankIssue !== null ? ' BLANK' : ''} ` +
|
|
1431
|
+
`| in=${response.inputTokens} out=${response.outputTokens}` +
|
|
1432
|
+
(criteriaTokens !== undefined ? ` | criteria in=${criteriaTokens.inputTokens} out=${criteriaTokens.outputTokens}` : ''),
|
|
1004
1433
|
);
|
|
1005
1434
|
}
|
|
1006
1435
|
const aggregate = aggregateCanvasResults(perCanvas, perCanvasResults);
|
|
@@ -1081,6 +1510,7 @@ function aggregateCanvasResults(
|
|
|
1081
1510
|
inputTokens += r.inputTokens ?? 0;
|
|
1082
1511
|
outputTokens += r.outputTokens ?? 0;
|
|
1083
1512
|
});
|
|
1513
|
+
const criteriaSpent = perCanvas.flatMap((c) => (c.criteriaTokens !== undefined ? [c.criteriaTokens] : []));
|
|
1084
1514
|
return {
|
|
1085
1515
|
passed: perCanvas.every((c) => c.passed),
|
|
1086
1516
|
finalScore: mean((r) => r.finalScore),
|
|
@@ -1089,6 +1519,14 @@ function aggregateCanvasResults(
|
|
|
1089
1519
|
...(critiques.length > 0 ? { critique: critiques.join('\n') } : {}),
|
|
1090
1520
|
inputTokens,
|
|
1091
1521
|
outputTokens,
|
|
1522
|
+
...(criteriaSpent.length > 0
|
|
1523
|
+
? {
|
|
1524
|
+
criteriaTokens: {
|
|
1525
|
+
inputTokens: criteriaSpent.reduce((sum, t) => sum + t.inputTokens, 0),
|
|
1526
|
+
outputTokens: criteriaSpent.reduce((sum, t) => sum + t.outputTokens, 0),
|
|
1527
|
+
},
|
|
1528
|
+
}
|
|
1529
|
+
: {}),
|
|
1092
1530
|
canvases: [...perCanvas],
|
|
1093
1531
|
};
|
|
1094
1532
|
}
|
|
@@ -1103,9 +1541,19 @@ export function summarizeVisualResult(result: VisualEvaluationResult): VisualEva
|
|
|
1103
1541
|
passed: c.passed,
|
|
1104
1542
|
contentHeight: c.contentHeight,
|
|
1105
1543
|
overflow: c.overflow,
|
|
1544
|
+
inkRatio: c.inkRatio,
|
|
1106
1545
|
judge: c.judge,
|
|
1107
1546
|
...(c.fit !== undefined ? { fit: c.fit } : {}),
|
|
1547
|
+
...(c.criteria !== undefined ? { criteria: c.criteria } : {}),
|
|
1548
|
+
...(c.presentation !== undefined ? { presentation: c.presentation } : {}),
|
|
1108
1549
|
}));
|
|
1550
|
+
// ggui#1436 — the roll-up: ids failed / unreadable across canvases, deduped; a must at n/a is listed, never passed.
|
|
1551
|
+
const blocks = result.canvases.map((c) => c.criteria).filter((b): b is CriteriaBlock => b !== undefined);
|
|
1552
|
+
const idsWhere = (severity: 'must' | 'should', verdict: 'fail' | 'n/a'): string[] => [
|
|
1553
|
+
...new Set(blocks.flatMap((b) => b.verdicts.filter((v) => v.severity === severity && v.verdict === verdict).map((v) => v.id))),
|
|
1554
|
+
];
|
|
1555
|
+
const rollup: CriteriaRollup | undefined =
|
|
1556
|
+
blocks.length > 0 ? { mustFailed: idsWhere('must', 'fail'), shouldFailed: idsWhere('should', 'fail'), mustNa: idsWhere('must', 'n/a') } : undefined;
|
|
1109
1557
|
// ggui#1195 — the fit stamp: the first canvas whose policy FAILS an
|
|
1110
1558
|
// overflow (the inline card), judged with a measurable height.
|
|
1111
1559
|
const fitCanvas = result.canvases.find((c) => canvasFitPolicy(c.canvas).overflow === 'fail' && c.contentHeight !== null);
|
|
@@ -1126,6 +1574,134 @@ export function summarizeVisualResult(result: VisualEvaluationResult): VisualEva
|
|
|
1126
1574
|
...(fit !== undefined ? { fit } : {}),
|
|
1127
1575
|
...(result.design !== undefined ? { design: result.design } : {}),
|
|
1128
1576
|
...(result.themeMode !== undefined ? { themeMode: result.themeMode } : {}),
|
|
1577
|
+
...(rollup !== undefined ? { criteria: rollup } : {}),
|
|
1578
|
+
};
|
|
1579
|
+
}
|
|
1580
|
+
|
|
1581
|
+
// ---------------------------------------------------------------------------
|
|
1582
|
+
// A stored capture, judged (ggui#1438)
|
|
1583
|
+
// ---------------------------------------------------------------------------
|
|
1584
|
+
|
|
1585
|
+
/** One stored capture to judge: the frame the judge saw at capture time, plus what the capture measured. */
|
|
1586
|
+
export interface StoredCaptureInput {
|
|
1587
|
+
readonly canvas: CanvasClass;
|
|
1588
|
+
/** The capture as saved (`canvas-<class>.png`). */
|
|
1589
|
+
readonly png: Buffer;
|
|
1590
|
+
/** The box the capture was taken at — the class box, or the declared one. */
|
|
1591
|
+
readonly viewport: CanvasViewport;
|
|
1592
|
+
/** The document's scroll height the capture recorded; `null` when it did not. */
|
|
1593
|
+
readonly contentHeight: number | null;
|
|
1594
|
+
readonly originalPrompt: string;
|
|
1595
|
+
readonly profile?: GenerationProfileInput;
|
|
1596
|
+
/** ggui#1436 — the bank and the card's context; absent = today's judge, no block. */
|
|
1597
|
+
readonly criteria?: { readonly bank: CriteriaBank; readonly context: CriteriaContextInput };
|
|
1598
|
+
/** The props the capture was rendered with — handed to the criteria block for the rows whose evidence names props. */
|
|
1599
|
+
readonly sampleProps?: JsonObject;
|
|
1600
|
+
}
|
|
1601
|
+
|
|
1602
|
+
export interface StoredCaptureVerdict {
|
|
1603
|
+
readonly canvas: CanvasClass;
|
|
1604
|
+
readonly viewport: CanvasViewport;
|
|
1605
|
+
readonly score: number;
|
|
1606
|
+
readonly passed: boolean;
|
|
1607
|
+
readonly issues: EvaluationIssue[];
|
|
1608
|
+
readonly judge: CanvasJudgeRecord;
|
|
1609
|
+
readonly contentHeight: number | null;
|
|
1610
|
+
readonly overflow: boolean;
|
|
1611
|
+
readonly inkRatio: number | null;
|
|
1612
|
+
readonly criteria?: CriteriaBlock;
|
|
1613
|
+
/** The scoring calls' spend. */
|
|
1614
|
+
readonly inputTokens: number;
|
|
1615
|
+
readonly outputTokens: number;
|
|
1616
|
+
/** ggui#1436 — the report-only criteria call's spend, apart from the scoring calls'; absent when none was asked or answered. */
|
|
1617
|
+
readonly criteriaTokens?: CriteriaCallTokens;
|
|
1618
|
+
}
|
|
1619
|
+
|
|
1620
|
+
export type StoredCaptureOutcome =
|
|
1621
|
+
| { readonly kind: 'ok'; readonly verdict: StoredCaptureVerdict }
|
|
1622
|
+
| { readonly kind: 'unavailable'; readonly reason: string };
|
|
1623
|
+
|
|
1624
|
+
/**
|
|
1625
|
+
* Judge a STORED capture exactly as the per-canvas leg judges a fresh one — the same K scoring calls on the
|
|
1626
|
+
* same frame, the same median, the same fit and blank verdicts, the same report-only criteria call — without
|
|
1627
|
+
* rendering.
|
|
1628
|
+
* This is the calibration's path (ggui#1438): the frame his eye marks is the frame the judge saw, and a
|
|
1629
|
+
* re-judge costs judge tokens only. Fit reads the capture's recorded `contentHeight`; ink is read off the
|
|
1630
|
+
* PNG inside the canvas's chrome, as at capture.
|
|
1631
|
+
*/
|
|
1632
|
+
export async function judgeStoredCapture(
|
|
1633
|
+
input: StoredCaptureInput,
|
|
1634
|
+
config: VisualEvalConfig,
|
|
1635
|
+
deps: Pick<VisualEvalDeps, 'judge'> = {},
|
|
1636
|
+
): Promise<StoredCaptureOutcome> {
|
|
1637
|
+
const judge = deps.judge ?? callMultimodalLLM;
|
|
1638
|
+
const model = config.model ?? getDefaultVisualModel(config.provider);
|
|
1639
|
+
const { canvas, viewport, png } = input;
|
|
1640
|
+
const chrome = canvasChrome(canvas);
|
|
1641
|
+
const ink = readInk(png, chrome, canvas);
|
|
1642
|
+
const inkRatio = ink?.ratio ?? null;
|
|
1643
|
+
const k = judgeCountFor(config, canvas);
|
|
1644
|
+
const profileBlock = buildStylingProfileJudgeBlock(input.profile);
|
|
1645
|
+
const criteriaContext: CriteriaContext | undefined = input.criteria !== undefined ? { ...input.criteria.context, canvas } : undefined;
|
|
1646
|
+
const criteriaSelected =
|
|
1647
|
+
input.criteria !== undefined && criteriaContext !== undefined ? selectCriteria(input.criteria.bank, criteriaContext) : undefined;
|
|
1648
|
+
const criteriaBlock =
|
|
1649
|
+
input.criteria !== undefined && criteriaSelected !== undefined
|
|
1650
|
+
? buildCriteriaJudgeBlock(input.criteria.bank, criteriaSelected, {
|
|
1651
|
+
frame: { canvas, width: viewport.width, height: viewport.height },
|
|
1652
|
+
...(input.sampleProps !== undefined ? { propsJson: JSON.stringify(input.sampleProps) } : {}),
|
|
1653
|
+
})
|
|
1654
|
+
: '';
|
|
1655
|
+
const { scoring: answers, criteria: criteriaAnswer } = await judgeFrame(judge, config, model, png, input.originalPrompt, profileBlock, canvas, k, criteriaBlock);
|
|
1656
|
+
const parsed = answers.filter((a): a is Extract<JudgedAnswer, { kind: 'ok' }> => a.kind === 'ok');
|
|
1657
|
+
if (parsed.length === 0) {
|
|
1658
|
+
const first = answers.find((a): a is Extract<JudgedAnswer, { kind: 'unparsable' }> => a.kind === 'unparsable');
|
|
1659
|
+
return { kind: 'unavailable', reason: first?.reason ?? 'judge answer unparsable' };
|
|
1660
|
+
}
|
|
1661
|
+
const samples = parsed.map((a) => a.result.finalScore);
|
|
1662
|
+
const median = medianOf(samples);
|
|
1663
|
+
const result = parsed[samples.indexOf(median)]!.result;
|
|
1664
|
+
const issues = [...result.issues];
|
|
1665
|
+
let passed = median >= config.passThreshold;
|
|
1666
|
+
const contentHeight = input.contentHeight;
|
|
1667
|
+
const overflow = contentHeight !== null && contentHeight > viewport.height;
|
|
1668
|
+
const policy = canvasFitPolicy(canvas);
|
|
1669
|
+
if (overflow && contentHeight !== null && policy.overflow !== 'none') {
|
|
1670
|
+
issues.push(canvasOverflowIssue(canvas, viewport, contentHeight, policy.overflow));
|
|
1671
|
+
if (policy.overflow === 'fail') passed = false;
|
|
1672
|
+
}
|
|
1673
|
+
if (ink !== null && ink.lastInkRow === null) {
|
|
1674
|
+
issues.push(canvasBlankIssue(canvas, viewport));
|
|
1675
|
+
passed = false;
|
|
1676
|
+
}
|
|
1677
|
+
const criteria: CriteriaBlock | undefined =
|
|
1678
|
+
input.criteria !== undefined && criteriaContext !== undefined && criteriaSelected !== undefined
|
|
1679
|
+
? resolveCriteriaBlock({
|
|
1680
|
+
bank: input.criteria.bank,
|
|
1681
|
+
context: criteriaContext,
|
|
1682
|
+
selected: criteriaSelected,
|
|
1683
|
+
answers: criteriaReads(criteriaAnswer),
|
|
1684
|
+
measurements: { overflow, contentHeight, viewportHeight: viewport.height, inkRatio },
|
|
1685
|
+
})
|
|
1686
|
+
: undefined;
|
|
1687
|
+
const criteriaTokens = criteriaCallTokens(criteriaAnswer);
|
|
1688
|
+
return {
|
|
1689
|
+
kind: 'ok',
|
|
1690
|
+
verdict: {
|
|
1691
|
+
canvas,
|
|
1692
|
+
viewport,
|
|
1693
|
+
score: median,
|
|
1694
|
+
passed,
|
|
1695
|
+
issues,
|
|
1696
|
+
judge: { k, rule: 'median', samples, sigma: populationSigma(samples), notes: parsed.map((a) => a.result.critique ?? '') },
|
|
1697
|
+
contentHeight,
|
|
1698
|
+
overflow,
|
|
1699
|
+
inkRatio,
|
|
1700
|
+
...(criteria !== undefined ? { criteria } : {}),
|
|
1701
|
+
inputTokens: parsed.reduce((sum, a) => sum + a.inputTokens, 0),
|
|
1702
|
+
outputTokens: parsed.reduce((sum, a) => sum + a.outputTokens, 0),
|
|
1703
|
+
...(criteriaTokens !== undefined ? { criteriaTokens } : {}),
|
|
1704
|
+
},
|
|
1129
1705
|
};
|
|
1130
1706
|
}
|
|
1131
1707
|
|
|
@@ -1224,15 +1800,33 @@ function parseVisualResponse(text: string, passThreshold: number): EvaluationRes
|
|
|
1224
1800
|
fix: i.fix || '',
|
|
1225
1801
|
}));
|
|
1226
1802
|
|
|
1803
|
+
// ggui#1436 — the criteria answers, tolerant: absent block → no field (an old answer stays
|
|
1804
|
+
// valid); an entry without a string id or with a verdict outside the vocabulary is dropped.
|
|
1805
|
+
const criteriaAnswers = parseCriteriaAnswers(raw.criteria);
|
|
1806
|
+
|
|
1227
1807
|
return {
|
|
1228
1808
|
passed: finalScore >= passThreshold,
|
|
1229
1809
|
finalScore,
|
|
1230
1810
|
dimensions,
|
|
1231
1811
|
issues,
|
|
1232
1812
|
critique: raw.critique,
|
|
1813
|
+
...(criteriaAnswers !== undefined ? { criteriaAnswers } : {}),
|
|
1233
1814
|
};
|
|
1234
1815
|
}
|
|
1235
1816
|
|
|
1817
|
+
function parseCriteriaAnswers(raw: unknown): CriteriaAnswer[] | undefined {
|
|
1818
|
+
if (!Array.isArray(raw)) return undefined;
|
|
1819
|
+
const out: CriteriaAnswer[] = [];
|
|
1820
|
+
for (const entry of raw) {
|
|
1821
|
+
if (typeof entry !== 'object' || entry === null) continue;
|
|
1822
|
+
const o = entry as { id?: unknown; verdict?: unknown; evidence?: unknown };
|
|
1823
|
+
if (typeof o.id !== 'string') continue;
|
|
1824
|
+
if (o.verdict !== 'pass' && o.verdict !== 'fail' && o.verdict !== 'n/a') continue;
|
|
1825
|
+
out.push({ id: o.id, verdict: o.verdict, evidence: typeof o.evidence === 'string' ? o.evidence : '' });
|
|
1826
|
+
}
|
|
1827
|
+
return out;
|
|
1828
|
+
}
|
|
1829
|
+
|
|
1236
1830
|
// ---------------------------------------------------------------------------
|
|
1237
1831
|
// Default models for visual evaluation
|
|
1238
1832
|
// ---------------------------------------------------------------------------
|
|
@@ -1308,7 +1902,7 @@ function toEvalIssue(issue: EvaluationIssue): EvalIssue {
|
|
|
1308
1902
|
// ---------------------------------------------------------------------------
|
|
1309
1903
|
|
|
1310
1904
|
/** What the fit half reads — the frame's inputs; nothing about a judge. */
|
|
1311
|
-
export type VisualFitConfig = Pick<VisualEvalConfig, 'sampleProps' | 'canvases' | 'canvasViewports' | 'designSrcDir'>;
|
|
1905
|
+
export type VisualFitConfig = Pick<VisualEvalConfig, 'sampleProps' | 'canvases' | 'canvasViewports' | 'hostPresentations' | 'designSrcDir'>;
|
|
1312
1906
|
|
|
1313
1907
|
/** One canvas's fit reading: the box, the measure, whether it overflowed — no score. */
|
|
1314
1908
|
export interface CanvasFitReading {
|
|
@@ -1316,8 +1910,12 @@ export interface CanvasFitReading {
|
|
|
1316
1910
|
readonly viewport: CanvasViewport;
|
|
1317
1911
|
readonly contentHeight: number | null;
|
|
1318
1912
|
readonly overflow: boolean;
|
|
1913
|
+
/** ggui#1120 — the capture's ink ratio; `0` is the blank, `null` an unreadable capture. */
|
|
1914
|
+
readonly inkRatio: number | null;
|
|
1319
1915
|
/** `true` when the order declared this canvas's box. */
|
|
1320
1916
|
readonly declared?: true;
|
|
1917
|
+
/** ggui#1492 — what the judge did with this canvas's host presentation, when one was supplied. */
|
|
1918
|
+
readonly presentation?: CanvasPresentationOutcome;
|
|
1321
1919
|
}
|
|
1322
1920
|
|
|
1323
1921
|
/** `measured` = every canvas was framed (its fit issues, possibly none); `unavailable` = no verdict, with why. */
|
|
@@ -1361,7 +1959,7 @@ export async function runVisualFit(
|
|
|
1361
1959
|
const issues: EvalIssue[] = [];
|
|
1362
1960
|
const readings: CanvasFitReading[] = [];
|
|
1363
1961
|
for (const canvas of config.canvases) {
|
|
1364
|
-
const frame = await frameCanvas(bundledCode, context, html, canvas, config.canvasViewports, deps);
|
|
1962
|
+
const frame = await frameCanvas(bundledCode, context, html, canvas, config.canvasViewports, deps, config.hostPresentations);
|
|
1365
1963
|
if (!frame.attempt.png) {
|
|
1366
1964
|
return { status: 'unavailable', reason: `${frame.attempt.reason ?? 'no browser available'} at canvas ${canvas}` };
|
|
1367
1965
|
}
|
|
@@ -1369,16 +1967,24 @@ export async function runVisualFit(
|
|
|
1369
1967
|
const overflow = contentHeight !== null && contentHeight > frame.viewport.height;
|
|
1370
1968
|
const verdict = fitVerdict(canvas, frame);
|
|
1371
1969
|
if (verdict !== null) issues.push(toEvalIssue(canvasScopedIssue(canvas, verdict)));
|
|
1970
|
+
const overflowXIssue = overflowXVerdict(canvas, frame);
|
|
1971
|
+
if (overflowXIssue !== null) issues.push(toEvalIssue(canvasScopedIssue(canvas, overflowXIssue)));
|
|
1972
|
+
const blank = blankVerdict(canvas, frame);
|
|
1973
|
+
if (blank !== null) issues.push(toEvalIssue(canvasScopedIssue(canvas, blank)));
|
|
1974
|
+
const inkRatio = frame.ink?.ratio ?? null;
|
|
1372
1975
|
readings.push({
|
|
1373
1976
|
canvas,
|
|
1374
1977
|
viewport: frame.viewport,
|
|
1375
1978
|
contentHeight,
|
|
1376
1979
|
overflow,
|
|
1980
|
+
inkRatio,
|
|
1377
1981
|
...(frame.declared ? { declared: true as const } : {}),
|
|
1982
|
+
...(frame.presentation !== undefined ? { presentation: frame.presentation } : {}),
|
|
1378
1983
|
});
|
|
1379
1984
|
console.log(
|
|
1380
1985
|
`[visual-fit] canvas=${canvas} ${frame.viewport.width}×${frame.viewport.height} ` +
|
|
1381
|
-
`content=${contentHeight ?? '?'}px${overflow ? ` OVERFLOW (${frame.policy.overflow})` : ''}
|
|
1986
|
+
`content=${contentHeight ?? '?'}px${overflow ? ` OVERFLOW (${frame.policy.overflow})` : ''} ` +
|
|
1987
|
+
`ink=${inkRatio ?? '?'}${blank !== null ? ' BLANK' : ''} | no judge`,
|
|
1382
1988
|
);
|
|
1383
1989
|
}
|
|
1384
1990
|
return { status: 'measured', issues, readings };
|