@ggui-ai/ui-gen 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/base.d.ts +7 -0
- package/dist/adapters/base.js +1 -0
- package/dist/adapters/base.js.map +1 -1
- package/dist/adapters/generation-dispatch.d.ts +17 -3
- package/dist/adapters/generation-dispatch.js +992 -3468
- package/dist/adapters/generation-dispatch.js.map +1 -1
- package/dist/adapters/index.js +70 -13
- package/dist/adapters/index.js.map +1 -1
- package/dist/adapters/types.d.ts +1 -1
- package/dist/adapters/types.js.map +1 -1
- package/dist/advanced/index.d.ts +4 -4
- package/dist/advanced/index.js +1060 -3520
- package/dist/advanced/index.js.map +1 -1
- package/dist/blueprint-validator.js +13 -3
- package/dist/blueprint-validator.js.map +1 -1
- package/dist/boilerplate.js +2 -2
- package/dist/boilerplate.js.map +1 -1
- package/dist/check/index.d.ts +1 -1
- package/dist/check/index.js.map +1 -1
- package/dist/{create-ui-generator-CxPvudbs.d.ts → create-ui-generator-D7kOXNjJ.d.ts} +2 -2
- package/dist/{dispatch-N8wdNOX-.d.ts → dispatch-DGvhmARp.d.ts} +1 -1
- package/dist/evaluation/axis-checks/index.d.ts +3 -3
- package/dist/evaluation/axis-checks/index.js +46 -0
- package/dist/evaluation/axis-checks/index.js.map +1 -1
- package/dist/evaluation/axis-checks/registry.d.ts +1 -1
- package/dist/evaluation/axis-checks/registry.js +46 -0
- package/dist/evaluation/axis-checks/registry.js.map +1 -1
- package/dist/evaluation/index.d.ts +38 -6
- package/dist/evaluation/index.js +682 -3357
- package/dist/evaluation/index.js.map +1 -1
- package/dist/harness/check/runtime-render/index.d.ts +2 -2
- package/dist/harness/check/runtime-render/index.js +13 -3
- package/dist/harness/check/runtime-render/index.js.map +1 -1
- package/dist/harness/check/runtime-render/render-check-worker.js +8 -3
- package/dist/harness/check/runtime-render/render-check-worker.js.map +1 -1
- package/dist/harness/index.d.ts +2 -2
- package/dist/harness/index.js +562 -30
- package/dist/harness/index.js.map +1 -1
- package/dist/harness/llm-trace-sink.d.ts +7 -1
- package/dist/harness/llm-trace-sink.js.map +1 -1
- package/dist/harness/result-types.d.ts +1 -1
- package/dist/harness/runtime.d.ts +11 -2
- package/dist/harness/runtime.js +3 -3
- package/dist/harness/runtime.js.map +1 -1
- package/dist/harness/types-public.d.ts +1 -1
- package/dist/harness/types-public.js +2 -2
- package/dist/harness/types-public.js.map +1 -1
- package/dist/harness/validator-trace-sink.d.ts +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +1060 -3520
- package/dist/index.js.map +1 -1
- package/dist/{llm-router-D7fjBhtr.d.ts → llm-router-DwwUK1bz.d.ts} +16 -3
- package/dist/patch.d.ts +1 -1
- package/dist/providers/index.js +5 -4
- package/dist/providers/index.js.map +1 -1
- package/dist/{render-check-WfOVO9kU.d.ts → render-check-C0CqVceF.d.ts} +17 -0
- package/dist/{types-public-DKR_h70H.d.ts → types-public-CzYqZclY.d.ts} +44 -1
- package/dist/validation/index.d.ts +1 -1
- package/dist/validation/index.js +1 -1
- package/dist/validation/index.js.map +1 -1
- package/dist/workflows.d.ts +1 -1
- package/package.json +16 -12
- package/src/adapters/base.ts +8 -0
- package/src/adapters/claude/claude-code-login.ts +73 -0
- package/src/adapters/claude/sdk.ts +8 -1
- package/src/adapters/generation-dispatch.ts +19 -2
- package/src/adapters/openai/raw.ts +11 -5
- package/src/adapters/openai/sdk.ts +8 -5
- package/src/adapters/openai/tokens.ts +67 -0
- package/src/adapters/provider-router.ts +28 -0
- package/src/adapters/types.ts +1 -1
- package/src/boilerplate/generate.ts +19 -4
- package/src/boilerplate/hard-sections.ts +1 -1
- package/src/create-ui-generator.ts +19 -10
- package/src/evaluation/axis-checks/checks/universal.ts +92 -0
- package/src/evaluation/design-identity.ts +16 -5
- package/src/evaluation/types-public.ts +42 -0
- package/src/evaluation/visual-evaluator.ts +231 -24
- package/src/harness/check/runtime-render/adapter.ts +9 -0
- package/src/harness/check/runtime-render/render-check.ts +37 -1
- package/src/harness/claude-code-login-query.ts +528 -0
- package/src/harness/coding/assemble-result.ts +10 -1
- package/src/harness/coding/run-eval-round.ts +179 -34
- package/src/harness/enforced-coding.ts +20 -0
- package/src/harness/llm-router.ts +390 -55
- package/src/harness/llm-trace-sink.ts +7 -1
- package/src/harness/runtime.ts +6 -0
- package/src/providers/openai.ts +31 -19
- package/src/validation/primitives.ts +1 -1
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
// that carries the same rendering inputs.
|
|
15
15
|
import { createHash } from 'node:crypto';
|
|
16
16
|
import { readFileSync, readdirSync, statSync } from 'node:fs';
|
|
17
|
-
import { resolve } from 'node:path';
|
|
17
|
+
import { relative, resolve } from 'node:path';
|
|
18
18
|
|
|
19
19
|
export interface JudgeDesignIdentity {
|
|
20
20
|
/** Absolute path of the design `src` tree the judge bundled against. */
|
|
@@ -37,20 +37,31 @@ export function isDesignRenderingInput(relativePath: string): boolean {
|
|
|
37
37
|
return true;
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* sha256 over every rendering input under `root` (sorted relative path +
|
|
42
|
+
* contents) — the identity of a design tree as it paints.
|
|
43
|
+
*
|
|
44
|
+
* A function of the TREE, not of the spelling of `root`: the relative keys
|
|
45
|
+
* are taken against the RESOLVED root, so `…/design/src`, `…/design/src/` and
|
|
46
|
+
* `…/ui-gen/../design/src` hash the same bytes. Before this the key was
|
|
47
|
+
* sliced by the unresolved root's length while each entry had already been
|
|
48
|
+
* resolved, so an un-normalised root produced a different hash for the same
|
|
49
|
+
* tree — a phantom design skew for any caller composing the path.
|
|
50
|
+
*/
|
|
41
51
|
export function designTreeSha256(root: string): string {
|
|
52
|
+
const base = resolve(root);
|
|
42
53
|
const files: string[] = [];
|
|
43
54
|
const walk = (dir: string): void => {
|
|
44
55
|
for (const name of readdirSync(dir).sort()) {
|
|
45
56
|
const full = resolve(dir, name);
|
|
46
57
|
if (statSync(full).isDirectory()) walk(full);
|
|
47
|
-
else if (isDesignRenderingInput(
|
|
58
|
+
else if (isDesignRenderingInput(relative(base, full))) files.push(full);
|
|
48
59
|
}
|
|
49
60
|
};
|
|
50
|
-
walk(
|
|
61
|
+
walk(base);
|
|
51
62
|
const h = createHash('sha256');
|
|
52
63
|
for (const f of files) {
|
|
53
|
-
h.update(
|
|
64
|
+
h.update(relative(base, f) + '\n');
|
|
54
65
|
h.update(readFileSync(f));
|
|
55
66
|
h.update('\n');
|
|
56
67
|
}
|
|
@@ -123,6 +123,24 @@ export interface CriterionCoverage {
|
|
|
123
123
|
readonly reason?: string;
|
|
124
124
|
}
|
|
125
125
|
|
|
126
|
+
/**
|
|
127
|
+
* Coverage of the screenshot JUDGE leg for one eval round (ggui#1221) — a
|
|
128
|
+
* different surface from the `visual` CRITERION in `criteriaCoverage` (the
|
|
129
|
+
* LLM evaluator's read of the source). `ran` = a frame was captured and
|
|
130
|
+
* judged; `skipped` = the leg was configured but produced no verdict
|
|
131
|
+
* (no browser, a launch or in-page failure, an unbundlable component, an
|
|
132
|
+
* unparsable judge answer) with `reason` naming why; `not-applicable` =
|
|
133
|
+
* the round had no visual leg configured. Before this row a skipped leg
|
|
134
|
+
* left NO trace on the result — `{ issues: [] }` read exactly like a
|
|
135
|
+
* clean run, and every bootstrap mint on a browserless runtime "passed"
|
|
136
|
+
* its visual round that way.
|
|
137
|
+
*/
|
|
138
|
+
export interface VisualCoverage {
|
|
139
|
+
readonly status: CriterionRunStatus;
|
|
140
|
+
/** Populated for `skipped` / `not-applicable` — why no verdict. */
|
|
141
|
+
readonly reason?: string;
|
|
142
|
+
}
|
|
143
|
+
|
|
126
144
|
/**
|
|
127
145
|
* The static criteria `runLLMEvaluation` runs, in stamp order — the
|
|
128
146
|
* ONE list both the evaluator's coverage stamp and the harness's
|
|
@@ -185,6 +203,13 @@ export interface EvalResult {
|
|
|
185
203
|
* never here — this object is serialized into reports.
|
|
186
204
|
*/
|
|
187
205
|
visual?: VisualEvalSummary;
|
|
206
|
+
/**
|
|
207
|
+
* Whether the screenshot judge leg ran, was skipped (with the reason) or
|
|
208
|
+
* was not configured — ALWAYS stamped by the harness's eval round
|
|
209
|
+
* (ggui#1221), so a consumer can tell a clean visual run from a leg
|
|
210
|
+
* that silently could not run. See {@link VisualCoverage}.
|
|
211
|
+
*/
|
|
212
|
+
visualCoverage?: VisualCoverage;
|
|
188
213
|
}
|
|
189
214
|
|
|
190
215
|
/** One canvas's visual verdict, PNG-free — see `EvalResult.visual`. */
|
|
@@ -227,6 +252,21 @@ export interface CanvasVisualSummary {
|
|
|
227
252
|
fit?: 'fill';
|
|
228
253
|
}
|
|
229
254
|
|
|
255
|
+
/**
|
|
256
|
+
* ggui#1195 — the fit reading on the canvas whose policy FAILS an overflow
|
|
257
|
+
* (the inline chat card): the box it was judged at (`ceiling` — the order's
|
|
258
|
+
* declared viewport when one was carried, else the class box), whether that
|
|
259
|
+
* box was declared, and how many px of content sit below it (0 = fits).
|
|
260
|
+
* Present only when such a canvas was judged with a measurable height, so
|
|
261
|
+
* the bar and the ledger read one number for "does the hello fit".
|
|
262
|
+
*/
|
|
263
|
+
export interface VisualFitStamp {
|
|
264
|
+
readonly canvas: CanvasClass;
|
|
265
|
+
readonly ceiling: { readonly width: number; readonly height: number };
|
|
266
|
+
readonly declared: boolean;
|
|
267
|
+
readonly overflowPx: number;
|
|
268
|
+
}
|
|
269
|
+
|
|
230
270
|
/** The visual leg's per-canvas summary — see `EvalResult.visual`. */
|
|
231
271
|
export interface VisualEvalSummary {
|
|
232
272
|
/** Mean of the canvas scores (rounded). */
|
|
@@ -234,6 +274,8 @@ export interface VisualEvalSummary {
|
|
|
234
274
|
/** Every canvas passed. */
|
|
235
275
|
passed: boolean;
|
|
236
276
|
canvases: CanvasVisualSummary[];
|
|
277
|
+
/** ggui#1195 — see {@link VisualFitStamp}; absent when no fail-policy canvas was judged. */
|
|
278
|
+
fit?: VisualFitStamp;
|
|
237
279
|
/** The design tree the judge painted with (ggui#1042): `src` + `srcSha256` — `design@judge`. */
|
|
238
280
|
design?: { readonly src: string; readonly srcSha256: string };
|
|
239
281
|
/** The mode the judge's tokens were composed in, when the caller said (ggui#1076). */
|
|
@@ -24,7 +24,7 @@ import { createHash } from 'node:crypto';
|
|
|
24
24
|
import { tmpdir } from 'os';
|
|
25
25
|
import { createVisionAgent, type AgentConfig } from '../harness/llm-router';
|
|
26
26
|
import type { EvaluationResult, EvaluationIssue, DimensionScores } from './types';
|
|
27
|
-
import type { CanvasJudgeRecord, CanvasVisualSummary, EvalIssue, VisualEvalSummary } from './types-public.js';
|
|
27
|
+
import type { CanvasJudgeRecord, CanvasVisualSummary, EvalIssue, VisualCoverage, VisualEvalSummary, VisualFitStamp } from './types-public.js';
|
|
28
28
|
import type { LaunchOptions } from 'puppeteer-core';
|
|
29
29
|
import { CANVAS_VIEWPORTS, displayModeForCanvas, type CanvasClass, type CanvasViewport } from '../design-mode.js';
|
|
30
30
|
|
|
@@ -55,6 +55,20 @@ export interface VisualEvalConfig {
|
|
|
55
55
|
* single screenshot at `viewport`; no `canvases` field on the result.
|
|
56
56
|
*/
|
|
57
57
|
canvases?: readonly CanvasClass[];
|
|
58
|
+
/**
|
|
59
|
+
* ggui#1195 — the box a canvas is COMPOSED for and CAPTURED at when the
|
|
60
|
+
* order declared one (the chat card at the visitor's real column, not the
|
|
61
|
+
* class's 400×640). Keyed by canvas; a canvas without an entry is judged
|
|
62
|
+
* at `CANVAS_VIEWPORTS[canvas]` as before. The composer's
|
|
63
|
+
* `rendering.viewport` and this entry are ONE value written once (the
|
|
64
|
+
* seam derives it), so the judge never scores a box the mint did not
|
|
65
|
+
* target. Only read in per-canvas mode. Captured at WHOLE CSS px: the
|
|
66
|
+
* browser's device-metrics override takes integers (measured — a 383.5 px
|
|
67
|
+
* width is refused by CDP), so a fractional declared box is rounded once
|
|
68
|
+
* where it enters and the capture, the ceiling and the reported
|
|
69
|
+
* `viewport` all carry that same integer box.
|
|
70
|
+
*/
|
|
71
|
+
canvasViewports?: Partial<Readonly<Record<CanvasClass, CanvasViewport>>>;
|
|
58
72
|
/**
|
|
59
73
|
* Optional provider-routing override — see
|
|
60
74
|
* `AgentConfig.routeOverride`. Threaded onto the agent this
|
|
@@ -110,6 +124,8 @@ export interface CanvasVisualResult {
|
|
|
110
124
|
judge: CanvasJudgeRecord;
|
|
111
125
|
/** How the judge composed the mount (ggui#1100): `'fill'` on every fullscreen canvas, absent on the inline card. */
|
|
112
126
|
fit?: 'fill';
|
|
127
|
+
/** ggui#1195 — `true` when the order DECLARED this canvas's box (`config.canvasViewports` carried an entry), whatever its value. */
|
|
128
|
+
declared?: true;
|
|
113
129
|
}
|
|
114
130
|
|
|
115
131
|
/** How a canvas is captured for the judge and what an overflow means there (ggui#1027). */
|
|
@@ -158,6 +174,23 @@ export function canvasFit(canvas: CanvasClass): 'fill' | undefined {
|
|
|
158
174
|
return displayModeForCanvas(canvas) === 'fullscreen' ? 'fill' : undefined;
|
|
159
175
|
}
|
|
160
176
|
|
|
177
|
+
/**
|
|
178
|
+
* The box a browser can be asked for: whole CSS px. CDP's
|
|
179
|
+
* `Emulation.setDeviceMetricsOverride` deserialises width/height as int32 and
|
|
180
|
+
* refuses a fraction (measured on Chrome: `383.5` → "int32 value expected"),
|
|
181
|
+
* while a host measuring its chat column with `getBoundingClientRect()` may
|
|
182
|
+
* legitimately declare one. Rounded once, at the judge's boundary.
|
|
183
|
+
*/
|
|
184
|
+
export function integerBox(viewport: { width: number; height: number }): { width: number; height: number } {
|
|
185
|
+
return { width: Math.round(viewport.width), height: Math.round(viewport.height) };
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** ggui#1195 — `true` when a canvas was judged at a box other than its class viewport (the order declared one). */
|
|
189
|
+
export function isDeclaredViewport(canvas: CanvasClass, viewport: CanvasViewport): boolean {
|
|
190
|
+
const classBox = CANVAS_VIEWPORTS[canvas];
|
|
191
|
+
return viewport.width !== classBox.width || viewport.height !== classBox.height;
|
|
192
|
+
}
|
|
193
|
+
|
|
161
194
|
/** The deterministic fit issue — one per overflowing canvas, in the judge's issue shape so it rides the same channel. */
|
|
162
195
|
export function canvasOverflowIssue(
|
|
163
196
|
canvas: CanvasClass,
|
|
@@ -166,11 +199,18 @@ export function canvasOverflowIssue(
|
|
|
166
199
|
verdict: 'fail' | 'warn',
|
|
167
200
|
): EvaluationIssue {
|
|
168
201
|
const hidden = contentHeight - viewport.height;
|
|
202
|
+
// ggui#1195 — when the judge captured at a DECLARED box, the refusal names
|
|
203
|
+
// both boxes: the ceiling the content was measured against and the class
|
|
204
|
+
// box a reader would otherwise assume.
|
|
205
|
+
const classBox = CANVAS_VIEWPORTS[canvas];
|
|
206
|
+
const box = isDeclaredViewport(canvas, viewport)
|
|
207
|
+
? `declared ${viewport.width}×${viewport.height}; class box ${classBox.width}×${classBox.height}`
|
|
208
|
+
: `${viewport.width}×${viewport.height}`;
|
|
169
209
|
return {
|
|
170
210
|
dimension: 'canvas-overflow',
|
|
171
211
|
severity: verdict === 'fail' ? 'critical' : 'major',
|
|
172
212
|
description:
|
|
173
|
-
`Rendered content is ${contentHeight}px tall on the ${canvas} canvas (${
|
|
213
|
+
`Rendered content is ${contentHeight}px tall on the ${canvas} canvas (${box}) — ` +
|
|
174
214
|
`${hidden}px ${verdict === 'fail' ? 'is cut off: the inline card does not scroll' : 'sits below the first screen'}.`,
|
|
175
215
|
fix:
|
|
176
216
|
'Fit the composition to the canvas: fewer and shorter sections, one compact row of chips, no hero taller than ' +
|
|
@@ -427,7 +467,10 @@ function buildRenderHTML(
|
|
|
427
467
|
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
|
428
468
|
body {
|
|
429
469
|
font-family: var(--ggui-font-family-sans, system-ui, -apple-system, sans-serif);
|
|
430
|
-
|
|
470
|
+
/* ggui#1083 — the host stand-in (neutral-50) under the theme's scrim (its tint at
|
|
471
|
+
its opacity): the ground a frosted host puts under the card, so the judge scores
|
|
472
|
+
the card where the visitor sees it. */
|
|
473
|
+
background: color-mix(in oklch, var(--ggui-scrim-tint, var(--ggui-color-ground, #ffffff)) calc(var(--ggui-scrim-opacity, 0.45) * 100%), var(--ggui-color-neutral-50, #ffffff));
|
|
431
474
|
color: var(--ggui-color-neutral-900, #111827);
|
|
432
475
|
}
|
|
433
476
|
.error { color: #dc2626; padding: 16px; font-family: monospace; white-space: pre-wrap; }
|
|
@@ -523,18 +566,20 @@ export async function resolveLaunchOptions(
|
|
|
523
566
|
): Promise<LaunchOptions> {
|
|
524
567
|
const env = deps.env ?? process.env;
|
|
525
568
|
const override = env.PUPPETEER_EXECUTABLE_PATH?.trim();
|
|
569
|
+
// Whole px at the one place `defaultViewport` is built — the single-shot path takes the caller's box verbatim.
|
|
570
|
+
const defaultViewport = integerBox(viewport);
|
|
526
571
|
if (override) {
|
|
527
572
|
return {
|
|
528
573
|
executablePath: override,
|
|
529
574
|
args: [...EXECUTABLE_PATH_LAUNCH_ARGS],
|
|
530
|
-
defaultViewport
|
|
575
|
+
defaultViewport,
|
|
531
576
|
headless: true,
|
|
532
577
|
};
|
|
533
578
|
}
|
|
534
579
|
const chromium = await (deps.loadChromium ?? loadSparticuzChromium)();
|
|
535
580
|
return {
|
|
536
581
|
args: chromium.args,
|
|
537
|
-
defaultViewport
|
|
582
|
+
defaultViewport,
|
|
538
583
|
executablePath: await chromium.executablePath(),
|
|
539
584
|
headless: true,
|
|
540
585
|
};
|
|
@@ -742,6 +787,51 @@ export interface VisualEvalDetailed {
|
|
|
742
787
|
}
|
|
743
788
|
|
|
744
789
|
|
|
790
|
+
/**
|
|
791
|
+
* One canvas, framed: the box it is captured at (ggui#1195 — the DECLARED box
|
|
792
|
+
* when the order carried one, else the class box), the canvas's fit policy,
|
|
793
|
+
* the runtime's fit for the mount (ggui#1100 — fullscreen canvases fill, the
|
|
794
|
+
* inline card does not) and the capture. The judge path and the fit-only path
|
|
795
|
+
* ({@link runVisualFit}) frame through this ONE function, so a fit verdict
|
|
796
|
+
* reached without the judge is taken on the frame the judge would have seen.
|
|
797
|
+
*/
|
|
798
|
+
interface CanvasFrame {
|
|
799
|
+
readonly viewport: CanvasViewport;
|
|
800
|
+
readonly policy: CanvasFitPolicy;
|
|
801
|
+
readonly fit: 'fill' | undefined;
|
|
802
|
+
/** `true` when `canvasViewports` carried an entry for this canvas, whatever its value. */
|
|
803
|
+
readonly declared: boolean;
|
|
804
|
+
readonly attempt: ScreenshotAttempt;
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
async function frameCanvas(
|
|
808
|
+
bundledCode: string,
|
|
809
|
+
context: VisualEvalContext,
|
|
810
|
+
html: string,
|
|
811
|
+
canvas: CanvasClass,
|
|
812
|
+
canvasViewports: VisualEvalConfig['canvasViewports'],
|
|
813
|
+
deps: ScreenshotDeps,
|
|
814
|
+
): Promise<CanvasFrame> {
|
|
815
|
+
const declaredBox = canvasViewports?.[canvas];
|
|
816
|
+
const viewport = declaredBox !== undefined ? integerBox(declaredBox) : CANVAS_VIEWPORTS[canvas];
|
|
817
|
+
const policy = canvasFitPolicy(canvas);
|
|
818
|
+
const fit = canvasFit(canvas);
|
|
819
|
+
const canvasHtml = fit !== undefined ? buildRenderHTML(bundledCode, context.cssTokens, fit) : html;
|
|
820
|
+
const attempt = await captureScreenshotDetailed(canvasHtml, viewport, deps, policy.capture);
|
|
821
|
+
return { viewport, policy, fit, declared: declaredBox !== undefined, attempt };
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
/**
|
|
825
|
+
* The deterministic fit verdict on one frame (ggui#1027): the issue the
|
|
826
|
+
* canvas's policy gives an overflow, or `null` when the content fits, could
|
|
827
|
+
* not be measured, or the canvas scrolls by design.
|
|
828
|
+
*/
|
|
829
|
+
function fitVerdict(canvas: CanvasClass, frame: CanvasFrame): EvaluationIssue | null {
|
|
830
|
+
const contentHeight = frame.attempt.contentHeight;
|
|
831
|
+
if (contentHeight === null || contentHeight <= frame.viewport.height || frame.policy.overflow === 'none') return null;
|
|
832
|
+
return canvasOverflowIssue(canvas, frame.viewport, contentHeight, frame.policy.overflow);
|
|
833
|
+
}
|
|
834
|
+
|
|
745
835
|
/** One judge call parsed, retried ONCE on a malformed answer (#1017); the FIRST reason is kept verbatim. */
|
|
746
836
|
type JudgedAnswer =
|
|
747
837
|
| { readonly kind: 'ok'; readonly result: EvaluationResult; readonly inputTokens: number; readonly outputTokens: number }
|
|
@@ -848,12 +938,8 @@ export async function runVisualEvaluationDetailed(
|
|
|
848
938
|
const perCanvas: CanvasVisualResult[] = [];
|
|
849
939
|
const perCanvasResults: EvaluationResult[] = [];
|
|
850
940
|
for (const canvas of config.canvases) {
|
|
851
|
-
const
|
|
852
|
-
const policy =
|
|
853
|
-
// ggui#1100: the page is composed per canvas with the runtime's fit — fullscreen canvases fill, the inline card does not.
|
|
854
|
-
const fit = canvasFit(canvas);
|
|
855
|
-
const canvasHtml = fit !== undefined ? buildRenderHTML(bundledCode, context.cssTokens, fit) : html;
|
|
856
|
-
const attempt = await captureScreenshotDetailed(canvasHtml, viewport, deps, policy.capture);
|
|
941
|
+
const frame = await frameCanvas(bundledCode, context, html, canvas, config.canvasViewports, deps);
|
|
942
|
+
const { viewport, policy, fit, attempt } = frame;
|
|
857
943
|
const screenshot = attempt.png;
|
|
858
944
|
if (!screenshot) {
|
|
859
945
|
const unavailableReason = attempt.reason ?? 'no browser available';
|
|
@@ -890,8 +976,9 @@ export async function runVisualEvaluationDetailed(
|
|
|
890
976
|
// The fit verdict (ggui#1027): deterministic, in the judge's issue channel.
|
|
891
977
|
const contentHeight = attempt.contentHeight;
|
|
892
978
|
const overflow = contentHeight !== null && contentHeight > viewport.height;
|
|
893
|
-
|
|
894
|
-
|
|
979
|
+
const fitIssue = fitVerdict(canvas, frame);
|
|
980
|
+
if (fitIssue !== null) {
|
|
981
|
+
result.issues.push(fitIssue);
|
|
895
982
|
if (policy.overflow === 'fail') result.passed = false;
|
|
896
983
|
}
|
|
897
984
|
perCanvasResults.push(result);
|
|
@@ -905,6 +992,7 @@ export async function runVisualEvaluationDetailed(
|
|
|
905
992
|
overflow,
|
|
906
993
|
judge: judgeRecord,
|
|
907
994
|
...(fit !== undefined ? { fit } : {}),
|
|
995
|
+
...(frame.declared ? { declared: true as const } : {}),
|
|
908
996
|
});
|
|
909
997
|
console.log(
|
|
910
998
|
`[visual-eval] canvas=${canvas} ${viewport.width}×${viewport.height} score=${result.finalScore}${k > 1 ? ` (median of ${judgeRecord.samples.length}/${k}: ${judgeRecord.samples.join(',')} σ=${judgeRecord.sigma})` : ''} ` +
|
|
@@ -960,6 +1048,11 @@ export async function runVisualEvaluationDetailed(
|
|
|
960
1048
|
* per-dimension means, `issues` = union (description prefixed with the
|
|
961
1049
|
* canvas), `critique` = one line per canvas, tokens summed.
|
|
962
1050
|
*/
|
|
1051
|
+
/** A canvas's issue as the aggregate carries it: the description prefixed `[<canvas>]`. */
|
|
1052
|
+
function canvasScopedIssue(canvas: CanvasClass, issue: EvaluationIssue): EvaluationIssue {
|
|
1053
|
+
return { ...issue, description: `[${canvas}] ${issue.description}` };
|
|
1054
|
+
}
|
|
1055
|
+
|
|
963
1056
|
function aggregateCanvasResults(
|
|
964
1057
|
perCanvas: readonly CanvasVisualResult[],
|
|
965
1058
|
results: readonly EvaluationResult[],
|
|
@@ -980,7 +1073,7 @@ function aggregateCanvasResults(
|
|
|
980
1073
|
let outputTokens = 0;
|
|
981
1074
|
results.forEach((r, i) => {
|
|
982
1075
|
const canvas = perCanvas[i]!.canvas;
|
|
983
|
-
for (const issue of r.issues) issues.push(
|
|
1076
|
+
for (const issue of r.issues) issues.push(canvasScopedIssue(canvas, issue));
|
|
984
1077
|
if (r.critique) critiques.push(`${canvas}: ${r.critique}`);
|
|
985
1078
|
inputTokens += r.inputTokens ?? 0;
|
|
986
1079
|
outputTokens += r.outputTokens ?? 0;
|
|
@@ -1010,10 +1103,24 @@ export function summarizeVisualResult(result: VisualEvaluationResult): VisualEva
|
|
|
1010
1103
|
judge: c.judge,
|
|
1011
1104
|
...(c.fit !== undefined ? { fit: c.fit } : {}),
|
|
1012
1105
|
}));
|
|
1106
|
+
// ggui#1195 — the fit stamp: the first canvas whose policy FAILS an
|
|
1107
|
+
// overflow (the inline card), judged with a measurable height.
|
|
1108
|
+
const fitCanvas = result.canvases.find((c) => canvasFitPolicy(c.canvas).overflow === 'fail' && c.contentHeight !== null);
|
|
1109
|
+
const fit: VisualFitStamp | undefined =
|
|
1110
|
+
fitCanvas !== undefined && fitCanvas.contentHeight !== null
|
|
1111
|
+
? {
|
|
1112
|
+
canvas: fitCanvas.canvas,
|
|
1113
|
+
ceiling: { width: fitCanvas.viewport.width, height: fitCanvas.viewport.height },
|
|
1114
|
+
// Declared = the order carried a box, even one equal to the class box (the review's point).
|
|
1115
|
+
declared: fitCanvas.declared === true,
|
|
1116
|
+
overflowPx: Math.max(0, fitCanvas.contentHeight - fitCanvas.viewport.height),
|
|
1117
|
+
}
|
|
1118
|
+
: undefined;
|
|
1013
1119
|
return {
|
|
1014
1120
|
score: result.finalScore,
|
|
1015
1121
|
passed: result.passed,
|
|
1016
1122
|
canvases,
|
|
1123
|
+
...(fit !== undefined ? { fit } : {}),
|
|
1017
1124
|
...(result.design !== undefined ? { design: result.design } : {}),
|
|
1018
1125
|
...(result.themeMode !== undefined ? { themeMode: result.themeMode } : {}),
|
|
1019
1126
|
};
|
|
@@ -1148,6 +1255,12 @@ export interface VisualEvalOutcome {
|
|
|
1148
1255
|
* the single-shot path (byte-identical to the pre-canvas result).
|
|
1149
1256
|
*/
|
|
1150
1257
|
summary?: VisualEvalSummary;
|
|
1258
|
+
/**
|
|
1259
|
+
* ggui#1221 — whether the judge leg ran or was skipped (with the reason
|
|
1260
|
+
* `runVisualEvaluationDetailed` names). Always present: the absence of a
|
|
1261
|
+
* summary used to be indistinguishable from a clean single-shot run.
|
|
1262
|
+
*/
|
|
1263
|
+
coverage: VisualCoverage;
|
|
1151
1264
|
}
|
|
1152
1265
|
|
|
1153
1266
|
/**
|
|
@@ -1157,19 +1270,113 @@ export interface VisualEvalOutcome {
|
|
|
1157
1270
|
export async function runVisualEval(
|
|
1158
1271
|
context: VisualEvalContext,
|
|
1159
1272
|
config: VisualEvalConfig,
|
|
1273
|
+
deps: VisualEvalDeps = {},
|
|
1160
1274
|
): Promise<VisualEvalOutcome> {
|
|
1161
|
-
const
|
|
1162
|
-
|
|
1275
|
+
const detailed = await runVisualEvaluationDetailed(context, config, deps);
|
|
1276
|
+
const result = detailed.result;
|
|
1277
|
+
if (!result) {
|
|
1278
|
+
// ggui#1221 — no browser / launch failure / unbundlable / unparsable judge:
|
|
1279
|
+
// the leg is SKIPPED, and says so on the outcome instead of leaving `{ issues: [] }`.
|
|
1280
|
+
const reason = detailed.unavailableReason ?? 'no browser available';
|
|
1281
|
+
return { issues: [], coverage: { status: 'skipped', reason } };
|
|
1282
|
+
}
|
|
1163
1283
|
|
|
1164
|
-
const issues: EvalIssue[] = (result.issues || []).map(
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1284
|
+
const issues: EvalIssue[] = (result.issues || []).map(toEvalIssue);
|
|
1285
|
+
const summary = summarizeVisualResult(result);
|
|
1286
|
+
const coverage: VisualCoverage = { status: 'ran' };
|
|
1287
|
+
return summary === undefined ? { issues, coverage } : { issues, summary, coverage };
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
/** A judge-shaped issue as the harness's tier-2 issue: critical ⇒ a blocking fail, anything else ⇒ a warn. */
|
|
1291
|
+
function toEvalIssue(issue: EvaluationIssue): EvalIssue {
|
|
1292
|
+
return {
|
|
1293
|
+
tier: 2,
|
|
1294
|
+
result: issue.severity === 'critical' ? 'fail' : 'warn',
|
|
1295
|
+
category: 'visual',
|
|
1168
1296
|
subcategory: issue.dimension,
|
|
1169
|
-
severity:
|
|
1297
|
+
severity: issue.severity === 'critical' ? 'critical' : 'major',
|
|
1170
1298
|
description: issue.description,
|
|
1171
1299
|
fix: issue.fix || '',
|
|
1172
|
-
}
|
|
1173
|
-
|
|
1174
|
-
|
|
1300
|
+
};
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
// ---------------------------------------------------------------------------
|
|
1304
|
+
// The fit half without the judge
|
|
1305
|
+
// ---------------------------------------------------------------------------
|
|
1306
|
+
|
|
1307
|
+
/** What the fit half reads — the frame's inputs; nothing about a judge. */
|
|
1308
|
+
export type VisualFitConfig = Pick<VisualEvalConfig, 'sampleProps' | 'canvases' | 'canvasViewports' | 'designSrcDir'>;
|
|
1309
|
+
|
|
1310
|
+
/** One canvas's fit reading: the box, the measure, whether it overflowed — no score. */
|
|
1311
|
+
export interface CanvasFitReading {
|
|
1312
|
+
readonly canvas: CanvasClass;
|
|
1313
|
+
readonly viewport: CanvasViewport;
|
|
1314
|
+
readonly contentHeight: number | null;
|
|
1315
|
+
readonly overflow: boolean;
|
|
1316
|
+
/** `true` when the order declared this canvas's box. */
|
|
1317
|
+
readonly declared?: true;
|
|
1318
|
+
}
|
|
1319
|
+
|
|
1320
|
+
/** `measured` = every canvas was framed (its fit issues, possibly none); `unavailable` = no verdict, with why. */
|
|
1321
|
+
export type VisualFitOutcome =
|
|
1322
|
+
| { readonly status: 'measured'; readonly issues: EvalIssue[]; readonly readings: CanvasFitReading[] }
|
|
1323
|
+
| { readonly status: 'unavailable'; readonly reason: string };
|
|
1324
|
+
|
|
1325
|
+
/**
|
|
1326
|
+
* The visual leg's deterministic half, taken WITHOUT the vision judge.
|
|
1327
|
+
*
|
|
1328
|
+
* The fit verdict needs a browser and nothing else: each canvas is framed at
|
|
1329
|
+
* its box (the declared one when the order carried it), its scroll height is
|
|
1330
|
+
* measured against the box's height, and the canvas's fit policy decides what
|
|
1331
|
+
* an overflow earns — the same frame ({@link frameCanvas}), the same measure
|
|
1332
|
+
* and the same issue ({@link fitVerdict}, canvas-scoped) as the judge path, so
|
|
1333
|
+
* the two reach one verdict. What is absent is the score. A caller whose
|
|
1334
|
+
* generation provider has no vision judge takes this half alone, so a chat
|
|
1335
|
+
* card cut off at its declared box is still reported in the loop, where the
|
|
1336
|
+
* round can buy the composition fix.
|
|
1337
|
+
*
|
|
1338
|
+
* Per-canvas only: without `canvases` there is no box to hold the content
|
|
1339
|
+
* against, and the outcome is `unavailable`. A frame that cannot be taken is
|
|
1340
|
+
* `unavailable` with the capture's reason — never a pass.
|
|
1341
|
+
*/
|
|
1342
|
+
export async function runVisualFit(
|
|
1343
|
+
context: VisualEvalContext,
|
|
1344
|
+
config: VisualFitConfig,
|
|
1345
|
+
deps: ScreenshotDeps = {},
|
|
1346
|
+
): Promise<VisualFitOutcome> {
|
|
1347
|
+
if (config.canvases === undefined || config.canvases.length === 0) {
|
|
1348
|
+
return { status: 'unavailable', reason: 'no canvases — the fit verdict is taken per canvas' };
|
|
1349
|
+
}
|
|
1350
|
+
const designSrc = config.designSrcDir ?? resolve(resolveDesignPackageDir(), 'src');
|
|
1351
|
+
let bundledCode: string;
|
|
1352
|
+
try {
|
|
1353
|
+
bundledCode = await bundleForRendering(context.compiledCode, config.sampleProps ?? {}, designSrc);
|
|
1354
|
+
} catch (e) {
|
|
1355
|
+
return { status: 'unavailable', reason: `bundle failed: ${e instanceof Error ? e.message : String(e)}` };
|
|
1356
|
+
}
|
|
1357
|
+
const html = buildRenderHTML(bundledCode, context.cssTokens);
|
|
1358
|
+
const issues: EvalIssue[] = [];
|
|
1359
|
+
const readings: CanvasFitReading[] = [];
|
|
1360
|
+
for (const canvas of config.canvases) {
|
|
1361
|
+
const frame = await frameCanvas(bundledCode, context, html, canvas, config.canvasViewports, deps);
|
|
1362
|
+
if (!frame.attempt.png) {
|
|
1363
|
+
return { status: 'unavailable', reason: `${frame.attempt.reason ?? 'no browser available'} at canvas ${canvas}` };
|
|
1364
|
+
}
|
|
1365
|
+
const contentHeight = frame.attempt.contentHeight;
|
|
1366
|
+
const overflow = contentHeight !== null && contentHeight > frame.viewport.height;
|
|
1367
|
+
const verdict = fitVerdict(canvas, frame);
|
|
1368
|
+
if (verdict !== null) issues.push(toEvalIssue(canvasScopedIssue(canvas, verdict)));
|
|
1369
|
+
readings.push({
|
|
1370
|
+
canvas,
|
|
1371
|
+
viewport: frame.viewport,
|
|
1372
|
+
contentHeight,
|
|
1373
|
+
overflow,
|
|
1374
|
+
...(frame.declared ? { declared: true as const } : {}),
|
|
1375
|
+
});
|
|
1376
|
+
console.log(
|
|
1377
|
+
`[visual-fit] canvas=${canvas} ${frame.viewport.width}×${frame.viewport.height} ` +
|
|
1378
|
+
`content=${contentHeight ?? '?'}px${overflow ? ` OVERFLOW (${frame.policy.overflow})` : ''} | no judge`,
|
|
1379
|
+
);
|
|
1380
|
+
}
|
|
1381
|
+
return { status: 'measured', issues, readings };
|
|
1175
1382
|
}
|
|
@@ -203,6 +203,15 @@ function toEvalIssue(issue: RenderCheckIssue): EvalIssue | null {
|
|
|
203
203
|
if (diag?.actionsFiredFromClicks?.length) {
|
|
204
204
|
diagParts.push(`other actions fired from clicks: ${diag.actionsFiredFromClicks.join(", ")}`);
|
|
205
205
|
}
|
|
206
|
+
if (diag?.inputPriming) {
|
|
207
|
+
// ggui#1187: an input-gated action that "did not dispatch" reads
|
|
208
|
+
// differently beside "primed 0" / a priming error than beside "primed 1".
|
|
209
|
+
diagParts.push(
|
|
210
|
+
"primed" in diag.inputPriming
|
|
211
|
+
? `input priming: ${diag.inputPriming.primed} control(s) filled before the click`
|
|
212
|
+
: `input priming failed: ${diag.inputPriming.error}`,
|
|
213
|
+
);
|
|
214
|
+
}
|
|
206
215
|
const diagSuffix = diagParts.length ? ` [observed: ${diagParts.join("; ")}]` : "";
|
|
207
216
|
|
|
208
217
|
switch (issue.check) {
|
|
@@ -45,6 +45,7 @@ import {
|
|
|
45
45
|
type ParentLink,
|
|
46
46
|
} from "./host-boundary.js";
|
|
47
47
|
import { hostGlobals, openRecord } from "../../../internal/open-record.js";
|
|
48
|
+
import { primeInputs } from "@ggui-ai/ui-visual-tester/prime-inputs";
|
|
48
49
|
|
|
49
50
|
/**
|
|
50
51
|
* Module-namespace-shaped value injected into `loadComponent`'s
|
|
@@ -98,9 +99,25 @@ export interface RenderCheckIssue {
|
|
|
98
99
|
readonly actionsFiredFromClicks?: readonly string[];
|
|
99
100
|
/** Surfaced to the agent: prop sourceTool hints from the contract (only if present). */
|
|
100
101
|
readonly sourceToolHints?: Readonly<Record<string, string>>;
|
|
102
|
+
/**
|
|
103
|
+
* For action-wiring (ggui#1187): what input priming did before the
|
|
104
|
+
* synthetic dispatch — how many controls it filled, or why it could not
|
|
105
|
+
* run. A "did not dispatch it" on an input-gated action reads differently
|
|
106
|
+
* beside `{ primed: 0 }` or `{ error }` than beside `{ primed: 1 }`.
|
|
107
|
+
*/
|
|
108
|
+
readonly inputPriming?: InputPrimingDiagnostic;
|
|
101
109
|
};
|
|
102
110
|
}
|
|
103
111
|
|
|
112
|
+
/**
|
|
113
|
+
* Outcome of `primeInputs` before the action-wiring probe (ggui#1187). A
|
|
114
|
+
* priming failure never aborts the check — it is recorded here, on every
|
|
115
|
+
* action-wiring issue, so it is visible in the feedback instead of vanishing.
|
|
116
|
+
*/
|
|
117
|
+
export type InputPrimingDiagnostic =
|
|
118
|
+
| { readonly primed: number }
|
|
119
|
+
| { readonly error: string };
|
|
120
|
+
|
|
104
121
|
export interface RenderCheckResult {
|
|
105
122
|
readonly ok: boolean;
|
|
106
123
|
readonly issues: readonly RenderCheckIssue[];
|
|
@@ -504,6 +521,25 @@ export async function runRenderCheckInProcess(
|
|
|
504
521
|
let streamsChecked = 0;
|
|
505
522
|
|
|
506
523
|
try {
|
|
524
|
+
// ggui#1187: prime form inputs before the action-wiring probe. A wired
|
|
525
|
+
// Send/Submit gated on a non-empty input (the chat-interface shape) is a
|
|
526
|
+
// no-op on empty, so an un-primed synthetic click never dispatches and
|
|
527
|
+
// the probe false-warns "did not dispatch it". Shared with the visual
|
|
528
|
+
// tester's primeInputs (#1021/#1040) so both prime identically. Priming
|
|
529
|
+
// sets input VALUES (not textContent) and fires input/change before the
|
|
530
|
+
// per-candidate fire-log window is opened, so it neither perturbs the
|
|
531
|
+
// prop-sensitivity text baseline nor false-positives the wiring probe.
|
|
532
|
+
// A priming failure never aborts the check — it is RECORDED on every
|
|
533
|
+
// action-wiring issue as `diagnostics.inputPriming`. (The first cut
|
|
534
|
+
// swallowed it, and a `ReferenceError` in the check worker's realm —
|
|
535
|
+
// no `HTMLInputElement` on `globalThis` — hid behind that catch while
|
|
536
|
+
// every chat cell kept warning.)
|
|
537
|
+
let inputPriming: InputPrimingDiagnostic;
|
|
538
|
+
try {
|
|
539
|
+
inputPriming = { primed: primeInputs(renderResult.container) };
|
|
540
|
+
} catch (err) {
|
|
541
|
+
inputPriming = { error: err instanceof Error ? err.message : String(err) };
|
|
542
|
+
}
|
|
507
543
|
// ── Check 2: Action wiring (BLOCK / unverified-as-warn) ───────────
|
|
508
544
|
if (input.contract?.actionSpec) {
|
|
509
545
|
const actionSpec = input.contract.actionSpec;
|
|
@@ -522,7 +558,7 @@ export async function runRenderCheckInProcess(
|
|
|
522
558
|
probe,
|
|
523
559
|
user,
|
|
524
560
|
});
|
|
525
|
-
if (issue) issues.push(issue);
|
|
561
|
+
if (issue) issues.push({ ...issue, diagnostics: { ...issue.diagnostics, inputPriming } });
|
|
526
562
|
}
|
|
527
563
|
}
|
|
528
564
|
|