@ggui-ai/ui-gen 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/generation-dispatch.d.ts +21 -4
- package/dist/adapters/generation-dispatch.js +915 -351
- package/dist/adapters/generation-dispatch.js.map +1 -1
- package/dist/adapters/index.js +28 -23
- package/dist/adapters/index.js.map +1 -1
- package/dist/advanced/index.d.ts +17 -12
- package/dist/advanced/index.js +968 -375
- package/dist/advanced/index.js.map +1 -1
- package/dist/boilerplate.d.ts +16 -1
- package/dist/boilerplate.js +579 -245
- package/dist/boilerplate.js.map +1 -1
- package/dist/check/index.d.ts +17 -3
- package/dist/check/index.js +34 -26
- package/dist/check/index.js.map +1 -1
- package/dist/{create-ui-generator-D6TKuv9V.d.ts → create-ui-generator-CCIeQaEe.d.ts} +77 -10
- package/dist/design-mode-BHEE7flj.d.ts +67 -0
- package/dist/{dispatch-BuZfTh9r.d.ts → dispatch-BA5dJcg-.d.ts} +4 -1
- package/dist/evaluation/axis-checks/index.d.ts +4 -3
- package/dist/evaluation/axis-checks/index.js +3 -1
- package/dist/evaluation/axis-checks/index.js.map +1 -1
- package/dist/evaluation/axis-checks/registry.d.ts +2 -1
- package/dist/evaluation/axis-checks/registry.js +1 -0
- package/dist/evaluation/axis-checks/registry.js.map +1 -1
- package/dist/evaluation/index.d.ts +143 -3
- package/dist/evaluation/index.js +3866 -92
- package/dist/evaluation/index.js.map +1 -1
- package/dist/{generate-BTfvELgl.d.ts → generate-8L_bc2iP.d.ts} +10 -1
- package/dist/harness/check/runtime-render/index.d.ts +2 -1
- package/dist/harness/index.d.ts +2 -1
- package/dist/harness/index.js +589 -248
- package/dist/harness/index.js.map +1 -1
- package/dist/harness/llm-trace-sink.d.ts +3 -3
- package/dist/harness/llm-trace-sink.js.map +1 -1
- package/dist/harness/prompts.d.ts +2 -1
- package/dist/harness/prompts.js +3 -0
- package/dist/harness/prompts.js.map +1 -1
- package/dist/harness/result-types.d.ts +17 -5
- package/dist/harness/result-types.js.map +1 -1
- package/dist/harness/runtime.d.ts +19 -3
- package/dist/harness/runtime.js +602 -261
- package/dist/harness/runtime.js.map +1 -1
- package/dist/harness/types-public.d.ts +28 -1
- package/dist/harness/types-public.js +589 -248
- package/dist/harness/types-public.js.map +1 -1
- package/dist/harness/validator-trace-sink.d.ts +2 -1
- package/dist/index.d.ts +8 -4
- package/dist/index.js +975 -373
- package/dist/index.js.map +1 -1
- package/dist/patch.d.ts +2 -1
- package/dist/{prompts-BVhtH3M1.d.ts → prompts-k6m6reqe.d.ts} +11 -1
- package/dist/providers/index.d.ts +2 -2
- package/dist/providers/index.js.map +1 -1
- package/dist/tools/get-primitives-ts.d.ts +1 -1
- package/dist/tools/get-primitives-ts.js +1 -1
- package/dist/tools/get-primitives-ts.js.map +1 -1
- package/dist/{types-public-CkNWEov9.d.ts → types-public-D24JLJgl.d.ts} +49 -5
- package/dist/validation/index.d.ts +1 -1
- package/dist/validation/index.js +4 -4
- package/dist/validation/index.js.map +1 -1
- package/dist/workflows.d.ts +2 -1
- package/package.json +12 -12
- package/src/adapters/generation-dispatch.ts +57 -4
- package/src/adapters/provider-router.ts +7 -6
- package/src/adapters/tools.ts +5 -5
- package/src/advanced/generator.ts +11 -10
- package/src/advanced/index.ts +5 -4
- package/src/boilerplate/free-design-prompt.ts +329 -0
- package/src/boilerplate/generate.ts +10 -1
- package/src/boilerplate/hard-sections.ts +242 -0
- package/src/boilerplate/render.ts +32 -8
- package/src/boilerplate/system-prompt.ts +81 -229
- package/src/boilerplate/templates/base-free.tsx.tmpl +24 -0
- package/src/check/run-tier0.ts +80 -20
- package/src/check/type-checker.ts +12 -2
- package/src/coding-agent/tools.ts +8 -2
- package/src/contract-context.ts +28 -0
- package/src/create-harness.ts +11 -0
- package/src/create-ui-generator.ts +71 -25
- package/src/design-mode.ts +182 -0
- package/src/design-system-docs.ts +17 -13
- package/src/evaluation/axis-checks/dispatch.ts +4 -0
- package/src/evaluation/axis-checks/extras.ts +5 -0
- package/src/evaluation/cost-tracker.ts +2 -3
- package/src/evaluation/index.ts +46 -2
- package/src/evaluation/llm-evaluator.ts +77 -21
- package/src/evaluation/types-public.ts +103 -8
- package/src/evaluation/visual-evaluator.ts +301 -29
- package/src/harness/coding/dupe-break.ts +0 -0
- package/src/harness/coding/init-session.ts +1 -0
- package/src/harness/coding/run-coding-turn.ts +3 -0
- package/src/harness/coding/run-eval-round.ts +15 -2
- package/src/harness/llm-trace-sink.ts +3 -3
- package/src/harness/result-types.ts +15 -3
- package/src/harness/runtime.ts +18 -0
- package/src/harness/types-public.ts +27 -0
- package/src/index.ts +26 -0
- package/src/internal/extract-call-signatures.ts +2 -2
- package/src/providers/bedrock.ts +4 -4
- package/src/run-check.ts +1 -0
- package/src/tools/get-primitives-ts.ts +1 -1
- package/src/validation/allowed-imports.ts +8 -2
- package/src/validation/primitives.ts +1 -1
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
// packages/ui-gen/src/design-mode.ts
|
|
2
|
+
//
|
|
3
|
+
// The ONE definition of the `designMode` option and the rendering-canvas
|
|
4
|
+
// vocabulary. Every triad surface (system prompt, boilerplate, tier-0
|
|
5
|
+
// gate, axis checks, evaluator criteria) AND the visual evaluator import
|
|
6
|
+
// these types from here — nothing re-declares the unions.
|
|
7
|
+
//
|
|
8
|
+
// - `constrained` (default) — today's triad: the component composes
|
|
9
|
+
// from the `@ggui-ai/design` primitives, prop enums and spacing
|
|
10
|
+
// scale, and tier-0 enforces the design vocabulary. Byte-identical
|
|
11
|
+
// to the pre-`designMode` behaviour (pinned by
|
|
12
|
+
// `design-mode.pin.test.ts`).
|
|
13
|
+
// - `free` — the design vocabulary is relaxed: raw HTML elements,
|
|
14
|
+
// inline styles and `<style>` blocks are allowed; the design package
|
|
15
|
+
// is importable but optional. Every HARD contract stays: the data
|
|
16
|
+
// contract via the boilerplate-typed wire hooks, the import
|
|
17
|
+
// allowlist + security bans, the host envelope, the budgets, the
|
|
18
|
+
// blueprint-reuse shape, and brand-bearing color via the closed
|
|
19
|
+
// `--ggui-color-*` token manifest (no literals, no fallbacks).
|
|
20
|
+
|
|
21
|
+
export const DESIGN_MODES = ["constrained", "free"] as const;
|
|
22
|
+
|
|
23
|
+
export type DesignMode = (typeof DESIGN_MODES)[number];
|
|
24
|
+
|
|
25
|
+
export const DEFAULT_DESIGN_MODE: DesignMode = "constrained";
|
|
26
|
+
|
|
27
|
+
// ─── Rendering canvas ───────────────────────────────────────────────────────
|
|
28
|
+
//
|
|
29
|
+
// ONE canvas vocabulary shared by the prompt input (the `free` prompt
|
|
30
|
+
// TELLS the agent its canvas; the `constrained` prompt keeps its
|
|
31
|
+
// shell/screen descriptors byte-for-byte) and the visual evaluator (one
|
|
32
|
+
// screenshot + judge score per class at `CANVAS_VIEWPORTS[class]`).
|
|
33
|
+
//
|
|
34
|
+
// A class is derived from DISPLAY MODE + width, never from width alone:
|
|
35
|
+
// an inline chat card is `xs-chat-card` whatever the host window
|
|
36
|
+
// measures (the bubble, not the window, bounds the component); a
|
|
37
|
+
// fullscreen-class surface follows its width through the three
|
|
38
|
+
// breakpoints. `canvasForViewportWidth` is monotonic in width per
|
|
39
|
+
// display mode (pinned by `design-mode.test.ts`).
|
|
40
|
+
|
|
41
|
+
export const CANVAS_CLASSES = [
|
|
42
|
+
"xs-chat-card",
|
|
43
|
+
"mobile-fullscreen-small",
|
|
44
|
+
"md",
|
|
45
|
+
"lg",
|
|
46
|
+
"xl",
|
|
47
|
+
] as const;
|
|
48
|
+
|
|
49
|
+
export type CanvasClass = (typeof CANVAS_CLASSES)[number];
|
|
50
|
+
|
|
51
|
+
export interface CanvasViewport {
|
|
52
|
+
readonly width: number;
|
|
53
|
+
readonly height: number;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** The viewport (CSS px) each class is rendered + judged at. */
|
|
57
|
+
export const CANVAS_VIEWPORTS: Readonly<Record<CanvasClass, CanvasViewport>> = {
|
|
58
|
+
"xs-chat-card": { width: 400, height: 640 },
|
|
59
|
+
"mobile-fullscreen-small": { width: 390, height: 844 },
|
|
60
|
+
md: { width: 768, height: 1024 },
|
|
61
|
+
lg: { width: 1024, height: 768 },
|
|
62
|
+
xl: { width: 1440, height: 900 },
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* The two display modes the canvas derivation distinguishes. `inline`
|
|
67
|
+
* is the chat card (MCP Apps `inline` display mode / the `chat` shell);
|
|
68
|
+
* `fullscreen` is every surface the component owns edge to edge or as
|
|
69
|
+
* a panel (`fullscreen`, `pip`, the `spatial` and `partial` shells) —
|
|
70
|
+
* there the width decides the class.
|
|
71
|
+
*/
|
|
72
|
+
export type CanvasDisplayMode = "inline" | "fullscreen";
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Map a shell name (`chat` / `fullscreen` / `spatial` / `partial`) or an
|
|
76
|
+
* MCP Apps display mode (`inline` / `fullscreen` / `pip`) onto the
|
|
77
|
+
* canvas display mode. Only `chat` and `inline` are the inline card;
|
|
78
|
+
* everything else is width-governed.
|
|
79
|
+
*/
|
|
80
|
+
export function canvasDisplayModeForShell(shell: string | undefined): CanvasDisplayMode {
|
|
81
|
+
return shell === "chat" || shell === "inline" ? "inline" : "fullscreen";
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Fullscreen-class breakpoints (inclusive lower bounds), CSS px. */
|
|
85
|
+
export const CANVAS_BREAKPOINTS = { md: 600, lg: 1024, xl: 1440 } as const;
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Derive the canvas class from display mode + container width.
|
|
89
|
+
*
|
|
90
|
+
* inline → `xs-chat-card` at every width
|
|
91
|
+
* fullscreen → width < 600 → `mobile-fullscreen-small`
|
|
92
|
+
* 600 – 1023 → `md`
|
|
93
|
+
* 1024 – 1439 → `lg`
|
|
94
|
+
* ≥ 1440 → `xl`
|
|
95
|
+
*/
|
|
96
|
+
export function canvasForViewportWidth(mode: CanvasDisplayMode, width: number): CanvasClass {
|
|
97
|
+
if (mode === "inline") return "xs-chat-card";
|
|
98
|
+
if (width < CANVAS_BREAKPOINTS.md) return "mobile-fullscreen-small";
|
|
99
|
+
if (width < CANVAS_BREAKPOINTS.lg) return "md";
|
|
100
|
+
if (width < CANVAS_BREAKPOINTS.xl) return "lg";
|
|
101
|
+
return "xl";
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Map today's shell × screen descriptors onto a canvas class when no
|
|
106
|
+
* width is known. `chat` is always the inline card; `spatial` is a
|
|
107
|
+
* ~600px floating panel (`md`); `fullscreen` follows the screen. Unknown
|
|
108
|
+
* values fall back to `lg` (the widest common fullscreen), matching the
|
|
109
|
+
* skeleton's `fullscreen` / `universal` defaults.
|
|
110
|
+
*/
|
|
111
|
+
export function canvasForRendering(
|
|
112
|
+
shellType: string | undefined,
|
|
113
|
+
screen: string | undefined,
|
|
114
|
+
): CanvasClass {
|
|
115
|
+
const shell = shellType ?? "fullscreen";
|
|
116
|
+
if (canvasDisplayModeForShell(shell) === "inline") return "xs-chat-card";
|
|
117
|
+
if (shell === "spatial") return "md";
|
|
118
|
+
switch (screen ?? "universal") {
|
|
119
|
+
case "mobile":
|
|
120
|
+
return "mobile-fullscreen-small";
|
|
121
|
+
case "tablet":
|
|
122
|
+
return "md";
|
|
123
|
+
default:
|
|
124
|
+
return "lg";
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export interface CanvasDescriptor {
|
|
129
|
+
/** Short human label. */
|
|
130
|
+
readonly label: string;
|
|
131
|
+
/** Inclusive width range the component must look right across, CSS px. */
|
|
132
|
+
readonly minWidthPx: number;
|
|
133
|
+
readonly maxWidthPx: number;
|
|
134
|
+
/** What the host draws around the component at this canvas. */
|
|
135
|
+
readonly host: string;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export const CANVAS_DESCRIPTORS: Readonly<Record<CanvasClass, CanvasDescriptor>> = {
|
|
139
|
+
"xs-chat-card": {
|
|
140
|
+
label: "extra-small chat card",
|
|
141
|
+
minWidthPx: 320,
|
|
142
|
+
maxWidthPx: 480,
|
|
143
|
+
host: "an inline card inside a chat message bubble — the host draws the card border, shadow and rounded corners; the component fills the bubble width and takes its natural height",
|
|
144
|
+
},
|
|
145
|
+
"mobile-fullscreen-small": {
|
|
146
|
+
label: "small mobile fullscreen",
|
|
147
|
+
minWidthPx: 320,
|
|
148
|
+
maxWidthPx: 599,
|
|
149
|
+
host: "a phone-sized viewport the component owns edge to edge — single column, touch targets at least 44px, the component manages its own scroll",
|
|
150
|
+
},
|
|
151
|
+
md: {
|
|
152
|
+
label: "medium panel",
|
|
153
|
+
minWidthPx: 600,
|
|
154
|
+
maxWidthPx: 1023,
|
|
155
|
+
host: "a tablet viewport or a floating panel (spatial / picture-in-picture) — one or two columns, medium density",
|
|
156
|
+
},
|
|
157
|
+
lg: {
|
|
158
|
+
label: "large fullscreen",
|
|
159
|
+
minWidthPx: 1024,
|
|
160
|
+
maxWidthPx: 1439,
|
|
161
|
+
host: "a desktop viewport the component owns edge to edge — multi-column layouts, hover states and pointer interactions are available",
|
|
162
|
+
},
|
|
163
|
+
xl: {
|
|
164
|
+
label: "extra-large fullscreen",
|
|
165
|
+
minWidthPx: 1440,
|
|
166
|
+
maxWidthPx: 1920,
|
|
167
|
+
host: "a wide desktop viewport the component owns edge to edge — cap content width or use the extra room deliberately; never stretch a single column across the whole width",
|
|
168
|
+
},
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* The explicit sentence the `free` prompt renders for a canvas — the
|
|
173
|
+
* agent is TOLD the canvas, it never guesses it.
|
|
174
|
+
*/
|
|
175
|
+
export function describeCanvas(canvas: CanvasClass): string {
|
|
176
|
+
const d = CANVAS_DESCRIPTORS[canvas];
|
|
177
|
+
return (
|
|
178
|
+
`This component renders on the \`${canvas}\` canvas (${d.label}): ` +
|
|
179
|
+
`${d.minWidthPx}–${d.maxWidthPx}px wide, ${d.host}. ` +
|
|
180
|
+
`Design for the whole range — the host may size the iframe anywhere inside it.`
|
|
181
|
+
);
|
|
182
|
+
}
|
|
@@ -36,20 +36,24 @@ These tokens adapt automatically to any theme (light, dark, branded). **ALWAYS u
|
|
|
36
36
|
|
|
37
37
|
| Token | CSS Variable | Default | Role |
|
|
38
38
|
|-------|-------------|---------|------|
|
|
39
|
-
|
|
|
40
|
-
|
|
|
41
|
-
|
|
|
42
|
-
|
|
|
39
|
+
| ground | var(--ggui-color-ground) | The page / canvas behind everything |
|
|
40
|
+
| onGround | var(--ggui-color-onGround) | Text on the canvas |
|
|
41
|
+
| container | var(--ggui-color-container) | A card, a bubble, a panel — the unit of content |
|
|
42
|
+
| onContainer | var(--ggui-color-onContainer) | Body text on a card |
|
|
43
|
+
| elevated | var(--ggui-color-elevated) | Menus, popovers, modals, toasts — floats above a card |
|
|
44
|
+
| onElevated | var(--ggui-color-onElevated) | Text on an elevated surface |
|
|
45
|
+
| sunken | var(--ggui-color-sunken) | Wells, inputs at rest, code blocks — recessed inside a card |
|
|
46
|
+
| onSunken | var(--ggui-color-onSunken) | Muted / secondary text |
|
|
43
47
|
| container | var(--ggui-color-container) | Primary-branded containers |
|
|
44
48
|
| onContainer | var(--ggui-color-onContainer) | Text on branded containers |
|
|
45
49
|
| outline | var(--ggui-color-outline) | Borders, dividers |
|
|
46
50
|
| outlineVariant | var(--ggui-color-outlineVariant) | Subtle borders |
|
|
47
51
|
|
|
48
52
|
**Usage pattern:**
|
|
49
|
-
- Page/section background → \`var(--ggui-color-
|
|
50
|
-
- Body text → \`var(--ggui-color-
|
|
51
|
-
- Card/panel background → \`var(--ggui-color-
|
|
52
|
-
- Secondary/muted text → \`var(--ggui-color-
|
|
53
|
+
- Page/section background → \`var(--ggui-color-ground)\`
|
|
54
|
+
- Body text → \`var(--ggui-color-onContainer)\`
|
|
55
|
+
- Card/panel background → \`var(--ggui-color-container)\`; recessed wells / inputs → \`var(--ggui-color-sunken)\`
|
|
56
|
+
- Secondary/muted text → \`var(--ggui-color-onSunken)\`
|
|
53
57
|
- Branded section/header → \`var(--ggui-color-container)\` bg + \`var(--ggui-color-onContainer)\` text
|
|
54
58
|
- Borders/dividers → \`var(--ggui-color-outline)\` or \`var(--ggui-color-outlineVariant)\`
|
|
55
59
|
|
|
@@ -66,11 +70,11 @@ Prefer the primitives' \`tone="success|warning|error|info"\` prop over hand-writ
|
|
|
66
70
|
- **NEVER** use rgba(), hsl(), or other CSS color functions with hardcoded values.
|
|
67
71
|
- **NEVER** use raw neutral-* or gray-* scale tokens (neutral-50, neutral-900, etc.) — these are internal to the theme and break in dark mode.
|
|
68
72
|
- **ALWAYS** use semantic tokens for text and backgrounds:
|
|
69
|
-
- Text: \`var(--ggui-color-
|
|
70
|
-
- Backgrounds: \`var(--ggui-color-
|
|
73
|
+
- Text: \`var(--ggui-color-onContainer)\` or \`var(--ggui-color-onSunken)\`
|
|
74
|
+
- Backgrounds: \`var(--ggui-color-container)\` or \`var(--ggui-color-sunken)\`
|
|
71
75
|
- Borders: \`var(--ggui-color-outline)\` or \`var(--ggui-color-outlineVariant)\`
|
|
72
76
|
- For branded elements use \`var(--ggui-color-primary-*)\` scale tokens — these ARE safe because primary adapts per theme.
|
|
73
|
-
- For card backgrounds use \`var(--ggui-color-
|
|
77
|
+
- For card backgrounds use \`var(--ggui-color-container)\` or \`var(--ggui-color-primary-50)\`
|
|
74
78
|
|
|
75
79
|
## Spacing
|
|
76
80
|
|
|
@@ -165,7 +169,7 @@ Use \`var(--ggui-spacing-N)\` for all padding, gap, and margin values. **Never u
|
|
|
165
169
|
\`\`\`
|
|
166
170
|
|
|
167
171
|
### Color usage guide
|
|
168
|
-
- **
|
|
172
|
+
- **Layering roles** (ground, container, elevated, sunken + their on* inks; outline): one kind of AREA each — use them for every background/text/border decision; they adapt to any theme
|
|
169
173
|
- **primary-50/100**: Section backgrounds, highlight strips, card headers
|
|
170
174
|
- **primary-200/300**: Borders, dividers, focus rings, input outlines
|
|
171
175
|
- **primary-500/600**: Icons, links, labels, badges, buttons, CTAs
|
|
@@ -174,7 +178,7 @@ Use \`var(--ggui-spacing-N)\` for all padding, gap, and margin values. **Never u
|
|
|
174
178
|
|
|
175
179
|
The primary palette is the app's brand — use it throughout (headers, accents, borders, interactive elements), not just on the submit button.
|
|
176
180
|
|
|
177
|
-
**Note:** Do NOT add fallback values to var() (e.g., var(--ggui-color-
|
|
181
|
+
**Note:** Do NOT add fallback values to var() (e.g., var(--ggui-color-container, #fafafa)). Just use var(--ggui-color-container) — the theme provides all values.
|
|
178
182
|
|
|
179
183
|
## Motion & Animation
|
|
180
184
|
|
|
@@ -7,12 +7,15 @@ import type { Classification } from "../../classifier/index.js";
|
|
|
7
7
|
import type { EvalIssue } from "../types-public.js";
|
|
8
8
|
import { REGISTRY } from "./registry.js";
|
|
9
9
|
import { matches, type AxisCheckInput } from "./types.js";
|
|
10
|
+
import type { DesignMode } from "../../design-mode.js";
|
|
10
11
|
|
|
11
12
|
export interface RunAxisChecksInput {
|
|
12
13
|
sourceCode: string;
|
|
13
14
|
compiledCode: string | null;
|
|
14
15
|
contract?: DataContract;
|
|
15
16
|
originalPrompt: string;
|
|
17
|
+
/** Which triad produced the source — see `AxisCheckInput.designMode`. */
|
|
18
|
+
designMode?: DesignMode;
|
|
16
19
|
}
|
|
17
20
|
|
|
18
21
|
export function runAxisChecks(
|
|
@@ -27,6 +30,7 @@ export function runAxisChecks(
|
|
|
27
30
|
...(input.contract !== undefined ? { contract: input.contract } : {}),
|
|
28
31
|
originalPrompt: input.originalPrompt,
|
|
29
32
|
classification,
|
|
33
|
+
...(input.designMode !== undefined ? { designMode: input.designMode } : {}),
|
|
30
34
|
};
|
|
31
35
|
|
|
32
36
|
const issues: EvalIssue[] = [];
|
|
@@ -145,6 +145,11 @@ const stepperAdopted: AxisCheck = {
|
|
|
145
145
|
values: ["multi-step"],
|
|
146
146
|
run(input: AxisCheckInput): EvalIssue[] {
|
|
147
147
|
if (input.compiledCode === null) return [];
|
|
148
|
+
// `free` design mode: the Stepper composite is optional (the prompt
|
|
149
|
+
// names it as a default, not a requirement), so the adoption
|
|
150
|
+
// mandate stands down. `layout.multi_step.state_present` — the
|
|
151
|
+
// behavioural rule — still fires in both modes.
|
|
152
|
+
if (input.designMode === "free") return [];
|
|
148
153
|
if (/<Stepper[\s/>]/.test(input.sourceCode)) return [];
|
|
149
154
|
return [
|
|
150
155
|
mkIssue(
|
|
@@ -15,9 +15,8 @@ import { MODEL_REGISTRY, type ModelId } from '@ggui-ai/protocol';
|
|
|
15
15
|
* This table used to hand-maintain its own per-1K prices, and drifted:
|
|
16
16
|
* on 2026-07-31 four of its six rows were wrong (Haiku 4.5 20% low,
|
|
17
17
|
* gemini-3-flash-preview 5-7x low, gpt-5.4-mini ~3x low, Luna 5x high
|
|
18
|
-
* after OpenAI's cut). Deriving from the registry —
|
|
19
|
-
*
|
|
20
|
-
* one place to update and one place that can be wrong.
|
|
18
|
+
* after OpenAI's cut). Deriving from the registry — whose Anthropic rows
|
|
19
|
+
* `@ggui-ai/protocol`'s own tests pin — means one place to update and one place that can be wrong.
|
|
21
20
|
*
|
|
22
21
|
* Resolution order mirrors the benchmark harness's
|
|
23
22
|
* `resolveJudgeCostModelId`: exact key, unique `/<model>` suffix, then
|
package/src/evaluation/index.ts
CHANGED
|
@@ -22,6 +22,8 @@ export type {
|
|
|
22
22
|
EvalOutcome,
|
|
23
23
|
EvalCategory,
|
|
24
24
|
EvalResult,
|
|
25
|
+
VisualEvalSummary,
|
|
26
|
+
CanvasVisualSummary,
|
|
25
27
|
EvalCriterion,
|
|
26
28
|
CriterionCoverage,
|
|
27
29
|
CriterionRunStatus,
|
|
@@ -39,6 +41,7 @@ export {
|
|
|
39
41
|
priorityForIssue,
|
|
40
42
|
isBlocked,
|
|
41
43
|
getActionableIssues,
|
|
44
|
+
criteriaFor,
|
|
42
45
|
getCriteriaByPriority,
|
|
43
46
|
getCriterionById,
|
|
44
47
|
getLLMCriteria,
|
|
@@ -84,5 +87,46 @@ export type { RunAxisChecksInput } from './axis-checks/index.js';
|
|
|
84
87
|
// rubric — scoring generation on criteria the generator was never taught
|
|
85
88
|
// is a constraint-alignment violation, so out-of-loop judges get this
|
|
86
89
|
// seam instead of a rubric extension.
|
|
87
|
-
export {
|
|
88
|
-
|
|
90
|
+
export {
|
|
91
|
+
callMultimodalLLM,
|
|
92
|
+
captureScreenshot,
|
|
93
|
+
resolveLaunchOptions,
|
|
94
|
+
runVisualEvaluation,
|
|
95
|
+
runVisualEval,
|
|
96
|
+
summarizeVisualResult,
|
|
97
|
+
} from './visual-evaluator.js';
|
|
98
|
+
export type {
|
|
99
|
+
VisualEvalConfig,
|
|
100
|
+
VisualEvalContext,
|
|
101
|
+
VisualEvalDeps,
|
|
102
|
+
VisualEvalOutcome,
|
|
103
|
+
VisualEvaluationResult,
|
|
104
|
+
CanvasVisualResult,
|
|
105
|
+
ScreenshotDeps,
|
|
106
|
+
ScreenshotLauncher,
|
|
107
|
+
ScreenshotBrowser,
|
|
108
|
+
ScreenshotPage,
|
|
109
|
+
ChromiumProvider,
|
|
110
|
+
} from './visual-evaluator.js';
|
|
111
|
+
|
|
112
|
+
// Canvas vocabulary — the per-canvas visual evaluation speaks it
|
|
113
|
+
// (`VisualEvalConfig.canvases`, `CanvasVisualResult.canvas`); re-exported
|
|
114
|
+
// here so a caller driving `runVisualEvaluation` from this subpath never
|
|
115
|
+
// has to reach the package root for the class names, viewports or
|
|
116
|
+
// descriptors.
|
|
117
|
+
export {
|
|
118
|
+
CANVAS_CLASSES,
|
|
119
|
+
CANVAS_VIEWPORTS,
|
|
120
|
+
CANVAS_BREAKPOINTS,
|
|
121
|
+
CANVAS_DESCRIPTORS,
|
|
122
|
+
canvasForViewportWidth,
|
|
123
|
+
canvasForRendering,
|
|
124
|
+
canvasDisplayModeForShell,
|
|
125
|
+
describeCanvas,
|
|
126
|
+
} from '../design-mode.js';
|
|
127
|
+
export type {
|
|
128
|
+
CanvasClass,
|
|
129
|
+
CanvasViewport,
|
|
130
|
+
CanvasDisplayMode,
|
|
131
|
+
CanvasDescriptor,
|
|
132
|
+
} from '../design-mode.js';
|
|
@@ -26,6 +26,7 @@ import { createAgent } from '../harness/llm-router';
|
|
|
26
26
|
import type { AgentConfig, LLMToolDef, LLMAgent } from '../harness/llm-router';
|
|
27
27
|
import type { JsonObject } from '@ggui-ai/protocol';
|
|
28
28
|
import { getCriterionById } from './types-public.js';
|
|
29
|
+
import { DEFAULT_DESIGN_MODE, type DesignMode } from '../design-mode.js';
|
|
29
30
|
|
|
30
31
|
// =============================================================================
|
|
31
32
|
// Helpers
|
|
@@ -66,6 +67,14 @@ export interface LLMEvalContext {
|
|
|
66
67
|
originalPrompt: string;
|
|
67
68
|
designContext?: string;
|
|
68
69
|
contract?: unknown;
|
|
70
|
+
/**
|
|
71
|
+
* Which triad produced the source. Selects the criteria text
|
|
72
|
+
* (`criteriaFor`) and the mother prompt's styling rules: in `free`
|
|
73
|
+
* mode the `tokens` criterion narrows to color and the `visual`
|
|
74
|
+
* criterion + "Design System Rules" block become arm-neutral. Absent
|
|
75
|
+
* means `constrained` (byte-identical to the pre-`designMode` prompt).
|
|
76
|
+
*/
|
|
77
|
+
designMode?: DesignMode;
|
|
69
78
|
}
|
|
70
79
|
|
|
71
80
|
export interface LLMEvalConfig {
|
|
@@ -108,14 +117,62 @@ export interface EvalContext {
|
|
|
108
117
|
originalPrompt: string;
|
|
109
118
|
contract?: unknown;
|
|
110
119
|
designSystemSummary?: string;
|
|
120
|
+
/** See `LLMEvalContext.designMode`. */
|
|
121
|
+
designMode?: DesignMode;
|
|
111
122
|
}
|
|
112
123
|
|
|
124
|
+
/**
|
|
125
|
+
* Arm-neutral styling rules for `designMode: 'free'` — replaces the
|
|
126
|
+
* constrained "Design System Rules" block. Mirrors the free prompt: color
|
|
127
|
+
* on the closed token manifest, everything else may be literal, raw HTML
|
|
128
|
+
* + CSS is a first-class choice.
|
|
129
|
+
*/
|
|
130
|
+
const FREE_STYLING_RULES = `## Styling Rules (important for evaluation)
|
|
131
|
+
- Color: every color must be a bare \`var(--ggui-color-*)\` token — a hardcoded #hex, rgb()/hsl() or CSS named color is a fail, and so is a literal fallback inside \`var(--ggui-*, …)\`.
|
|
132
|
+
- Spacing, typography, radius, shadow geometry and layout values MAY be literals (\`padding: '20px'\`, \`fontSize: 18\`, \`borderRadius: 12\`) — NEVER flag them.
|
|
133
|
+
- Raw HTML elements with inline styles or \`<style>\` blocks are a first-class choice — never flag "not using the design system" or reward using it.
|
|
134
|
+
- When \`@ggui-ai/design\` primitives ARE used, their enum/scale props (\`variant="primary"\`, \`size="lg"\`, \`gap="md"\`, \`tone="muted"\`, \`surface="accent"\`) are valid.
|
|
135
|
+
- One \`export default function Component\` — helper components as named functions above it are fine`;
|
|
136
|
+
|
|
137
|
+
const CONSTRAINED_STYLING_RULES = `## Design System Rules (important for evaluation)
|
|
138
|
+
- Spacing props (\`gap\`, \`padding\`, \`margin\`) take a t-shirt-scale name: \`gap="md"\`, \`padding="lg"\` (\`none|xs|sm|md|lg|xl|2xl\`). **These ARE design tokens** — each resolves to a \`--ggui-spacing-*\` variable. A raw \`var(--ggui-spacing-*)\` string is an accepted escape hatch. NEVER flag a scale name as "hardcoded". Only a numeric prop (\`padding={24}\`) or a raw CSS length (\`gap="13px"\`) bypasses the scale and should be warned.
|
|
139
|
+
- All colors must use CSS variables: \`color="var(--ggui-color-primary-600)"\` — hardcoded #hex or rgb() is a fail
|
|
140
|
+
- Component props that take enum/scale values — \`variant="primary"\`, \`size="lg"\`, \`gap="md"\`, \`padding="lg"\`, \`shadow="md"\`, \`radius="lg"\`, \`tone="muted"\`, \`surface="accent"\` — are all design system tokens and are always valid
|
|
141
|
+
- One \`export default function Component\` — helper components as named functions above it are fine`;
|
|
142
|
+
|
|
143
|
+
const CONSTRAINED_VISUAL_REFERENCE = `**visual**: Design system consistency.
|
|
144
|
+
- Uses CSS variables (var(--ggui-*)) for colors, not hardcoded values
|
|
145
|
+
- Consistent use of spacing tokens
|
|
146
|
+
- Proper use of component variants (primary, outline, ghost)
|
|
147
|
+
- Shadow, border-radius from design system tokens`;
|
|
148
|
+
|
|
149
|
+
const FREE_VISUAL_REFERENCE = `**visual**: Visual composition.
|
|
150
|
+
- Clear hierarchy (one focal point, headings distinct from body), consistent spacing rhythm
|
|
151
|
+
- Coherent palette drawn from the theme color tokens
|
|
152
|
+
- Fit to the canvas: fluid width, nothing cramped or stretched
|
|
153
|
+
- No reward or penalty for using (or not using) the design-system primitives`;
|
|
154
|
+
|
|
155
|
+
const CONSTRAINED_RAW_CLICK_TARGET_GAP = `- A raw \`<div>\` / \`<span>\` used as a click target without \`as={Clickable}\`
|
|
156
|
+
(no role, no keyboard) — or any hand-rolled interactive element.`;
|
|
157
|
+
|
|
158
|
+
const FREE_RAW_CLICK_TARGET_GAP = `- A raw \`<div>\` / \`<span>\` used as a click target with no \`role="button"\`,
|
|
159
|
+
\`tabIndex\` and keyboard handler — a real \`<button>\` / \`<a href>\` is the
|
|
160
|
+
correct raw element and is never a gap.`;
|
|
161
|
+
|
|
113
162
|
export function buildMotherPrompt(ctx: EvalContext): string {
|
|
114
163
|
const parts: string[] = [];
|
|
115
|
-
|
|
116
|
-
|
|
164
|
+
const designMode = ctx.designMode ?? DEFAULT_DESIGN_MODE;
|
|
165
|
+
const free = designMode === 'free';
|
|
166
|
+
|
|
167
|
+
parts.push(
|
|
168
|
+
free
|
|
169
|
+
? `You are evaluating a generated React component. It was composed freely — raw HTML elements, inline styles and <style> blocks are allowed and the ggui design package is optional — against a data contract and a theme-token color rule.
|
|
170
|
+
The code has already passed automated checks (compilation, security, imports, color tokens).
|
|
171
|
+
Your job is to evaluate ONE specific quality criterion per call.`
|
|
172
|
+
: `You are evaluating a generated React component built with the ggui design system.
|
|
117
173
|
The code has already passed automated checks (compilation, security, imports, design tokens).
|
|
118
|
-
Your job is to evaluate ONE specific quality criterion per call
|
|
174
|
+
Your job is to evaluate ONE specific quality criterion per call.`,
|
|
175
|
+
);
|
|
119
176
|
|
|
120
177
|
parts.push(`## About This Prompt
|
|
121
178
|
This system prompt has 2 sections:
|
|
@@ -173,11 +230,7 @@ ${JSON.stringify(ctx.contract).slice(0, 6000)}
|
|
|
173
230
|
${ctx.designSystemSummary}`);
|
|
174
231
|
}
|
|
175
232
|
|
|
176
|
-
parts.push(
|
|
177
|
-
- Spacing props (\`gap\`, \`padding\`, \`margin\`) take a t-shirt-scale name: \`gap="md"\`, \`padding="lg"\` (\`none|xs|sm|md|lg|xl|2xl\`). **These ARE design tokens** — each resolves to a \`--ggui-spacing-*\` variable. A raw \`var(--ggui-spacing-*)\` string is an accepted escape hatch. NEVER flag a scale name as "hardcoded". Only a numeric prop (\`padding={24}\`) or a raw CSS length (\`gap="13px"\`) bypasses the scale and should be warned.
|
|
178
|
-
- All colors must use CSS variables: \`color="var(--ggui-color-primary-600)"\` — hardcoded #hex or rgb() is a fail
|
|
179
|
-
- Component props that take enum/scale values — \`variant="primary"\`, \`size="lg"\`, \`gap="md"\`, \`padding="lg"\`, \`shadow="md"\`, \`radius="lg"\`, \`tone="muted"\`, \`surface="accent"\` — are all design system tokens and are always valid
|
|
180
|
-
- One \`export default function Component\` — helper components as named functions above it are fine
|
|
233
|
+
parts.push(`${free ? FREE_STYLING_RULES : CONSTRAINED_STYLING_RULES}
|
|
181
234
|
|
|
182
235
|
## Evaluation Criteria Reference
|
|
183
236
|
|
|
@@ -221,11 +274,7 @@ ${ctx.designSystemSummary}`);
|
|
|
221
274
|
- Error boundaries or error messages for failure cases
|
|
222
275
|
- Empty states ("No items found") when data arrays could be empty
|
|
223
276
|
|
|
224
|
-
|
|
225
|
-
- Uses CSS variables (var(--ggui-*)) for colors, not hardcoded values
|
|
226
|
-
- Consistent use of spacing tokens
|
|
227
|
-
- Proper use of component variants (primary, outline, ghost)
|
|
228
|
-
- Shadow, border-radius from design system tokens
|
|
277
|
+
${free ? FREE_VISUAL_REFERENCE : CONSTRAINED_VISUAL_REFERENCE}
|
|
229
278
|
|
|
230
279
|
## Primitive Accessibility (built-in — do NOT flag as missing)
|
|
231
280
|
|
|
@@ -259,8 +308,7 @@ role, aria-*, label, or keyboard support — it is already there:
|
|
|
259
308
|
to text is CORRECTLY silent — never ask for \`aria-hidden\`, it is there.
|
|
260
309
|
|
|
261
310
|
REAL accessibility gaps worth flagging:
|
|
262
|
-
|
|
263
|
-
(no role, no keyboard) — or any hand-rolled interactive element.
|
|
311
|
+
${free ? FREE_RAW_CLICK_TARGET_GAP : CONSTRAINED_RAW_CLICK_TARGET_GAP}
|
|
264
312
|
- A hand-rolled control whose state (checked / selected / expanded /
|
|
265
313
|
pressed) is conveyed ONLY by styling — e.g. a styled-div "checkbox"
|
|
266
314
|
with a check icon + strikethrough label but no \`role="checkbox"\` /
|
|
@@ -329,11 +377,15 @@ function makeTier2Tool(criterion: string): LLMToolDef {
|
|
|
329
377
|
// Per-criterion user prompts
|
|
330
378
|
// =============================================================================
|
|
331
379
|
|
|
332
|
-
function getCriterionPrompt(
|
|
380
|
+
function getCriterionPrompt(
|
|
381
|
+
criterion: string,
|
|
382
|
+
numberedSource: string,
|
|
383
|
+
designMode: DesignMode,
|
|
384
|
+
): string {
|
|
333
385
|
const sourceBlock = `\nSource code (with line numbers):\n\`\`\`tsx\n${numberedSource}\n\`\`\``;
|
|
334
386
|
|
|
335
387
|
// Use criteria.ts as single source of truth when available
|
|
336
|
-
const criterionDef = getCriterionById(criterion);
|
|
388
|
+
const criterionDef = getCriterionById(criterion, designMode);
|
|
337
389
|
if (criterionDef && criterionDef.tier > 0) {
|
|
338
390
|
return `${criterionDef.evalInstruction}
|
|
339
391
|
${sourceBlock}
|
|
@@ -489,7 +541,7 @@ Return your evaluation using the evaluate_visual tool.
|
|
|
489
541
|
|
|
490
542
|
Example outputs:
|
|
491
543
|
- Clean: {"result": "pass"}
|
|
492
|
-
- Minor issues: {"result": "warn", "issues": ["Line 45: hardcoded '#f3f4f6' should use var(--ggui-color-
|
|
544
|
+
- Minor issues: {"result": "warn", "issues": ["Line 45: hardcoded '#f3f4f6' should use var(--ggui-color-sunken)", "Line 80: numeric padding={16} should use a scale name like padding='md'"]}
|
|
493
545
|
- Severe: {"result": "fail", "issues": ["Component uses entirely custom styling with no design system tokens"]}
|
|
494
546
|
|
|
495
547
|
Use 'fail' only for pervasive design system violations. Use 'warn' for isolated cases.`;
|
|
@@ -553,10 +605,11 @@ async function evalCriterion(
|
|
|
553
605
|
numberedSource: string,
|
|
554
606
|
agent: LLMAgent,
|
|
555
607
|
model: string,
|
|
608
|
+
designMode: DesignMode,
|
|
556
609
|
): Promise<CriterionResult> {
|
|
557
610
|
const start = Date.now();
|
|
558
611
|
const tool = tier === 1 ? makeTier1Tool(criterion) : makeTier2Tool(criterion);
|
|
559
|
-
const userPrompt = getCriterionPrompt(criterion, numberedSource);
|
|
612
|
+
const userPrompt = getCriterionPrompt(criterion, numberedSource, designMode);
|
|
560
613
|
|
|
561
614
|
try {
|
|
562
615
|
const response = await agent.callTools(
|
|
@@ -679,6 +732,7 @@ export async function preWarmEval(
|
|
|
679
732
|
originalPrompt: context.originalPrompt,
|
|
680
733
|
contract: context.contract,
|
|
681
734
|
designSystemSummary: context.designContext,
|
|
735
|
+
designMode: context.designMode,
|
|
682
736
|
});
|
|
683
737
|
|
|
684
738
|
return { motherPrompt, dynamicCriteria: [] };
|
|
@@ -777,10 +831,12 @@ export async function runLLMEvaluation(
|
|
|
777
831
|
: (providerName as 'openai' | 'google');
|
|
778
832
|
|
|
779
833
|
// Use pre-warmed context if available, otherwise build fresh
|
|
834
|
+
const designMode = context.designMode ?? DEFAULT_DESIGN_MODE;
|
|
780
835
|
const motherPrompt = preWarmedContext?.motherPrompt ?? buildMotherPrompt({
|
|
781
836
|
originalPrompt: context.originalPrompt,
|
|
782
837
|
contract: context.contract,
|
|
783
838
|
designSystemSummary: context.designContext,
|
|
839
|
+
designMode,
|
|
784
840
|
});
|
|
785
841
|
|
|
786
842
|
// Add line numbers to source
|
|
@@ -810,8 +866,8 @@ export async function runLLMEvaluation(
|
|
|
810
866
|
|
|
811
867
|
// Run ALL criteria in parallel: 7 static (2 tier 1 + 5 tier 2)
|
|
812
868
|
const results = await Promise.all([
|
|
813
|
-
...TIER_1_CRITERIA.map(c => evalCriterion(c, 1, motherPrompt, numberedSource, agent, model)),
|
|
814
|
-
...TIER_2_CRITERIA.map(c => evalCriterion(c, 2, motherPrompt, numberedSource, agent, model)),
|
|
869
|
+
...TIER_1_CRITERIA.map(c => evalCriterion(c, 1, motherPrompt, numberedSource, agent, model, designMode)),
|
|
870
|
+
...TIER_2_CRITERIA.map(c => evalCriterion(c, 2, motherPrompt, numberedSource, agent, model, designMode)),
|
|
815
871
|
]);
|
|
816
872
|
|
|
817
873
|
// Merge results
|