@ggui-ai/ui-gen 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/generation-dispatch.d.ts +9 -2
- package/dist/adapters/generation-dispatch.js +693 -89
- package/dist/adapters/generation-dispatch.js.map +1 -1
- package/dist/adapters/index.js +28 -11
- package/dist/adapters/index.js.map +1 -1
- package/dist/advanced/index.d.ts +5 -3
- package/dist/advanced/index.js +714 -101
- package/dist/advanced/index.js.map +1 -1
- package/dist/blueprint-validator.js +267 -3
- package/dist/blueprint-validator.js.map +1 -1
- package/dist/boilerplate.js +14 -2
- package/dist/boilerplate.js.map +1 -1
- package/dist/check/index.d.ts +1 -1
- package/dist/check/index.js +11 -1
- package/dist/check/index.js.map +1 -1
- package/dist/{create-ui-generator-B9kV4U9h.d.ts → create-ui-generator-CF0EL0Li.d.ts} +13 -1
- package/dist/{dispatch-Deo9qzrG.d.ts → dispatch-D75M-OfZ.d.ts} +1 -1
- package/dist/evaluation/axis-checks/index.d.ts +3 -3
- package/dist/evaluation/axis-checks/index.js +101 -0
- package/dist/evaluation/axis-checks/index.js.map +1 -1
- package/dist/evaluation/axis-checks/registry.d.ts +1 -1
- package/dist/evaluation/axis-checks/registry.js +101 -0
- package/dist/evaluation/axis-checks/registry.js.map +1 -1
- package/dist/evaluation/index.d.ts +46 -3
- package/dist/evaluation/index.js +358 -17
- package/dist/evaluation/index.js.map +1 -1
- package/dist/harness/check/runtime-render/index.d.ts +2 -2
- package/dist/harness/check/runtime-render/index.js +268 -3
- package/dist/harness/check/runtime-render/index.js.map +1 -1
- package/dist/harness/check/runtime-render/render-check-worker.d.ts +2 -0
- package/dist/harness/check/runtime-render/render-check-worker.js +1332 -0
- package/dist/harness/check/runtime-render/render-check-worker.js.map +1 -0
- package/dist/harness/index.d.ts +2 -2
- package/dist/harness/index.js +263 -18
- package/dist/harness/index.js.map +1 -1
- package/dist/harness/llm-trace-sink.d.ts +1 -1
- package/dist/harness/llm-trace-sink.js.map +1 -1
- package/dist/harness/prompts.js +17 -3
- package/dist/harness/prompts.js.map +1 -1
- package/dist/harness/result-types.d.ts +1 -1
- package/dist/harness/runtime.d.ts +9 -2
- package/dist/harness/runtime.js +20 -6
- package/dist/harness/runtime.js.map +1 -1
- package/dist/harness/types-public.d.ts +1 -1
- package/dist/harness/types-public.js +14 -2
- package/dist/harness/types-public.js.map +1 -1
- package/dist/harness/validator-trace-sink.d.ts +1 -1
- package/dist/index.d.ts +5 -3
- package/dist/index.js +708 -95
- package/dist/index.js.map +1 -1
- package/dist/{llm-router-1TyI0_TV.d.ts → llm-router-BEVtbAb9.d.ts} +61 -4
- package/dist/patch.d.ts +1 -1
- package/dist/provider-adapter.d.ts +16 -6
- package/dist/provider-adapter.js.map +1 -1
- package/dist/providers/index.d.ts +19 -6
- package/dist/providers/index.js.map +1 -1
- package/dist/{render-check-CNZjHD5g.d.ts → render-check-BZoIE8-u.d.ts} +21 -1
- package/dist/{types-public-C36k1q6s.d.ts → types-public-DIpjC-OA.d.ts} +1 -1
- package/dist/workflows.d.ts +1 -1
- package/package.json +12 -12
- package/src/adapters/generation-dispatch.ts +9 -0
- package/src/adapters/openrouter/client.ts +2 -2
- package/src/adapters/openrouter/types.ts +10 -0
- package/src/boilerplate/generate.ts +10 -2
- package/src/boilerplate/json-schema-ts.ts +22 -2
- package/src/boilerplate/system-prompt.ts +1 -0
- package/src/compile.ts +1 -1
- package/src/contract-context.ts +10 -2
- package/src/create-ui-generator.ts +14 -0
- package/src/design-system-docs.ts +6 -4
- package/src/evaluation/axis-checks/checks/state-ui-affordance.ts +162 -0
- package/src/evaluation/axis-checks/registry.ts +2 -0
- package/src/evaluation/index.ts +11 -0
- package/src/evaluation/llm-evaluator.ts +14 -1
- package/src/evaluation/visual-evaluator.ts +44 -65
- package/src/harness/check/runtime-render/adapter.ts +18 -0
- package/src/harness/check/runtime-render/index.ts +8 -0
- package/src/harness/check/runtime-render/prepare-mockup.ts +14 -0
- package/src/harness/check/runtime-render/production-act-shim.ts +158 -0
- package/src/harness/check/runtime-render/render-check-host.ts +201 -0
- package/src/harness/check/runtime-render/render-check-worker.ts +80 -0
- package/src/harness/check/runtime-render/render-check.ts +112 -1
- package/src/harness/coding/run-eval-round.ts +61 -5
- package/src/harness/index.ts +1 -1
- package/src/harness/llm-router.ts +397 -18
- package/src/harness/llm-trace-sink.ts +2 -1
- package/src/harness/runtime.ts +7 -0
- package/src/index.ts +5 -5
- package/src/provider-adapter.ts +16 -6
- package/src/providers/bedrock.ts +5 -4
- package/src/providers/index.ts +16 -4
- package/src/tools/render-check.ts +19 -7
|
@@ -127,6 +127,16 @@ export class OpenRouterError extends Error {
|
|
|
127
127
|
message: string,
|
|
128
128
|
public readonly status: number,
|
|
129
129
|
public readonly code?: string,
|
|
130
|
+
/**
|
|
131
|
+
* Response headers, when the caller had access to them at throw
|
|
132
|
+
* time (#489). Only `client.ts`'s two HTTP-response-driven throw
|
|
133
|
+
* sites populate this — the "no response body" throw has no
|
|
134
|
+
* response to read headers from. Duck-typed against the same
|
|
135
|
+
* `{ get(name): string | null }` shape `llm-router.ts`'s
|
|
136
|
+
* `extractRetryAfterSec` already reads from Anthropic/OpenAI SDK
|
|
137
|
+
* errors, so OpenRouter gets retry-after support for free.
|
|
138
|
+
*/
|
|
139
|
+
public readonly headers?: { get(name: string): string | null },
|
|
130
140
|
) {
|
|
131
141
|
super(message);
|
|
132
142
|
this.name = 'OpenRouterError';
|
|
@@ -191,14 +191,22 @@ export function generateBoilerplate(
|
|
|
191
191
|
const propsFields: string[] = [];
|
|
192
192
|
const propsProperties = contract?.propsSpec?.properties ?? {};
|
|
193
193
|
for (const [key, entry] of Object.entries(propsProperties)) {
|
|
194
|
-
|
|
194
|
+
// Required is OPT-IN (`required === true`) — the same reading the
|
|
195
|
+
// wire gate applies (`contract-validator.ts` builds the JSON-Schema
|
|
196
|
+
// `required` list from `=== true`), the contract summary's `!`
|
|
197
|
+
// marker uses, and coverage annotates with. The old `!== false`
|
|
198
|
+
// reading typed omitted-`required` props as always-present, so a
|
|
199
|
+
// render the gate legitimately accepts (agent omits an optional
|
|
200
|
+
// prop) crashed on `props.x.map(...)` — ggui#528.
|
|
201
|
+
const required = entry.required === true;
|
|
195
202
|
const nullable = entry.schema.nullable === true;
|
|
196
203
|
const tsType = jsonSchemaTypeToTs(entry.schema);
|
|
197
204
|
const fullType = nullable ? `${tsType} | null` : tsType;
|
|
198
|
-
// Build comment: description + default
|
|
205
|
+
// Build comment: description + default hint + optionality contract
|
|
199
206
|
const parts: string[] = [];
|
|
200
207
|
if (entry.description) parts.push(entry.description);
|
|
201
208
|
if (entry.default !== undefined) parts.push(`(default: ${JSON.stringify(entry.default)})`);
|
|
209
|
+
if (!required) parts.push("(optional — the agent may omit it; guard or fall back)");
|
|
202
210
|
const desc = parts.length > 0 ? ` // ${parts.join(' ')}` : "";
|
|
203
211
|
propsFields.push(` ${key}${required ? "" : "?"}: ${fullType};${desc}`);
|
|
204
212
|
}
|
|
@@ -35,6 +35,21 @@ export function jsonSchemaTypeToTs(schema: JsonSchema): string {
|
|
|
35
35
|
.join(' | ');
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
// ── Draft-07 type array (`['string','null']` — the canonical
|
|
39
|
+
// nullable form since draft-2026-08-19): members' TS union. Each
|
|
40
|
+
// member re-enters with the single type so object/array members
|
|
41
|
+
// keep their properties/items; `nullable` is folded in here (not in
|
|
42
|
+
// the recursion) so it can't double-append. ──
|
|
43
|
+
if (Array.isArray(schema.type)) {
|
|
44
|
+
const { nullable: _nullable, ...rest } = schema;
|
|
45
|
+
const members = [...new Set(schema.type)].map((member) =>
|
|
46
|
+
jsonSchemaTypeToTs({ ...rest, type: member }),
|
|
47
|
+
);
|
|
48
|
+
const unique = [...new Set(members)];
|
|
49
|
+
if (schema.nullable && !unique.includes('null')) unique.push('null');
|
|
50
|
+
return unique.join(' | ');
|
|
51
|
+
}
|
|
52
|
+
|
|
38
53
|
let result: string;
|
|
39
54
|
|
|
40
55
|
switch (schema.type) {
|
|
@@ -54,8 +69,13 @@ export function jsonSchemaTypeToTs(schema: JsonSchema): string {
|
|
|
54
69
|
case 'array': {
|
|
55
70
|
if (schema.items) {
|
|
56
71
|
const itemType = jsonSchemaTypeToTs(schema.items);
|
|
57
|
-
// Use Array<T> for complex object types, T[] for simple
|
|
58
|
-
|
|
72
|
+
// Use Array<T> for complex object types, T[] for simple.
|
|
73
|
+
// `items.type` may be a type SET — treat any set containing
|
|
74
|
+
// 'object' like the single-string form.
|
|
75
|
+
const itemsDeclareObject = Array.isArray(schema.items.type)
|
|
76
|
+
? schema.items.type.includes('object')
|
|
77
|
+
: schema.items.type === 'object';
|
|
78
|
+
result = (itemsDeclareObject && schema.items.properties)
|
|
59
79
|
? `Array<${itemType}>`
|
|
60
80
|
: (itemType.includes('|') ? `(${itemType})[]` : `${itemType}[]`);
|
|
61
81
|
} else {
|
|
@@ -349,6 +349,7 @@ A \`DataContract\` declares everything a GguiSession exchanges with the outside
|
|
|
349
349
|
|
|
350
350
|
Props arrive at mount and may be partial on first render; later prop changes reach the SAME mounted component via \`ggui_amend\` (in-place repaint) on the live channel. Stream channels start empty and fill over time. Context slots start at their declared default (often \`null\`). **Never assume a field exists before you read it.**
|
|
351
351
|
|
|
352
|
+
- **Optional contract props**: a \`Props\` field marked \`?\` in the boilerplate may be omitted by the agent ENTIRELY — the wire gate only enforces \`required: true\` entries, and the harness verifies your component with those props stripped. Guard them (\`?? default\`, \`?.\`, or an empty-state branch) from the FIRST render.
|
|
352
353
|
- **Array iteration**: always default to \`[]\` before \`.map\`/\`.filter\`/\`.length\`. Use \`(props.items ?? []).map(...)\` not \`props.items.map(...)\`. Same for stream.all, stream.latest, etc.
|
|
353
354
|
- **Object access**: optional-chain through nested fields. \`props.user?.name ?? 'Anonymous'\` not \`props.user.name\`.
|
|
354
355
|
- **Number ops**: default before arithmetic. \`(props.count ?? 0) + 1\` not \`props.count + 1\`.
|
package/src/compile.ts
CHANGED
|
@@ -153,7 +153,7 @@ export async function compileComponentCode(source: string): Promise<string> {
|
|
|
153
153
|
* `response.sourceCode` so downstream consumers that want human-readable
|
|
154
154
|
* source (benchmarks, blueprint cache seeding) still have it.
|
|
155
155
|
*
|
|
156
|
-
* const raw = createUiGenerator(
|
|
156
|
+
* const raw = createUiGenerator();
|
|
157
157
|
* const oss = withBrowserCompile(raw);
|
|
158
158
|
* const out = await oss.generate({ ... });
|
|
159
159
|
* // out.response.componentCode → ESM (browser-ready)
|
package/src/contract-context.ts
CHANGED
|
@@ -257,11 +257,19 @@ export function buildContractsContext(
|
|
|
257
257
|
const uiRequirements: string[] = [];
|
|
258
258
|
if (contract.propsSpec?.properties) {
|
|
259
259
|
for (const [name, entry] of Object.entries(contract.propsSpec.properties)) {
|
|
260
|
-
|
|
261
|
-
|
|
260
|
+
// Opt-in required (`=== true`) — the wire gate's reading, ggui#528.
|
|
261
|
+
// An omitted-`required` prop is OPTIONAL: the agent may leave it
|
|
262
|
+
// out of a legitimate render, so the derived instruction must
|
|
263
|
+
// demand a presence-safe component, not unconditional display.
|
|
264
|
+
const required = (entry as { required?: boolean }).required === true;
|
|
265
|
+
if (required) {
|
|
262
266
|
uiRequirements.push(
|
|
263
267
|
`- **Required prop \`${name}\`** must appear somewhere in rendered DOM (display the value, or use it to drive a label/aria/key).`,
|
|
264
268
|
);
|
|
269
|
+
} else {
|
|
270
|
+
uiRequirements.push(
|
|
271
|
+
`- **Optional prop \`${name}\`** may be ABSENT at runtime — the component must render without it (guard the access or apply its default) and must display it when it IS provided.`,
|
|
272
|
+
);
|
|
265
273
|
}
|
|
266
274
|
}
|
|
267
275
|
}
|
|
@@ -61,6 +61,7 @@ import {
|
|
|
61
61
|
import type { RenderingContext } from './contract-context.js';
|
|
62
62
|
import { resolveRoute, applyRouteToEnv } from './adapters/provider-router.js';
|
|
63
63
|
import type { QualityConfig } from './evaluation/types-public.js';
|
|
64
|
+
import type { AgentConfig } from './harness/llm-router.js';
|
|
64
65
|
|
|
65
66
|
/** The slug for the OSS default seed generator. */
|
|
66
67
|
const DEFAULT_TIER: GeneratorTier = 'default';
|
|
@@ -151,6 +152,17 @@ export interface CreateUiGeneratorOptions {
|
|
|
151
152
|
* for callers that don't need concurrent generations in-process.
|
|
152
153
|
*/
|
|
153
154
|
readonly disableEnvMutation?: boolean;
|
|
155
|
+
/**
|
|
156
|
+
* Retry observer (#489) — see `AgentConfig.onRetry`. Bound ONCE at
|
|
157
|
+
* factory construction time (mirrors `disableEnvMutation`), not
|
|
158
|
+
* per-call: `generate()` is invoked once per request, but the
|
|
159
|
+
* generator instance itself is typically long-lived and cached by
|
|
160
|
+
* the caller, so a single observer covers every generation it runs.
|
|
161
|
+
* Fires whenever any agent's `apiCall()` retries a rate-limited
|
|
162
|
+
* request. Absent (default) is a no-op — retries still happen and
|
|
163
|
+
* still log via `console.warn`, just without a caller-side hook.
|
|
164
|
+
*/
|
|
165
|
+
readonly onRetry?: AgentConfig['onRetry'];
|
|
154
166
|
}
|
|
155
167
|
|
|
156
168
|
export function createUiGenerator(
|
|
@@ -163,6 +175,7 @@ export function createUiGenerator(
|
|
|
163
175
|
qualityConfig,
|
|
164
176
|
gadgetCatalog,
|
|
165
177
|
disableEnvMutation = false,
|
|
178
|
+
onRetry,
|
|
166
179
|
} = options;
|
|
167
180
|
|
|
168
181
|
const identity = resolveIdentity(options);
|
|
@@ -302,6 +315,7 @@ export function createUiGenerator(
|
|
|
302
315
|
: {}),
|
|
303
316
|
enableRuntimeRender,
|
|
304
317
|
...(routeOverride !== undefined ? { routeOverride } : {}),
|
|
318
|
+
...(onRetry !== undefined ? { onRetry } : {}),
|
|
305
319
|
});
|
|
306
320
|
|
|
307
321
|
const metadata: GenerationMetadata = {
|
|
@@ -54,10 +54,12 @@ These tokens adapt automatically to any theme (light, dark, branded). **ALWAYS u
|
|
|
54
54
|
- Borders/dividers → \`var(--ggui-color-outline)\` or \`var(--ggui-color-outlineVariant)\`
|
|
55
55
|
|
|
56
56
|
### State Colors
|
|
57
|
-
|
|
58
|
-
- var(--ggui-color-
|
|
59
|
-
- var(--ggui-color-
|
|
60
|
-
- var(--ggui-color-
|
|
57
|
+
Semantic families ship as SCALES; the \`500\` stop is the role color (per-mode brightness), \`50\` the tint, \`800\` the on-tint text:
|
|
58
|
+
- var(--ggui-color-success-500) — success states (tint bg: success-50, text on tint: success-800)
|
|
59
|
+
- var(--ggui-color-warning-500) — warning states (warning-50 / warning-800)
|
|
60
|
+
- var(--ggui-color-error-500) — error states, destructive actions (error-50 / error-800)
|
|
61
|
+
- var(--ggui-color-info-500) — informational (info-50 / info-800)
|
|
62
|
+
Prefer the primitives' \`tone="success|warning|error|info"\` prop over hand-written vars — it resolves to the same stops.
|
|
61
63
|
|
|
62
64
|
### IMPORTANT: Color Rules
|
|
63
65
|
- **NEVER** use hardcoded hex colors. ONLY use var(--ggui-color-*) tokens.
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
// packages/ui-gen/src/evaluation/axis-checks/checks/state-ui-affordance.ts
|
|
2
|
+
//
|
|
3
|
+
// `state.ui_affordance.state_aria_present` (ggui#408) — a hand-rolled
|
|
4
|
+
// stateful control must expose its state in ARIA.
|
|
5
|
+
//
|
|
6
|
+
// Why this is a DETERMINISTIC check and not (only) an LLM criterion:
|
|
7
|
+
// the Experiment-53 baseline showed the defect class is structurally
|
|
8
|
+
// invisible to the LLM leg — `state=ui-affordance` with no writes
|
|
9
|
+
// classifies `riskTier=low`, and the low-risk bypass skips tier-1+2
|
|
10
|
+
// entirely, so 10/10 ARIA-less generations were never evaluated at
|
|
11
|
+
// all. Axis checks run BEFORE the bypass decision, and a `fail` among
|
|
12
|
+
// them disables the bypass for that round — which both surfaces the
|
|
13
|
+
// finding and routes it into the coding loop. Prototype validation
|
|
14
|
+
// against the 27-cell baseline corpus: precision 1.00, recall 0.80
|
|
15
|
+
// (both false negatives closed here: lowercase tags — `<div
|
|
16
|
+
// as={Clickable}>` is the WORSE variant since `as` is inert on a
|
|
17
|
+
// native element — and callback-prop indirection in extracted
|
|
18
|
+
// subcomponents).
|
|
19
|
+
//
|
|
20
|
+
// Detection predicate (v2 — "the element's own onClick mutates the
|
|
21
|
+
// affordance state"; v1's styling-dataflow precondition scored recall
|
|
22
|
+
// 0.30, do not resurrect it): an element fires when its `onClick`
|
|
23
|
+
// references a `useState` setter, a local handler that calls one, or
|
|
24
|
+
// an `on[A-Z]…` callback prop (extracted-subcomponent shape) — AND it
|
|
25
|
+
// is not a semantics-owning primitive, carries no state ARIA
|
|
26
|
+
// (`aria-checked/pressed/selected/expanded/current` or any `role=`),
|
|
27
|
+
// and contains no state-carrying primitive in its subtree (pushing
|
|
28
|
+
// duplicate ARIA onto a wrapper around `<Checkbox>` is itself a
|
|
29
|
+
// defect the judge fails — never demand it).
|
|
30
|
+
//
|
|
31
|
+
// Gate: `{axis: "state", values: ["ui-affordance"]}` ONLY. The
|
|
32
|
+
// predicate must not reach `state=payload/draft` (form-field wrappers
|
|
33
|
+
// with onClick would false-flag) or `state=merge` (kanban's per-item
|
|
34
|
+
// onClick is a MOVE, not a boolean affordance).
|
|
35
|
+
|
|
36
|
+
import type { EvalIssue } from "../../types-public.js";
|
|
37
|
+
import type { AxisCheck, AxisCheckInput } from "../types.js";
|
|
38
|
+
import { mkIssue } from "../helpers.js";
|
|
39
|
+
|
|
40
|
+
const STATE_ARIA_OR_ROLE = /aria-(checked|pressed|selected|expanded|current)\s*=|role\s*=/;
|
|
41
|
+
const STATE_PRIMITIVE_IN_SUBTREE = /<(Checkbox|Toggle|Tabs|Accordion)[\s/>]/;
|
|
42
|
+
/** Primitives that own their own interaction semantics — never flagged. */
|
|
43
|
+
const SEMANTIC_PRIMITIVES = new Set(["Button", "Checkbox", "Toggle", "Link", "MenuItem"]);
|
|
44
|
+
|
|
45
|
+
/** Collect `const [x, setX] = useState(...)` setter names. */
|
|
46
|
+
function collectSetters(src: string): Set<string> {
|
|
47
|
+
const setters = new Set<string>();
|
|
48
|
+
for (const m of src.matchAll(/const\s*\[\s*\w+\s*,\s*(set\w+)\s*\]\s*=\s*useState/g)) {
|
|
49
|
+
setters.add(m[1]);
|
|
50
|
+
}
|
|
51
|
+
return setters;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Collect local handler names whose body (nearby) calls a setter. */
|
|
55
|
+
function collectHandlers(src: string, setters: Set<string>): Set<string> {
|
|
56
|
+
const handlers = new Set<string>();
|
|
57
|
+
for (const m of src.matchAll(/const\s+(\w+)\s*=\s*(?:\([^)]*\)|\w+)\s*=>\s*\{?/g)) {
|
|
58
|
+
const body = src.slice(m.index + m[0].length, m.index + m[0].length + 400);
|
|
59
|
+
for (const s of setters) {
|
|
60
|
+
if (body.includes(s)) {
|
|
61
|
+
handlers.add(m[1]);
|
|
62
|
+
break;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return handlers;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** The attribute block of an opening tag: from after the tag name to its `>` (brace-aware). */
|
|
70
|
+
function attrBlock(src: string, start: number): { block: string; end: number } {
|
|
71
|
+
let depth = 0;
|
|
72
|
+
for (let i = start; i < src.length; i++) {
|
|
73
|
+
const c = src[i];
|
|
74
|
+
if (c === "{") depth++;
|
|
75
|
+
else if (c === "}") depth--;
|
|
76
|
+
else if (c === ">" && depth === 0) return { block: src.slice(start, i), end: i };
|
|
77
|
+
}
|
|
78
|
+
return { block: src.slice(start, start + 900), end: start + 900 };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** The brace-balanced expression of `onClick={…}`, or null when absent. */
|
|
82
|
+
function onClickValue(block: string): string | null {
|
|
83
|
+
const m = block.match(/onClick\s*=\s*\{/);
|
|
84
|
+
if (!m || m.index === undefined) return null;
|
|
85
|
+
const open = m.index + m[0].length - 1;
|
|
86
|
+
let depth = 0;
|
|
87
|
+
for (let i = open; i < block.length; i++) {
|
|
88
|
+
if (block[i] === "{") depth++;
|
|
89
|
+
else if (block[i] === "}") {
|
|
90
|
+
depth--;
|
|
91
|
+
if (depth === 0) return block.slice(open + 1, i);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return block.slice(open + 1);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function subtree(src: string, pos: number, tag: string): string {
|
|
98
|
+
const close = src.indexOf(`</${tag}>`, pos);
|
|
99
|
+
return src.slice(pos, close === -1 ? pos + 700 : close);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function runStateAriaPresent(input: AxisCheckInput): EvalIssue[] {
|
|
103
|
+
if (input.compiledCode === null) return [];
|
|
104
|
+
const src = input.sourceCode;
|
|
105
|
+
const setters = collectSetters(src);
|
|
106
|
+
const handlers = collectHandlers(src, setters);
|
|
107
|
+
const issues: EvalIssue[] = [];
|
|
108
|
+
const flaggedLines = new Set<number>();
|
|
109
|
+
|
|
110
|
+
// Locally-defined components: their CALL SITES can never satisfy the
|
|
111
|
+
// check (the ARIA lives inside the definition, which this check
|
|
112
|
+
// already covers via the callback-prop clause) — flagging the call
|
|
113
|
+
// site is a permanently-unresolvable finding that stalls the
|
|
114
|
+
// eval-fix loop in the recurrence guard (Experiment 54: exactly the
|
|
115
|
+
// 2/27 stuck cells). Skip call sites of components defined in this
|
|
116
|
+
// file; the definition's own inner element carries the check.
|
|
117
|
+
const localComponents = new Set<string>();
|
|
118
|
+
for (const m of src.matchAll(/(?:function\s+([A-Z]\w*)\s*\(|const\s+([A-Z]\w*)\s*=)/g)) {
|
|
119
|
+
localComponents.add(m[1] ?? m[2]);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
for (const m of src.matchAll(/<([A-Za-z][A-Za-z0-9]*)\b/g)) {
|
|
123
|
+
const tag = m[1];
|
|
124
|
+
if (m.index === undefined) continue;
|
|
125
|
+
if (localComponents.has(tag)) continue;
|
|
126
|
+
const { block, end } = attrBlock(src, m.index + m[0].length);
|
|
127
|
+
const clickValue = onClickValue(block);
|
|
128
|
+
if (clickValue === null) continue;
|
|
129
|
+
const mutatesAffordance =
|
|
130
|
+
[...setters].some((s) => clickValue.includes(s)) ||
|
|
131
|
+
[...handlers].some((h) => new RegExp(`\\b${h}\\b`).test(clickValue)) ||
|
|
132
|
+
// Extracted-subcomponent shape: the toggle handler arrives as an
|
|
133
|
+
// `on[A-Z]…` callback prop and is invoked from onClick — inside a
|
|
134
|
+
// ui-affordance component that IS the affordance mutation.
|
|
135
|
+
/\bon[A-Z]\w*\b/.test(clickValue);
|
|
136
|
+
if (!mutatesAffordance) continue;
|
|
137
|
+
if (SEMANTIC_PRIMITIVES.has(tag)) continue;
|
|
138
|
+
if (STATE_ARIA_OR_ROLE.test(block)) continue;
|
|
139
|
+
if (STATE_PRIMITIVE_IN_SUBTREE.test(subtree(src, end, tag))) continue;
|
|
140
|
+
|
|
141
|
+
const line = src.slice(0, m.index).split("\n").length;
|
|
142
|
+
if (flaggedLines.has(line)) continue;
|
|
143
|
+
flaggedLines.add(line);
|
|
144
|
+
issues.push(
|
|
145
|
+
mkIssue(
|
|
146
|
+
"state.ui_affordance.state_aria_present",
|
|
147
|
+
`Line ${line}: <${tag}> toggles the UI affordance from its onClick but exposes no state ARIA — its checked/selected/expanded state is invisible to assistive technology (styling is the only carrier).`,
|
|
148
|
+
`Add the matching state attribute to the SAME element that carries the onClick: \`role="checkbox"\` + \`aria-checked={…}\` for a toggle row, \`aria-pressed\` for a toggle button, \`aria-selected\` for a selection, \`aria-expanded\` for an open/close affordance — or replace the hand-rolled control with the state-carrying primitive (Checkbox / Toggle / Tabs / Accordion). \`as={Clickable}\` adds button + keyboard semantics but NOT state.`,
|
|
149
|
+
),
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
return issues;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export const STATE_UI_AFFORDANCE_CHECKS: readonly AxisCheck[] = [
|
|
156
|
+
{
|
|
157
|
+
id: "state.ui_affordance.state_aria_present",
|
|
158
|
+
axis: "state",
|
|
159
|
+
values: ["ui-affordance"],
|
|
160
|
+
run: runStateAriaPresent,
|
|
161
|
+
},
|
|
162
|
+
];
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import type { AxisCheck } from "./types.js";
|
|
7
7
|
import { UNIVERSAL_CHECKS } from "./checks/universal.js";
|
|
8
8
|
import { STATE_MERGE_CHECKS } from "./checks/state-merge.js";
|
|
9
|
+
import { STATE_UI_AFFORDANCE_CHECKS } from "./checks/state-ui-affordance.js";
|
|
9
10
|
import { REALTIME_CHECKS } from "./checks/realtime.js";
|
|
10
11
|
import { WRITES_CHECKS } from "./checks/writes.js";
|
|
11
12
|
import { STATE_PAYLOAD_CHECKS } from "./checks/state-payload.js";
|
|
@@ -15,6 +16,7 @@ import { EXTRA_CHECKS } from "./extras.js";
|
|
|
15
16
|
export const REGISTRY: readonly AxisCheck[] = [
|
|
16
17
|
...UNIVERSAL_CHECKS,
|
|
17
18
|
...STATE_MERGE_CHECKS,
|
|
19
|
+
...STATE_UI_AFFORDANCE_CHECKS,
|
|
18
20
|
...REALTIME_CHECKS,
|
|
19
21
|
...WRITES_CHECKS,
|
|
20
22
|
...STATE_PAYLOAD_CHECKS,
|
package/src/evaluation/index.ts
CHANGED
|
@@ -71,3 +71,14 @@ export type {
|
|
|
71
71
|
} from './types.js';
|
|
72
72
|
export { runAxisChecks } from './axis-checks/index.js';
|
|
73
73
|
export type { RunAxisChecksInput } from './axis-checks/index.js';
|
|
74
|
+
|
|
75
|
+
// The visual judge's raw multimodal call — arbitrary prompt + screenshot
|
|
76
|
+
// Buffer, routed through the same `createVisionAgent`/`apiCall()` choke
|
|
77
|
+
// point as every in-loop evaluation call (the #504 contract). Exposed so
|
|
78
|
+
// standalone visual-judging instruments (host-fit probes, external
|
|
79
|
+
// benches) can reuse the judging machinery WITHOUT entering the in-loop
|
|
80
|
+
// rubric — scoring generation on criteria the generator was never taught
|
|
81
|
+
// is a constraint-alignment violation, so out-of-loop judges get this
|
|
82
|
+
// seam instead of a rubric extension.
|
|
83
|
+
export { callMultimodalLLM } from './visual-evaluator.js';
|
|
84
|
+
export type { VisualEvalConfig } from './visual-evaluator.js';
|
|
@@ -71,6 +71,14 @@ export interface LLMEvalConfig {
|
|
|
71
71
|
* `process.env`. Absent (default) preserves existing behavior.
|
|
72
72
|
*/
|
|
73
73
|
routeOverride?: AgentConfig['routeOverride'];
|
|
74
|
+
/**
|
|
75
|
+
* Provider-429-retry observer (#489) — see `AgentConfig.onRetry`.
|
|
76
|
+
* Threaded onto the agent this evaluator constructs so a rate-limited
|
|
77
|
+
* retry inside an evaluation turn's `apiCall()` reaches the same
|
|
78
|
+
* caller the coding leg reaches. Absent (default) preserves existing
|
|
79
|
+
* behavior (retries still happen; no observer fires).
|
|
80
|
+
*/
|
|
81
|
+
onRetry?: AgentConfig['onRetry'];
|
|
74
82
|
}
|
|
75
83
|
|
|
76
84
|
/** A request-specific evaluation criterion generated by the LLM. */
|
|
@@ -756,7 +764,12 @@ export async function runLLMEvaluation(
|
|
|
756
764
|
// Pass the full config (not just the bare provider string) so a
|
|
757
765
|
// caller-supplied `routeOverride` (#484) reaches the constructed
|
|
758
766
|
// `LLMAgent` instance and it never falls back to `process.env`.
|
|
759
|
-
const agent = createAgent({
|
|
767
|
+
const agent = createAgent({
|
|
768
|
+
provider: routerProvider,
|
|
769
|
+
model,
|
|
770
|
+
routeOverride: config.routeOverride,
|
|
771
|
+
onRetry: config.onRetry,
|
|
772
|
+
});
|
|
760
773
|
|
|
761
774
|
// Dynamic criteria removed — they generated overly strict edge-case checks
|
|
762
775
|
// that wasted coding turns on nitpicks. Will be replaced by a holistic
|
|
@@ -16,6 +16,7 @@ import { resolve, dirname } from 'path';
|
|
|
16
16
|
import { fileURLToPath } from 'url';
|
|
17
17
|
import { writeFileSync, mkdirSync, unlinkSync } from 'fs';
|
|
18
18
|
import { tmpdir } from 'os';
|
|
19
|
+
import { createVisionAgent, type AgentConfig } from '../harness/llm-router';
|
|
19
20
|
import type { EvaluationResult, EvaluationIssue, DimensionScores } from './types';
|
|
20
21
|
import type { EvalIssue } from './types-public.js';
|
|
21
22
|
|
|
@@ -34,6 +35,20 @@ export interface VisualEvalConfig {
|
|
|
34
35
|
sampleProps?: Record<string, unknown>;
|
|
35
36
|
/** Viewport dimensions for screenshot */
|
|
36
37
|
viewport?: { width: number; height: number };
|
|
38
|
+
/**
|
|
39
|
+
* Optional provider-routing override — see
|
|
40
|
+
* `AgentConfig.routeOverride`. Threaded onto the agent this
|
|
41
|
+
* evaluator constructs so its multimodal call never falls back to
|
|
42
|
+
* `process.env` for credentials or model routing.
|
|
43
|
+
*/
|
|
44
|
+
routeOverride?: AgentConfig['routeOverride'];
|
|
45
|
+
/**
|
|
46
|
+
* Provider-429-retry observer — see `AgentConfig.onRetry`.
|
|
47
|
+
* Without it, a rate-limited retry inside the evaluation call
|
|
48
|
+
* happens (the agent's retry loop is unconditional) but never
|
|
49
|
+
* reaches the caller's structured log.
|
|
50
|
+
*/
|
|
51
|
+
onRetry?: AgentConfig['onRetry'];
|
|
37
52
|
}
|
|
38
53
|
|
|
39
54
|
export interface VisualEvalContext {
|
|
@@ -247,75 +262,37 @@ Respond with ONLY a JSON object (no markdown, no explanation):
|
|
|
247
262
|
"critique": "<2-3 sentences: what's good, what needs improvement, does it achieve the user's goal?>"
|
|
248
263
|
}`;
|
|
249
264
|
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
265
|
+
/**
|
|
266
|
+
* Send the screenshot to the configured multimodal provider through
|
|
267
|
+
* the shared `LLMAgent` machinery. Routing through `createVisionAgent`
|
|
268
|
+
* (rather than constructing provider SDK clients inline) is what puts
|
|
269
|
+
* this call inside the same `apiCall()` choke point as every other
|
|
270
|
+
* completion: provider 429s retry with `Retry-After` honoring, the
|
|
271
|
+
* caller's `onRetry` observer sees each retry, and `routeOverride`
|
|
272
|
+
* governs credentials and model routing (including the Anthropic
|
|
273
|
+
* Bedrock path) instead of `process.env`.
|
|
274
|
+
*/
|
|
256
275
|
async function callMultimodalLLM(
|
|
257
|
-
|
|
276
|
+
config: VisualEvalConfig,
|
|
258
277
|
model: string,
|
|
259
278
|
prompt: string,
|
|
260
279
|
screenshot: Buffer,
|
|
261
280
|
originalPrompt: string,
|
|
262
|
-
)
|
|
281
|
+
) {
|
|
263
282
|
const userPrompt = `## Original Request\n${originalPrompt}\n\nEvaluate the screenshot of the generated component.`;
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
baseURL: 'https://api.anthropic.com',
|
|
270
|
-
});
|
|
271
|
-
const response = await client.messages.create({
|
|
272
|
-
model: model.startsWith('anthropic/') ? model.slice(10) : model,
|
|
273
|
-
max_tokens: 1500,
|
|
274
|
-
system: prompt,
|
|
275
|
-
messages: [{
|
|
276
|
-
role: 'user',
|
|
277
|
-
content: [
|
|
278
|
-
{
|
|
279
|
-
type: 'image',
|
|
280
|
-
source: {
|
|
281
|
-
type: 'base64',
|
|
282
|
-
media_type: 'image/png',
|
|
283
|
-
data: screenshot.toString('base64'),
|
|
284
|
-
},
|
|
285
|
-
},
|
|
286
|
-
{ type: 'text', text: userPrompt },
|
|
287
|
-
],
|
|
288
|
-
}],
|
|
289
|
-
});
|
|
290
|
-
return {
|
|
291
|
-
text: response.content.filter(b => b.type === 'text').map(b => b.text).join(''),
|
|
292
|
-
inputTokens: response.usage.input_tokens,
|
|
293
|
-
outputTokens: response.usage.output_tokens,
|
|
294
|
-
};
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
// Google Gemini
|
|
298
|
-
const { GoogleGenAI } = await import('@google/genai');
|
|
299
|
-
const client = new GoogleGenAI({
|
|
300
|
-
apiKey: process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY,
|
|
301
|
-
});
|
|
302
|
-
const response = await client.models.generateContent({
|
|
303
|
-
model: model.startsWith('google/') ? model.slice(7) : model,
|
|
304
|
-
contents: [{
|
|
305
|
-
role: 'user',
|
|
306
|
-
parts: [
|
|
307
|
-
{ inlineData: { mimeType: 'image/png', data: screenshot.toString('base64') } },
|
|
308
|
-
{ text: userPrompt },
|
|
309
|
-
],
|
|
310
|
-
}],
|
|
311
|
-
config: { systemInstruction: prompt },
|
|
283
|
+
const agent = createVisionAgent({
|
|
284
|
+
provider: config.provider === 'claude' ? 'anthropic' : config.provider,
|
|
285
|
+
model,
|
|
286
|
+
routeOverride: config.routeOverride,
|
|
287
|
+
onRetry: config.onRetry,
|
|
312
288
|
});
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
289
|
+
return agent.callVision(
|
|
290
|
+
model,
|
|
291
|
+
prompt,
|
|
292
|
+
userPrompt,
|
|
293
|
+
{ mediaType: 'image/png', base64: screenshot.toString('base64') },
|
|
294
|
+
1500,
|
|
295
|
+
);
|
|
319
296
|
}
|
|
320
297
|
|
|
321
298
|
// ---------------------------------------------------------------------------
|
|
@@ -358,7 +335,7 @@ export async function runVisualEvaluation(
|
|
|
358
335
|
// Send to multimodal LLM
|
|
359
336
|
const model = config.model ?? getDefaultVisualModel(config.provider);
|
|
360
337
|
const response = await callMultimodalLLM(
|
|
361
|
-
config
|
|
338
|
+
config,
|
|
362
339
|
model,
|
|
363
340
|
VISUAL_EVAL_PROMPT,
|
|
364
341
|
screenshot,
|
|
@@ -377,9 +354,11 @@ export async function runVisualEvaluation(
|
|
|
377
354
|
}
|
|
378
355
|
|
|
379
356
|
/**
|
|
380
|
-
* Exported for testing — bundles and builds the HTML for rendering
|
|
357
|
+
* Exported for testing — bundles and builds the HTML for rendering;
|
|
358
|
+
* `callMultimodalLLM` so the #504 routing contract (config →
|
|
359
|
+
* `createVisionAgent`) is assertable without a screenshot pipeline.
|
|
381
360
|
*/
|
|
382
|
-
export { bundleForRendering, buildRenderHTML };
|
|
361
|
+
export { bundleForRendering, buildRenderHTML, callMultimodalLLM };
|
|
383
362
|
|
|
384
363
|
// ---------------------------------------------------------------------------
|
|
385
364
|
// Response parsing
|
|
@@ -217,6 +217,24 @@ function toEvalIssue(issue: RenderCheckIssue): EvalIssue | null {
|
|
|
217
217
|
fix: classifyRenderCrashFix(issue.reason),
|
|
218
218
|
};
|
|
219
219
|
|
|
220
|
+
case "optional-props-omitted":
|
|
221
|
+
// Same crash class as render-no-throw, surfaced by the second
|
|
222
|
+
// render pass with optional props stripped (ggui#528): the wire
|
|
223
|
+
// gate only requires `required: true` props, so a legitimate
|
|
224
|
+
// render may omit these and the component must survive it.
|
|
225
|
+
return {
|
|
226
|
+
tier: 0,
|
|
227
|
+
result,
|
|
228
|
+
category: "crash",
|
|
229
|
+
subcategory,
|
|
230
|
+
severity: "critical",
|
|
231
|
+
description: `Component crashed when optional props were omitted: ${issue.reason}`,
|
|
232
|
+
fix:
|
|
233
|
+
`Guard every optional prop before member access (props.${subject.split(",")[0]?.trim() ?? "x"}?.map(...), ` +
|
|
234
|
+
`a default via destructuring, or an early empty-state return). Optional props (no \`required: true\` in the contract) ` +
|
|
235
|
+
`may be absent at runtime even though the harness fills them.`,
|
|
236
|
+
};
|
|
237
|
+
|
|
220
238
|
case "action-wiring": {
|
|
221
239
|
const fix = issue.outcome === "unverified"
|
|
222
240
|
? `Source shows the action callback flowing into a non-native or custom-component prop. If wiring is intentional (e.g., Dropdown.onChange, drag-drop), this warn is informational — manual/browser verification is required to confirm. Otherwise wire to a native onClick={() => ${subject}(payload)} on <button> or design-system <Button>.`
|
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
// - declared props appear in the DOM (warn)
|
|
10
10
|
// - declared stream events update the DOM when emitted (warn)
|
|
11
11
|
|
|
12
|
+
import { installProductionActShim } from "./production-act-shim.js";
|
|
13
|
+
|
|
12
14
|
export { runRenderCheck, type RenderCheckResult, type RenderCheckIssue } from "./render-check.js";
|
|
13
15
|
export { createProbe, createProbeWireConfig, type Probe } from "./probe.js";
|
|
14
16
|
export { prepareMockupProps, type MockupPropsResult } from "./prepare-mockup.js";
|
|
@@ -51,6 +53,12 @@ export async function warmupRuntimeRenderProbe(): Promise<{ ms: number; loaded:
|
|
|
51
53
|
missing += 1;
|
|
52
54
|
}
|
|
53
55
|
};
|
|
56
|
+
// Must run before RTL's first import in this process — hosts that
|
|
57
|
+
// call this warmup at process start (server-side blueprint
|
|
58
|
+
// validation) land RTL in the module cache, with its act snapshot
|
|
59
|
+
// taken, here — long before the first real probe call. See
|
|
60
|
+
// production-act-shim.ts's docstring for why.
|
|
61
|
+
installProductionActShim();
|
|
54
62
|
await Promise.all([
|
|
55
63
|
tryLoad("happy-dom"),
|
|
56
64
|
tryLoad("@testing-library/react"),
|
|
@@ -152,6 +152,20 @@ function synthesizeFromSchema(
|
|
|
152
152
|
return synthesizeFromSchema(schema.anyOf[0], hint, warnings, depth + 1);
|
|
153
153
|
}
|
|
154
154
|
|
|
155
|
+
// Draft-07 type array (`['string','null']` — the canonical nullable
|
|
156
|
+
// form since draft-2026-08-19): synthesize from the first non-null
|
|
157
|
+
// member ('null' only when it is the sole member). The depth guard
|
|
158
|
+
// above bounds the recursion.
|
|
159
|
+
if (Array.isArray(schema.type)) {
|
|
160
|
+
const primary = schema.type.find((t) => t !== "null") ?? "null";
|
|
161
|
+
return synthesizeFromSchema(
|
|
162
|
+
{ ...schema, type: primary },
|
|
163
|
+
hint,
|
|
164
|
+
warnings,
|
|
165
|
+
depth + 1,
|
|
166
|
+
);
|
|
167
|
+
}
|
|
168
|
+
|
|
155
169
|
switch (schema.type) {
|
|
156
170
|
case "string": {
|
|
157
171
|
// Format-aware placeholder
|