@ggui-ai/ui-gen 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/generation-dispatch.d.ts +2 -2
- package/dist/adapters/generation-dispatch.js +192 -58
- package/dist/adapters/generation-dispatch.js.map +1 -1
- package/dist/adapters/index.js +7 -9
- package/dist/adapters/index.js.map +1 -1
- package/dist/advanced/index.d.ts +3 -3
- package/dist/advanced/index.js +192 -58
- package/dist/advanced/index.js.map +1 -1
- package/dist/blueprint-validator.js +2 -7
- package/dist/blueprint-validator.js.map +1 -1
- package/dist/boilerplate.js +4 -4
- package/dist/boilerplate.js.map +1 -1
- package/dist/check/index.d.ts +1 -1
- package/dist/check/index.js +57 -16
- package/dist/check/index.js.map +1 -1
- package/dist/{create-ui-generator-CF0EL0Li.d.ts → create-ui-generator-D6TKuv9V.d.ts} +2 -2
- package/dist/{dispatch-D75M-OfZ.d.ts → dispatch-BuZfTh9r.d.ts} +1 -1
- package/dist/evaluation/axis-checks/index.d.ts +3 -3
- package/dist/evaluation/axis-checks/index.js.map +1 -1
- package/dist/evaluation/axis-checks/registry.d.ts +1 -1
- package/dist/evaluation/index.d.ts +3 -3
- package/dist/evaluation/index.js +79 -16
- package/dist/evaluation/index.js.map +1 -1
- package/dist/harness/check/runtime-render/index.d.ts +1 -1
- package/dist/harness/check/runtime-render/index.js +2 -7
- package/dist/harness/check/runtime-render/index.js.map +1 -1
- package/dist/harness/index.d.ts +2 -2
- package/dist/harness/index.js +64 -19
- package/dist/harness/index.js.map +1 -1
- package/dist/harness/prompts.js +2 -0
- package/dist/harness/prompts.js.map +1 -1
- package/dist/harness/result-types.d.ts +1 -1
- package/dist/harness/runtime.d.ts +2 -2
- package/dist/harness/runtime.js +5 -5
- package/dist/harness/runtime.js.map +1 -1
- package/dist/harness/types-public.d.ts +1 -1
- package/dist/harness/types-public.js +4 -4
- package/dist/harness/types-public.js.map +1 -1
- package/dist/harness/validator-trace-sink.d.ts +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +192 -58
- package/dist/index.js.map +1 -1
- package/dist/{llm-router-BEVtbAb9.d.ts → llm-router-D7fjBhtr.d.ts} +39 -0
- package/dist/patch.d.ts +1 -1
- package/dist/provider-adapter-contract.js +1017 -731
- package/dist/provider-adapter-contract.js.map +1 -1
- package/dist/{types-public-DIpjC-OA.d.ts → types-public-CkNWEov9.d.ts} +64 -1
- package/dist/validation/index.d.ts +1 -1
- package/dist/validation/index.js +1 -1
- package/dist/validation/index.js.map +1 -1
- package/dist/workflows.d.ts +1 -1
- package/package.json +21 -21
- package/src/adapters/provider-router.ts +9 -0
- package/src/boilerplate/system-prompt.ts +4 -4
- package/src/check/run-tier0.ts +109 -22
- package/src/check/type-checker.ts +12 -2
- package/src/evaluation/index.ts +4 -0
- package/src/evaluation/llm-evaluator.ts +52 -8
- package/src/evaluation/types-public.ts +79 -0
- package/src/evaluation/visual-evaluator.ts +27 -5
- package/src/harness/coding/run-eval-round.ts +32 -6
- package/src/harness/llm-router.ts +128 -2
- package/src/validation/primitives.ts +1 -1
package/dist/workflows.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { C as Classification } from './axes-CzLEMDeB.js';
|
|
2
2
|
import { Workflow } from './harness/types-public.js';
|
|
3
3
|
import '@ggui-ai/protocol';
|
|
4
|
-
import './types-public-
|
|
4
|
+
import './types-public-CkNWEov9.js';
|
|
5
5
|
import './types-BOvHNG7K.js';
|
|
6
6
|
import './llm.js';
|
|
7
7
|
import './policy.js';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ggui-ai/ui-gen",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.15.0",
|
|
4
4
|
"description": "UI generation harness for ggui — the LLM system prompt, multi-turn coding loop, and evaluation engine that turn a DataContract into a React component. Implements the UiGenerator contract from @ggui-ai/mcp-server-core; consumed by @ggui-ai/mcp-server.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"keywords": [
|
|
@@ -183,14 +183,14 @@
|
|
|
183
183
|
}
|
|
184
184
|
},
|
|
185
185
|
"peerDependencies": {
|
|
186
|
-
"@google/adk": "^0.
|
|
187
|
-
"@openai/agents": "^0.
|
|
186
|
+
"@google/adk": "^2.0.0",
|
|
187
|
+
"@openai/agents": "^0.17.0",
|
|
188
188
|
"puppeteer-core": "^24.40.0 || ^25.0.0",
|
|
189
189
|
"react": "^18.0.0 || ^19.0.0",
|
|
190
190
|
"react-dom": "^18.0.0 || ^19.0.0",
|
|
191
|
-
"@ggui-ai/mcp-server-core": "0.
|
|
192
|
-
"@ggui-ai/protocol": "0.
|
|
193
|
-
"@ggui-ai/project-config": "0.
|
|
191
|
+
"@ggui-ai/mcp-server-core": "0.15.0",
|
|
192
|
+
"@ggui-ai/protocol": "0.15.0",
|
|
193
|
+
"@ggui-ai/project-config": "0.15.0"
|
|
194
194
|
},
|
|
195
195
|
"peerDependenciesMeta": {
|
|
196
196
|
"@google/adk": {
|
|
@@ -210,34 +210,34 @@
|
|
|
210
210
|
}
|
|
211
211
|
},
|
|
212
212
|
"dependencies": {
|
|
213
|
-
"@anthropic-ai/bedrock-sdk": "^0.
|
|
213
|
+
"@anthropic-ai/bedrock-sdk": "^0.33.3",
|
|
214
214
|
"@anthropic-ai/claude-agent-sdk": "0.3.229",
|
|
215
|
-
"@anthropic-ai/sdk": "^0.
|
|
215
|
+
"@anthropic-ai/sdk": "^0.123.0",
|
|
216
216
|
"@google/genai": "^2.8.0",
|
|
217
217
|
"@testing-library/dom": "^10.4.0",
|
|
218
218
|
"@testing-library/react": "^16.0.0",
|
|
219
219
|
"@testing-library/user-event": "^14.6.1",
|
|
220
220
|
"@typescript-eslint/parser": "^8.0.0",
|
|
221
221
|
"diff": "^8.0.3",
|
|
222
|
-
"esbuild": "^0.
|
|
222
|
+
"esbuild": "^0.28.2",
|
|
223
223
|
"eslint": "^8.56.0",
|
|
224
224
|
"eslint-plugin-react": "^7.37.0",
|
|
225
225
|
"eslint-plugin-react-hooks": "^5.0.0",
|
|
226
226
|
"happy-dom": "^20.8.9",
|
|
227
227
|
"isomorphic-git": "^1.37.4",
|
|
228
228
|
"memfs": "^4.56.11",
|
|
229
|
-
"openai": "^
|
|
229
|
+
"openai": "^7.9.0",
|
|
230
230
|
"typescript": "^5.0.0",
|
|
231
231
|
"zod": "^4.3.6",
|
|
232
|
-
"@ggui-ai/
|
|
233
|
-
"@ggui-ai/
|
|
234
|
-
"@ggui-ai/ui-visual-tester": "0.
|
|
235
|
-
"@ggui-ai/
|
|
236
|
-
"@ggui-ai/
|
|
232
|
+
"@ggui-ai/gadgets": "0.15.0",
|
|
233
|
+
"@ggui-ai/sandbox": "0.15.0",
|
|
234
|
+
"@ggui-ai/ui-visual-tester": "0.15.0",
|
|
235
|
+
"@ggui-ai/wire": "0.15.0",
|
|
236
|
+
"@ggui-ai/design": "0.15.0"
|
|
237
237
|
},
|
|
238
238
|
"devDependencies": {
|
|
239
|
-
"@google/adk": "^0.
|
|
240
|
-
"@openai/agents": "^0.
|
|
239
|
+
"@google/adk": "^2.0.0",
|
|
240
|
+
"@openai/agents": "^0.17.0",
|
|
241
241
|
"@sparticuz/chromium": "^149.0.0",
|
|
242
242
|
"@types/eslint": "^9.6.1",
|
|
243
243
|
"@types/node": "^22.0.0",
|
|
@@ -248,12 +248,12 @@
|
|
|
248
248
|
"react-dom": "^19.0.0",
|
|
249
249
|
"tsup": "^8.5.0",
|
|
250
250
|
"vitest": "^3.2.6",
|
|
251
|
-
"@ggui-ai/mcp-server-core": "0.
|
|
252
|
-
"@ggui-ai/project-config": "0.
|
|
253
|
-
"@ggui-ai/protocol": "0.
|
|
251
|
+
"@ggui-ai/mcp-server-core": "0.15.0",
|
|
252
|
+
"@ggui-ai/project-config": "0.15.0",
|
|
253
|
+
"@ggui-ai/protocol": "0.15.0"
|
|
254
254
|
},
|
|
255
255
|
"engines": {
|
|
256
|
-
"node": ">=
|
|
256
|
+
"node": ">=22.0.0"
|
|
257
257
|
},
|
|
258
258
|
"author": "ggui contributors <hello@ggui.ai>",
|
|
259
259
|
"scripts": {
|
|
@@ -30,6 +30,15 @@ export function getBedrockModelId(model: string): string {
|
|
|
30
30
|
'anthropic/claude-haiku-4-5': 'us.anthropic.claude-haiku-4-5-20251001-v1:0',
|
|
31
31
|
'anthropic/claude-sonnet-4-6': 'us.anthropic.claude-sonnet-4-6',
|
|
32
32
|
'anthropic/claude-opus-4-6': 'us.anthropic.claude-opus-4-6-v1:0',
|
|
33
|
+
// Claude 5-family (ggui#706, strings verified 2026-09-02 against
|
|
34
|
+
// platform.claude.com; Bedrock base form `anthropic.<id>`, upcast to
|
|
35
|
+
// the cross-region profile). Explicit so the ids are pinned by test,
|
|
36
|
+
// not merely reachable through the generic fallback below.
|
|
37
|
+
'anthropic/claude-fable-5-1': 'us.anthropic.claude-fable-5-1',
|
|
38
|
+
'anthropic/claude-opus-5': 'us.anthropic.claude-opus-5',
|
|
39
|
+
'anthropic/claude-sonnet-5': 'us.anthropic.claude-sonnet-5',
|
|
40
|
+
// Legacy (still available; registry state `legacy`, not deprecated).
|
|
41
|
+
'anthropic/claude-fable-5': 'us.anthropic.claude-fable-5',
|
|
33
42
|
};
|
|
34
43
|
// Normalize the Bedrock dot-form opt-in (e.g. `anthropic.claude-haiku-4-5`,
|
|
35
44
|
// produced when cloud-pod's `resolvePoolRoute` strips a `bedrock/` opt-in
|
|
@@ -574,17 +574,17 @@ MANDATORY:
|
|
|
574
574
|
1. NEVER use hardcoded hex colors like \`#7c3aed\` — ONLY \`var(--ggui-color-*)\` tokens.
|
|
575
575
|
2. NEVER use CSS gradients with custom colors. If you need a gradient: \`linear-gradient(to bottom, var(--ggui-color-primary-500), var(--ggui-color-primary-700))\`.
|
|
576
576
|
3. NEVER invent your own palette. The system provides primary, neutral, success, warning, error, and info — use ONLY these.
|
|
577
|
-
4. NEVER include literal fallback values in
|
|
577
|
+
4. NEVER include literal fallback values in ANY token reference — colors, spacing, typography, radius, shadows alike. Write \`var(--ggui-spacing-4)\` bare, never \`var(--ggui-spacing-4, 16px)\`; \`var(--ggui-color-primary-600)\` bare, never \`var(--ggui-color-primary-600, #0284c7)\`. The runtime injects every token; a literal fallback paints the wrong value exactly when the operator's theme matters most.
|
|
578
578
|
|
|
579
579
|
Token categories:
|
|
580
580
|
- Brand: \`var(--ggui-color-primary-600)\`, \`var(--ggui-color-primary-50)\`
|
|
581
581
|
- Text: \`var(--ggui-color-onSurface)\`, \`var(--ggui-color-onSurfaceVariant)\`
|
|
582
582
|
- Backgrounds: \`var(--ggui-color-surface)\`, \`var(--ggui-color-surfaceVariant)\`
|
|
583
583
|
- Borders: \`var(--ggui-color-outline)\`
|
|
584
|
-
- Spacing: \`var(--ggui-spacing-4
|
|
585
|
-
- Typography: \`var(--ggui-font-size-sm
|
|
584
|
+
- Spacing: \`var(--ggui-spacing-4)\`, \`var(--ggui-spacing-6)\`
|
|
585
|
+
- Typography: \`var(--ggui-font-size-sm)\`, \`var(--ggui-font-weight-semibold)\`
|
|
586
586
|
- Shadows: \`var(--ggui-shape-shadow-sm)\`, \`var(--ggui-shape-shadow-md)\`, \`var(--ggui-shape-shadow-lg)\`
|
|
587
|
-
- Radius: \`var(--ggui-shape-radius-md
|
|
587
|
+
- Radius: \`var(--ggui-shape-radius-md)\`, \`var(--ggui-shape-radius-lg)\`
|
|
588
588
|
|
|
589
589
|
Prefer primitives' built-in styling props over inline styles when possible.
|
|
590
590
|
|
package/src/check/run-tier0.ts
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
import ts from 'typescript';
|
|
16
16
|
import type { DataContract } from '@ggui-ai/protocol';
|
|
17
17
|
import { HOOK_NAME_RE, listContractGadgets } from '@ggui-ai/protocol';
|
|
18
|
+
import { consumedTokenManifest } from '@ggui-ai/design/themes';
|
|
18
19
|
import { validateAllContracts, type ContractIssue } from './contract-validation.js';
|
|
19
20
|
import { typecheck } from './type-checker.js';
|
|
20
21
|
import { lintReactHooks, type ReactLintDiagnostic } from './react-linter.js';
|
|
@@ -260,6 +261,17 @@ function detectCertainDoubleWiredActions(
|
|
|
260
261
|
return issues;
|
|
261
262
|
}
|
|
262
263
|
|
|
264
|
+
/**
|
|
265
|
+
* The consumed-token manifest as a closed set (ggui#613 residual 3 —
|
|
266
|
+
* the generation-time closure leg of the #598 record). Every theme
|
|
267
|
+
* MUST define exactly these names (the coverage gate); a generated
|
|
268
|
+
* component may therefore consume ONLY these names. Anything else
|
|
269
|
+
* is a reference no theme is obligated to define, and under the s4
|
|
270
|
+
* fallback ban it paints nothing.
|
|
271
|
+
*/
|
|
272
|
+
const MANIFEST_TOKENS: ReadonlySet<string> = new Set(consumedTokenManifest);
|
|
273
|
+
const TOKEN_REF_RE = /var\(\s*(--ggui-[a-zA-Z0-9-]+)/g;
|
|
274
|
+
|
|
263
275
|
/** Map wire kind to the matching `@ggui-ai/wire` hook name. */
|
|
264
276
|
const HOOK_NAME_FOR: Readonly<Record<WireKind, string>> = {
|
|
265
277
|
action: 'useAction',
|
|
@@ -507,30 +519,45 @@ export async function runTier0Checks(
|
|
|
507
519
|
// patterns flowing untouched. Asset-fixed brand colors get a
|
|
508
520
|
// typed escape on Box (`assetColor` + `assetSemantic`) — see
|
|
509
521
|
// T-4 follow-up.
|
|
522
|
+
// Literal fallback inside ANY token reference (ggui#598 slice 4
|
|
523
|
+
// origin-kill, generalized by ggui#613 residual 4 from color to
|
|
524
|
+
// every token): the runtime injects every token — colors, spacing,
|
|
525
|
+
// typography, radius, shadows — so a literal fallback can only
|
|
526
|
+
// ever paint default-brand pixels under a failed injection, which
|
|
527
|
+
// must be visible (unset) rather than silently default. The old
|
|
528
|
+
// hex-only exemption plus a remediation that suggested
|
|
529
|
+
// `var(--x, ${hex})` actively taught the model to launder invented
|
|
530
|
+
// values INTO fallbacks (adversarial-cycle find). A token-to-token
|
|
531
|
+
// fallback (`var(--a, var(--b))`) is not a literal — it stays on
|
|
532
|
+
// the manifest check alone.
|
|
533
|
+
const literalFallbackMatch = line.match(
|
|
534
|
+
/var\(\s*(--ggui-[a-zA-Z0-9-]+)\s*,\s*([^)]+)\)/,
|
|
535
|
+
);
|
|
536
|
+
const literalFallback =
|
|
537
|
+
literalFallbackMatch !== null &&
|
|
538
|
+
!literalFallbackMatch[2]!.trim().startsWith('var(')
|
|
539
|
+
? { token: literalFallbackMatch[1]!, literal: literalFallbackMatch[2]!.trim() }
|
|
540
|
+
: null;
|
|
541
|
+
if (literalFallback !== null && !assetEscapedLines.has(lineNum)) {
|
|
542
|
+
issues.push({
|
|
543
|
+
tier: 0,
|
|
544
|
+
result: 'fail',
|
|
545
|
+
category: 'tokens',
|
|
546
|
+
subcategory: 'token-fallback',
|
|
547
|
+
severity: 'critical',
|
|
548
|
+
description: `Literal fallback "${literalFallback.literal}" inside a token reference — the runtime injects every token, and the fallback paints the wrong value exactly when the theme matters most.`,
|
|
549
|
+
fix: `Drop the fallback: write var(${literalFallback.token}) bare. If the token might not exist, that is a vocabulary problem to fix by choosing a token from the design-system docs, never by a literal.`,
|
|
550
|
+
line: lineNum,
|
|
551
|
+
});
|
|
552
|
+
}
|
|
553
|
+
|
|
510
554
|
const hexMatch = line.match(/#[0-9a-fA-F]{3,8}\b/);
|
|
511
555
|
if (hexMatch && !assetEscapedLines.has(lineNum)) {
|
|
512
|
-
//
|
|
513
|
-
//
|
|
514
|
-
//
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
// exemption plus a remediation text that suggested
|
|
518
|
-
// `var(--x, ${hex})` actively taught the model to launder its
|
|
519
|
-
// invented hex INTO fallbacks (adversarial-cycle find).
|
|
520
|
-
const fallbackVarMatch = line.match(
|
|
521
|
-
/var\((--ggui-[a-zA-Z0-9-]+)\s*,[^)]*#[0-9a-fA-F]{3,8}/,
|
|
522
|
-
);
|
|
523
|
-
if (fallbackVarMatch) {
|
|
524
|
-
issues.push({
|
|
525
|
-
tier: 0,
|
|
526
|
-
result: 'fail',
|
|
527
|
-
category: 'tokens',
|
|
528
|
-
subcategory: 'token-fallback',
|
|
529
|
-
severity: 'critical',
|
|
530
|
-
description: `Literal fallback "${hexMatch[0]}" inside a token reference — the runtime injects every token, and the fallback paints the wrong brand exactly when the theme matters most.`,
|
|
531
|
-
fix: `Drop the fallback: write var(${fallbackVarMatch[1]}) bare. If the token might not exist, that is a vocabulary problem to fix by choosing a token from the design-system docs, never by a literal.`,
|
|
532
|
-
line: lineNum,
|
|
533
|
-
});
|
|
556
|
+
// A hex that lives inside a token fallback is already reported
|
|
557
|
+
// above as `token-fallback` — don't double-report it as
|
|
558
|
+
// `hex-color`.
|
|
559
|
+
if (literalFallback !== null && literalFallback.literal.includes(hexMatch[0])) {
|
|
560
|
+
// reported above
|
|
534
561
|
} else if (!line.includes('// fallback')) {
|
|
535
562
|
issues.push({
|
|
536
563
|
tier: 0,
|
|
@@ -569,6 +596,31 @@ export async function runTier0Checks(
|
|
|
569
596
|
});
|
|
570
597
|
}
|
|
571
598
|
|
|
599
|
+
// Off-manifest token references (ggui#613 residual 3): a
|
|
600
|
+
// `var(--ggui-*)` whose name is outside the consumed-token
|
|
601
|
+
// manifest. The manifest is the closed vocabulary the prompt
|
|
602
|
+
// teaches AND the set every theme must define — so a name outside
|
|
603
|
+
// it is invented, and with literal fallbacks banned it renders
|
|
604
|
+
// UNSET. Fail, like the other token-contract violations; one issue
|
|
605
|
+
// per distinct name per line.
|
|
606
|
+
const offManifest = new Set<string>();
|
|
607
|
+
for (const m of line.matchAll(TOKEN_REF_RE)) {
|
|
608
|
+
const name = m[1]!;
|
|
609
|
+
if (!MANIFEST_TOKENS.has(name)) offManifest.add(name);
|
|
610
|
+
}
|
|
611
|
+
for (const name of offManifest) {
|
|
612
|
+
issues.push({
|
|
613
|
+
tier: 0,
|
|
614
|
+
result: 'fail',
|
|
615
|
+
category: 'tokens',
|
|
616
|
+
subcategory: 'off-manifest-token',
|
|
617
|
+
severity: 'critical',
|
|
618
|
+
description: `Token "${name}" is not in the design system's token manifest — no theme defines it, so it renders unset.`,
|
|
619
|
+
fix: `Choose a token that exists: the closed vocabulary is the manifest the design-system docs list (colors: var(--ggui-color-primary-500), var(--ggui-color-onSurface), …; spacing: var(--ggui-spacing-4); type: var(--ggui-font-size-sm)). Never invent a name and never add a literal fallback.`,
|
|
620
|
+
line: lineNum,
|
|
621
|
+
});
|
|
622
|
+
}
|
|
623
|
+
|
|
572
624
|
// Raw CSS length on a spacing / radius prop — bypasses the design
|
|
573
625
|
// scale. Same reasoning as the hardcoded-color checks above: a
|
|
574
626
|
// literal `gap="8px"` defeats the theme's spacing scale exactly as
|
|
@@ -746,6 +798,41 @@ export async function runTier0Checks(
|
|
|
746
798
|
);
|
|
747
799
|
}
|
|
748
800
|
|
|
801
|
+
// ── Connection suppression (ggui#670 Phase 2) ────────────────
|
|
802
|
+
// `useRender().isConnected` is a LIVE read of the relay latch, and
|
|
803
|
+
// the ATTEMPT is the runtime's self-heal sensor (#440/#443
|
|
804
|
+
// attempt-always): a dispatch fired while the latch stands is how the
|
|
805
|
+
// runtime discovers the host can relay again. Binding `disabled` to
|
|
806
|
+
// the connection state strips the handler and blinds that sensor —
|
|
807
|
+
// the control stays dead after recovery. The runtime owns the dead
|
|
808
|
+
// zone's presentation (ggui#670 Phase 3: standing notice, then a
|
|
809
|
+
// per-gesture cue) — generated code never suppresses on it. Direct
|
|
810
|
+
// binding only, and only when the source reads
|
|
811
|
+
// `useRender` at all — a same-named `props.isConnected` (a device-
|
|
812
|
+
// status card) is not the connection state. Whole-source scan so a
|
|
813
|
+
// multi-line attribute is caught; the issue reports the attribute's
|
|
814
|
+
// line. The lookbehind keeps `aria-disabled={…}` (the idiom) out.
|
|
815
|
+
if (/\buseRender\s*\(/.test(sourceCode)) {
|
|
816
|
+
const suppressionRe = /(?<![\w-])disabled\s*=\s*\{([^}]*)\}/gs;
|
|
817
|
+
for (const m of sourceCode.matchAll(suppressionRe)) {
|
|
818
|
+
const expr = m[1]!;
|
|
819
|
+
if (!/\bisConnected\b/.test(expr)) continue;
|
|
820
|
+
const attrLine = sourceCode.slice(0, m.index ?? 0).split('\n').length;
|
|
821
|
+
issues.push({
|
|
822
|
+
tier: 0,
|
|
823
|
+
result: 'fail',
|
|
824
|
+
category: 'interactivity',
|
|
825
|
+
subcategory: 'connection-suppression',
|
|
826
|
+
severity: 'critical',
|
|
827
|
+
description:
|
|
828
|
+
`\`disabled\` is bound to the connection state (\`${expr.trim().replace(/\s+/g, ' ')}\`) — that strips the click handler, and the click is how the runtime detects that the host can relay actions again. The control stays dead after recovery.`,
|
|
829
|
+
fix:
|
|
830
|
+
'Leave the control enabled — never bind `disabled` to the connection state. The runtime presents the dead zone itself (a standing notice, then a per-gesture cue) and the attempt is how it detects recovery; a suppressed control stays dead after the host recovers.',
|
|
831
|
+
line: attrLine,
|
|
832
|
+
});
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
|
|
749
836
|
// ── Optional props accessed without guard ──────────────────
|
|
750
837
|
// Check if Props interface has optional fields that are accessed without ?. or ?? or &&
|
|
751
838
|
const propsInterfaceMatch = sourceCode.match(/interface Props\s*\{([^}]+)\}/);
|
|
@@ -316,10 +316,18 @@ async function loadVfs(): Promise<Map<string, VfsEntry>> {
|
|
|
316
316
|
/**
|
|
317
317
|
* Try to resolve a relative import within the VFS.
|
|
318
318
|
* Given `containingFile` (e.g. `node_modules/@ggui-ai/design/primitives/index.d.ts`)
|
|
319
|
-
* and `importPath` (e.g. `./Card`
|
|
319
|
+
* and `importPath` (e.g. `./Card` or, as every published declaration says
|
|
320
|
+
* since 0.15.0, `./Card.js`), try:
|
|
320
321
|
* - <resolved>.d.ts
|
|
321
322
|
* - <resolved>/index.d.ts
|
|
322
323
|
* - <resolved>.ts
|
|
324
|
+
* A `.js` / `.mjs` / `.cjs` extension on the specifier is the ESM
|
|
325
|
+
* convention for "the module that compiles to this file" — TypeScript's
|
|
326
|
+
* NodeNext maps `./Card.js` to `./Card.d.ts`, and so does this: the
|
|
327
|
+
* extension is stripped before the candidates are tried. Without that, a
|
|
328
|
+
* `.js`-suffixed re-export resolved to nothing, the imported names typed
|
|
329
|
+
* as `any`, and the checker reported zero errors on code it must reject
|
|
330
|
+
* (ggui 0.15.0 publish-gate, type-checker.test.ts).
|
|
323
331
|
*/
|
|
324
332
|
function resolveRelativeInVfs(
|
|
325
333
|
vfs: Map<string, VfsEntry>,
|
|
@@ -329,7 +337,9 @@ function resolveRelativeInVfs(
|
|
|
329
337
|
// Strip leading '/' from containingFile — TS may prepend getCurrentDirectory()
|
|
330
338
|
const normalized = containingFile.startsWith('/') ? containingFile.slice(1) : containingFile;
|
|
331
339
|
const dir = path.posix.dirname(normalized);
|
|
332
|
-
|
|
340
|
+
// `./x.js` (`.mjs`, `.cjs`) names the declaration `./x.d.ts` — strip it.
|
|
341
|
+
const specifier = importPath.replace(/\.(?:m|c)?js$/, '');
|
|
342
|
+
const resolved = path.posix.normalize(`${dir}/${specifier}`);
|
|
333
343
|
|
|
334
344
|
// Try .d.ts extension
|
|
335
345
|
const withDts = `${resolved}.d.ts`;
|
package/src/evaluation/index.ts
CHANGED
|
@@ -23,6 +23,8 @@ export type {
|
|
|
23
23
|
EvalCategory,
|
|
24
24
|
EvalResult,
|
|
25
25
|
EvalCriterion,
|
|
26
|
+
CriterionCoverage,
|
|
27
|
+
CriterionRunStatus,
|
|
26
28
|
QualityMode,
|
|
27
29
|
QualityConfig,
|
|
28
30
|
AxisCheck,
|
|
@@ -30,6 +32,8 @@ export type {
|
|
|
30
32
|
} from './types-public.js';
|
|
31
33
|
export {
|
|
32
34
|
CRITERIA,
|
|
35
|
+
LLM_EVAL_STATIC_CRITERIA,
|
|
36
|
+
notApplicableCoverage,
|
|
33
37
|
DEFAULT_QUALITY_CONFIG,
|
|
34
38
|
matches,
|
|
35
39
|
priorityForIssue,
|
|
@@ -14,7 +14,14 @@
|
|
|
14
14
|
// Caching: Anthropic ephemeral cache, OpenAI auto-cache, Google explicit cache.
|
|
15
15
|
// Same system prompt across all 7 calls → first call caches, 6 remaining hit cache.
|
|
16
16
|
|
|
17
|
-
import type {
|
|
17
|
+
import type {
|
|
18
|
+
EvalIssue,
|
|
19
|
+
EvalResult,
|
|
20
|
+
EvalCategory,
|
|
21
|
+
CriterionCoverage,
|
|
22
|
+
CriterionRunStatus,
|
|
23
|
+
} from './types-public.js';
|
|
24
|
+
import { LLM_EVAL_STATIC_CRITERIA } from './types-public.js';
|
|
18
25
|
import { createAgent } from '../harness/llm-router';
|
|
19
26
|
import type { AgentConfig, LLMToolDef, LLMAgent } from '../harness/llm-router';
|
|
20
27
|
import type { JsonObject } from '@ggui-ai/protocol';
|
|
@@ -518,13 +525,26 @@ interface CriterionResult {
|
|
|
518
525
|
tier: 1 | 2;
|
|
519
526
|
issues: EvalIssue[];
|
|
520
527
|
pass: boolean;
|
|
528
|
+
/**
|
|
529
|
+
* Did this criterion actually produce a verdict? The evaluator
|
|
530
|
+
* fail-opens (`pass: true`) on LLM errors and missing tool calls so a
|
|
531
|
+
* flaky call never blocks generation — this field is what keeps that
|
|
532
|
+
* fail-open honest: `skipped` means `pass` carries zero evidence.
|
|
533
|
+
*/
|
|
534
|
+
coverage: CriterionRunStatus;
|
|
535
|
+
/** Populated for `skipped` — why the criterion produced no verdict. */
|
|
536
|
+
skipReason?: string;
|
|
521
537
|
tokens: { in: number; out: number };
|
|
522
538
|
durationMs: number;
|
|
523
539
|
}
|
|
524
540
|
|
|
525
|
-
/**
|
|
526
|
-
|
|
527
|
-
|
|
541
|
+
/**
|
|
542
|
+
* All valid categories for LLM evaluation — derived from the ONE static
|
|
543
|
+
* list in types-public so the coverage stamp here and the harness's
|
|
544
|
+
* bypass stamp (`notApplicableCoverage`) can never disagree on the set.
|
|
545
|
+
*/
|
|
546
|
+
const TIER_1_CRITERIA = LLM_EVAL_STATIC_CRITERIA.filter((c) => c.tier === 1).map((c) => c.criterion);
|
|
547
|
+
const TIER_2_CRITERIA = LLM_EVAL_STATIC_CRITERIA.filter((c) => c.tier === 2).map((c) => c.criterion);
|
|
528
548
|
|
|
529
549
|
async function evalCriterion(
|
|
530
550
|
criterion: string,
|
|
@@ -557,6 +577,8 @@ async function evalCriterion(
|
|
|
557
577
|
tier,
|
|
558
578
|
issues: [],
|
|
559
579
|
pass: true,
|
|
580
|
+
coverage: 'skipped',
|
|
581
|
+
skipReason: 'no tool call returned',
|
|
560
582
|
tokens: { in: response.inputTokens, out: response.outputTokens },
|
|
561
583
|
durationMs,
|
|
562
584
|
};
|
|
@@ -614,18 +636,25 @@ async function evalCriterion(
|
|
|
614
636
|
tier,
|
|
615
637
|
issues,
|
|
616
638
|
pass,
|
|
639
|
+
coverage: 'ran',
|
|
617
640
|
tokens: { in: response.inputTokens, out: response.outputTokens },
|
|
618
641
|
durationMs,
|
|
619
642
|
};
|
|
620
643
|
} catch (e) {
|
|
621
644
|
const durationMs = Date.now() - start;
|
|
622
|
-
|
|
623
|
-
|
|
645
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
646
|
+
console.error(`[eval] ${criterion}: error — ${message}`);
|
|
647
|
+
// On error, treat as pass to avoid blocking on flaky LLM calls.
|
|
648
|
+
// `coverage: 'skipped'` discloses the fail-open: this criterion
|
|
649
|
+
// produced NO verdict, and consumers must not read its `pass`
|
|
650
|
+
// membership as evidence.
|
|
624
651
|
return {
|
|
625
652
|
criterion,
|
|
626
653
|
tier,
|
|
627
654
|
issues: [],
|
|
628
655
|
pass: true,
|
|
656
|
+
coverage: 'skipped',
|
|
657
|
+
skipReason: message,
|
|
629
658
|
tokens: { in: 0, out: 0 },
|
|
630
659
|
durationMs,
|
|
631
660
|
};
|
|
@@ -809,22 +838,37 @@ export async function runLLMEvaluation(
|
|
|
809
838
|
|
|
810
839
|
const wallMs = Date.now() - evalStart;
|
|
811
840
|
|
|
841
|
+
// Per-criterion coverage — one row per criterion, ALWAYS all of them,
|
|
842
|
+
// so a consumer can tell "ran clean" from "silently didn't run"
|
|
843
|
+
// (the fail-open paths above return pass:true with zero evidence).
|
|
844
|
+
const criteriaCoverage: CriterionCoverage[] = results.map((r) => ({
|
|
845
|
+
criterion: r.criterion,
|
|
846
|
+
tier: r.tier,
|
|
847
|
+
status: r.coverage,
|
|
848
|
+
...(r.skipReason !== undefined ? { reason: r.skipReason } : {}),
|
|
849
|
+
}));
|
|
850
|
+
const skippedCount = criteriaCoverage.filter((c) => c.status === 'skipped').length;
|
|
851
|
+
|
|
812
852
|
// Log per-criterion results
|
|
813
853
|
for (const r of results) {
|
|
814
854
|
const issueCount = r.issues.length;
|
|
815
|
-
const status = r.
|
|
855
|
+
const status = r.coverage === 'skipped'
|
|
856
|
+
? 'SKIPPED'
|
|
857
|
+
: r.pass ? 'PASS' : r.tier === 1 ? 'FAIL' : (r.issues.some(i => i.result === 'fail') ? 'FAIL' : 'WARN');
|
|
816
858
|
const issueLabel = issueCount > 0 ? ` (${issueCount} issue${issueCount > 1 ? 's' : ''})` : '';
|
|
817
859
|
console.log(`[eval] ${r.criterion}: ${status}${issueLabel} ${(r.durationMs / 1000).toFixed(1)}s`);
|
|
818
860
|
}
|
|
819
861
|
console.log(
|
|
820
862
|
`[eval] total: ${(wallMs / 1000).toFixed(1)}s wall (${totalCriteriaCount} parallel) | ` +
|
|
821
863
|
`in=${totalIn} out=${totalOut} cached~=${cachedTokensEstimate} | ` +
|
|
822
|
-
`issues=${allIssues.length} pass=${passCategories.length}
|
|
864
|
+
`issues=${allIssues.length} pass=${passCategories.length}` +
|
|
865
|
+
(skippedCount > 0 ? ` skipped=${skippedCount}` : ''),
|
|
823
866
|
);
|
|
824
867
|
|
|
825
868
|
return {
|
|
826
869
|
issues: allIssues,
|
|
827
870
|
pass: passCategories,
|
|
871
|
+
criteriaCoverage,
|
|
828
872
|
inputTokens: totalIn,
|
|
829
873
|
outputTokens: totalOut,
|
|
830
874
|
};
|
|
@@ -87,9 +87,88 @@ export interface RuntimeProbeMeta {
|
|
|
87
87
|
readonly reason?: string;
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
+
/**
|
|
91
|
+
* Per-criterion execution status — three-valued so "the criterion
|
|
92
|
+
* silently produced no verdict" is never conflated with "the criterion
|
|
93
|
+
* ran clean", and neither is conflated with "not evaluated by design".
|
|
94
|
+
* Same doctrine and vocabulary family as `RuntimeProbeStatus` above.
|
|
95
|
+
*
|
|
96
|
+
* - `ran` — the LLM call completed and returned a verdict
|
|
97
|
+
* for this criterion; its findings (possibly
|
|
98
|
+
* none) are in `issues`.
|
|
99
|
+
* - `skipped` — the call errored or returned no tool call. The
|
|
100
|
+
* criterion carries ZERO evidence; the evaluator
|
|
101
|
+
* fail-opens it into `pass` to avoid blocking
|
|
102
|
+
* generation on flaky LLM calls, so consumers
|
|
103
|
+
* MUST consult this field before treating `pass`
|
|
104
|
+
* membership as a verdict. Counts AGAINST
|
|
105
|
+
* coverage.
|
|
106
|
+
* - `not-applicable` — the evaluator was bypassed by design for this
|
|
107
|
+
* cell (e.g. the same-image low-risk bypass:
|
|
108
|
+
* axis checks clean, tier-1/2 never invoked).
|
|
109
|
+
* `reason` names the bypass. Excluded from
|
|
110
|
+
* coverage denominators — it is neither evidence
|
|
111
|
+
* nor a silent absence.
|
|
112
|
+
*/
|
|
113
|
+
export type CriterionRunStatus = "ran" | "skipped" | "not-applicable";
|
|
114
|
+
|
|
115
|
+
/** Coverage row for one evaluation criterion of one eval run. */
|
|
116
|
+
export interface CriterionCoverage {
|
|
117
|
+
readonly criterion: string;
|
|
118
|
+
readonly tier: 1 | 2;
|
|
119
|
+
readonly status: CriterionRunStatus;
|
|
120
|
+
/** Populated for `skipped` / `not-applicable` — why no verdict. */
|
|
121
|
+
readonly reason?: string;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* The static criteria `runLLMEvaluation` runs, in stamp order — the
|
|
126
|
+
* ONE list both the evaluator's coverage stamp and the harness's
|
|
127
|
+
* bypass stamp derive from, so the two can never disagree on the set.
|
|
128
|
+
* Tier 1 = binary pass/fail gates; tier 2 = fail/warn/pass quality.
|
|
129
|
+
*/
|
|
130
|
+
export const LLM_EVAL_STATIC_CRITERIA: ReadonlyArray<{ readonly criterion: string; readonly tier: 1 | 2 }> = [
|
|
131
|
+
{ criterion: "functionality", tier: 1 },
|
|
132
|
+
{ criterion: "crash", tier: 1 },
|
|
133
|
+
{ criterion: "interactivity", tier: 2 },
|
|
134
|
+
{ criterion: "accessibility", tier: 2 },
|
|
135
|
+
{ criterion: "layout", tier: 2 },
|
|
136
|
+
{ criterion: "loading", tier: 2 },
|
|
137
|
+
{ criterion: "visual", tier: 2 },
|
|
138
|
+
];
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Coverage rows for a cell where the evaluator was BYPASSED BY DESIGN
|
|
142
|
+
* (never invoked) — one `not-applicable` row per static criterion, all
|
|
143
|
+
* carrying `reason`. Harness exits that skip tier-1/2 evaluation stamp
|
|
144
|
+
* this so the cell is neither a silent absence (field missing) nor
|
|
145
|
+
* evidence. Same shape contract as `runLLMEvaluation`'s stamp: always
|
|
146
|
+
* every criterion. Lives here (not in llm-evaluator) because the bypass
|
|
147
|
+
* exit runs before the evaluator module is lazily loaded.
|
|
148
|
+
*/
|
|
149
|
+
export function notApplicableCoverage(reason: string): CriterionCoverage[] {
|
|
150
|
+
return LLM_EVAL_STATIC_CRITERIA.map(({ criterion, tier }) => ({
|
|
151
|
+
criterion,
|
|
152
|
+
tier,
|
|
153
|
+
status: "not-applicable",
|
|
154
|
+
reason,
|
|
155
|
+
}));
|
|
156
|
+
}
|
|
157
|
+
|
|
90
158
|
export interface EvalResult {
|
|
91
159
|
issues: EvalIssue[];
|
|
92
160
|
pass: string[];
|
|
161
|
+
/**
|
|
162
|
+
* Per-criterion coverage for this eval run — one row per static
|
|
163
|
+
* criterion, ALWAYS all of them when present. Stamped by
|
|
164
|
+
* `runLLMEvaluation` (`ran` / `skipped`) and by the harness on
|
|
165
|
+
* evaluator-bypass exits (`not-applicable` + reason). The harness
|
|
166
|
+
* carries it unchanged through every later re-stamp of this result.
|
|
167
|
+
* Absent = coverage UNKNOWN (paths that predate the instrument, e.g.
|
|
168
|
+
* `parseEvalResponse` replays) — never assume `ran`, never assume
|
|
169
|
+
* bypass.
|
|
170
|
+
*/
|
|
171
|
+
criteriaCoverage?: CriterionCoverage[];
|
|
93
172
|
/**
|
|
94
173
|
* Runtime-render probe execution meta. Stamped by the exit-decision
|
|
95
174
|
* probe runner; absent on eval paths that never invoke the probe.
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
// poor visual hierarchy, missing whitespace.
|
|
13
13
|
|
|
14
14
|
import { build } from 'esbuild';
|
|
15
|
+
import { getCssTokens } from '@ggui-ai/design/rendering';
|
|
15
16
|
import { resolve, dirname } from 'path';
|
|
16
17
|
import { fileURLToPath } from 'url';
|
|
17
18
|
import { writeFileSync, mkdirSync, unlinkSync } from 'fs';
|
|
@@ -69,6 +70,22 @@ export interface VisualEvalContext {
|
|
|
69
70
|
* Produces a self-contained IIFE that only needs React + ReactDOM from CDN.
|
|
70
71
|
* Resolves @ggui-ai/design/* imports from the local monorepo packages.
|
|
71
72
|
*/
|
|
73
|
+
/**
|
|
74
|
+
* The on-disk root of `@ggui-ai/design`, for the esbuild aliases that
|
|
75
|
+
* let JUDGED components resolve the same design source production
|
|
76
|
+
* ships. Exported for the ggui#613 regression pin: the previous inline
|
|
77
|
+
* `'..','..','..','packages','design'` resolved a pre-`oss/`-migration
|
|
78
|
+
* path (`oss/packages/packages/design`) that exists from NEITHER the
|
|
79
|
+
* src nor the dist layout — the visual leg's bundle broke silently at
|
|
80
|
+
* the migration. Three ups from `<pkg>/src/evaluation` (or
|
|
81
|
+
* `<pkg>/dist/evaluation`) is `oss/packages/`; the design package is
|
|
82
|
+
* its direct child.
|
|
83
|
+
*/
|
|
84
|
+
export function resolveDesignPackageDir(): string {
|
|
85
|
+
const selfDir = dirname(fileURLToPath(import.meta.url));
|
|
86
|
+
return resolve(selfDir, '..', '..', '..', 'design');
|
|
87
|
+
}
|
|
88
|
+
|
|
72
89
|
async function bundleForRendering(
|
|
73
90
|
compiledCode: string,
|
|
74
91
|
sampleProps: Record<string, unknown>,
|
|
@@ -79,10 +96,7 @@ async function bundleForRendering(
|
|
|
79
96
|
const componentFile = resolve(tmpDir, 'component.tsx');
|
|
80
97
|
const entryFile = resolve(tmpDir, 'entry.tsx');
|
|
81
98
|
|
|
82
|
-
|
|
83
|
-
const __filename = fileURLToPath(import.meta.url);
|
|
84
|
-
const __dirname = dirname(__filename);
|
|
85
|
-
const designPkgDir = resolve(__dirname, '..', '..', '..', 'packages', 'design');
|
|
99
|
+
const designPkgDir = resolveDesignPackageDir();
|
|
86
100
|
|
|
87
101
|
// Write the compiled component as a separate module
|
|
88
102
|
writeFileSync(componentFile, compiledCode);
|
|
@@ -138,13 +152,21 @@ function buildRenderHTML(
|
|
|
138
152
|
bundledCode: string,
|
|
139
153
|
cssTokens?: string,
|
|
140
154
|
): string {
|
|
155
|
+
// Default to the design tokens production's no-theme branch injects
|
|
156
|
+
// (getCssTokens → default theme, light) — ggui#613: under the s4
|
|
157
|
+
// fallback ban the generated component's `var(--ggui-*)` references
|
|
158
|
+
// carry no literal fallbacks, so an un-injected judged render shows
|
|
159
|
+
// unset colors and the multimodal judge grades broken output. An
|
|
160
|
+
// explicit caller value still wins (theme-aware callers pass their
|
|
161
|
+
// own).
|
|
162
|
+
const tokens = cssTokens ?? getCssTokens();
|
|
141
163
|
return `<!DOCTYPE html>
|
|
142
164
|
<html>
|
|
143
165
|
<head>
|
|
144
166
|
<meta charset="utf-8">
|
|
145
167
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
146
168
|
<style>
|
|
147
|
-
${
|
|
169
|
+
${tokens}
|
|
148
170
|
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
|
149
171
|
body {
|
|
150
172
|
font-family: var(--ggui-font-family-sans, system-ui, -apple-system, sans-serif);
|