@ggui-ai/ui-gen 0.13.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/dist/adapters/generation-dispatch.d.ts +2 -2
  2. package/dist/adapters/generation-dispatch.js +192 -58
  3. package/dist/adapters/generation-dispatch.js.map +1 -1
  4. package/dist/adapters/index.js +7 -9
  5. package/dist/adapters/index.js.map +1 -1
  6. package/dist/advanced/index.d.ts +3 -3
  7. package/dist/advanced/index.js +192 -58
  8. package/dist/advanced/index.js.map +1 -1
  9. package/dist/blueprint-validator.js +2 -7
  10. package/dist/blueprint-validator.js.map +1 -1
  11. package/dist/boilerplate.js +4 -4
  12. package/dist/boilerplate.js.map +1 -1
  13. package/dist/check/index.d.ts +1 -1
  14. package/dist/check/index.js +57 -16
  15. package/dist/check/index.js.map +1 -1
  16. package/dist/{create-ui-generator-CF0EL0Li.d.ts → create-ui-generator-D6TKuv9V.d.ts} +2 -2
  17. package/dist/{dispatch-D75M-OfZ.d.ts → dispatch-BuZfTh9r.d.ts} +1 -1
  18. package/dist/evaluation/axis-checks/index.d.ts +3 -3
  19. package/dist/evaluation/axis-checks/index.js.map +1 -1
  20. package/dist/evaluation/axis-checks/registry.d.ts +1 -1
  21. package/dist/evaluation/index.d.ts +3 -3
  22. package/dist/evaluation/index.js +79 -16
  23. package/dist/evaluation/index.js.map +1 -1
  24. package/dist/harness/check/runtime-render/index.d.ts +1 -1
  25. package/dist/harness/check/runtime-render/index.js +2 -7
  26. package/dist/harness/check/runtime-render/index.js.map +1 -1
  27. package/dist/harness/index.d.ts +2 -2
  28. package/dist/harness/index.js +64 -19
  29. package/dist/harness/index.js.map +1 -1
  30. package/dist/harness/prompts.js +2 -0
  31. package/dist/harness/prompts.js.map +1 -1
  32. package/dist/harness/result-types.d.ts +1 -1
  33. package/dist/harness/runtime.d.ts +2 -2
  34. package/dist/harness/runtime.js +5 -5
  35. package/dist/harness/runtime.js.map +1 -1
  36. package/dist/harness/types-public.d.ts +1 -1
  37. package/dist/harness/types-public.js +4 -4
  38. package/dist/harness/types-public.js.map +1 -1
  39. package/dist/harness/validator-trace-sink.d.ts +1 -1
  40. package/dist/index.d.ts +3 -3
  41. package/dist/index.js +192 -58
  42. package/dist/index.js.map +1 -1
  43. package/dist/{llm-router-BEVtbAb9.d.ts → llm-router-D7fjBhtr.d.ts} +39 -0
  44. package/dist/patch.d.ts +1 -1
  45. package/dist/provider-adapter-contract.js +1017 -731
  46. package/dist/provider-adapter-contract.js.map +1 -1
  47. package/dist/{types-public-DIpjC-OA.d.ts → types-public-CkNWEov9.d.ts} +64 -1
  48. package/dist/validation/index.d.ts +1 -1
  49. package/dist/validation/index.js +1 -1
  50. package/dist/validation/index.js.map +1 -1
  51. package/dist/workflows.d.ts +1 -1
  52. package/package.json +21 -21
  53. package/src/adapters/provider-router.ts +9 -0
  54. package/src/boilerplate/system-prompt.ts +4 -4
  55. package/src/check/run-tier0.ts +109 -22
  56. package/src/check/type-checker.ts +12 -2
  57. package/src/evaluation/index.ts +4 -0
  58. package/src/evaluation/llm-evaluator.ts +52 -8
  59. package/src/evaluation/types-public.ts +79 -0
  60. package/src/evaluation/visual-evaluator.ts +27 -5
  61. package/src/harness/coding/run-eval-round.ts +32 -6
  62. package/src/harness/llm-router.ts +128 -2
  63. package/src/validation/primitives.ts +1 -1
@@ -1,7 +1,7 @@
1
1
  import { C as Classification } from './axes-CzLEMDeB.js';
2
2
  import { Workflow } from './harness/types-public.js';
3
3
  import '@ggui-ai/protocol';
4
- import './types-public-DIpjC-OA.js';
4
+ import './types-public-CkNWEov9.js';
5
5
  import './types-BOvHNG7K.js';
6
6
  import './llm.js';
7
7
  import './policy.js';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ggui-ai/ui-gen",
3
- "version": "0.13.0",
3
+ "version": "0.15.0",
4
4
  "description": "UI generation harness for ggui — the LLM system prompt, multi-turn coding loop, and evaluation engine that turn a DataContract into a React component. Implements the UiGenerator contract from @ggui-ai/mcp-server-core; consumed by @ggui-ai/mcp-server.",
5
5
  "license": "Apache-2.0",
6
6
  "keywords": [
@@ -183,14 +183,14 @@
183
183
  }
184
184
  },
185
185
  "peerDependencies": {
186
- "@google/adk": "^0.5.0",
187
- "@openai/agents": "^0.7.1",
186
+ "@google/adk": "^2.0.0",
187
+ "@openai/agents": "^0.17.0",
188
188
  "puppeteer-core": "^24.40.0 || ^25.0.0",
189
189
  "react": "^18.0.0 || ^19.0.0",
190
190
  "react-dom": "^18.0.0 || ^19.0.0",
191
- "@ggui-ai/mcp-server-core": "0.13.0",
192
- "@ggui-ai/protocol": "0.13.0",
193
- "@ggui-ai/project-config": "0.13.0"
191
+ "@ggui-ai/mcp-server-core": "0.15.0",
192
+ "@ggui-ai/protocol": "0.15.0",
193
+ "@ggui-ai/project-config": "0.15.0"
194
194
  },
195
195
  "peerDependenciesMeta": {
196
196
  "@google/adk": {
@@ -210,34 +210,34 @@
210
210
  }
211
211
  },
212
212
  "dependencies": {
213
- "@anthropic-ai/bedrock-sdk": "^0.32.1",
213
+ "@anthropic-ai/bedrock-sdk": "^0.33.3",
214
214
  "@anthropic-ai/claude-agent-sdk": "0.3.229",
215
- "@anthropic-ai/sdk": "^0.116.0",
215
+ "@anthropic-ai/sdk": "^0.123.0",
216
216
  "@google/genai": "^2.8.0",
217
217
  "@testing-library/dom": "^10.4.0",
218
218
  "@testing-library/react": "^16.0.0",
219
219
  "@testing-library/user-event": "^14.6.1",
220
220
  "@typescript-eslint/parser": "^8.0.0",
221
221
  "diff": "^8.0.3",
222
- "esbuild": "^0.25.0",
222
+ "esbuild": "^0.28.2",
223
223
  "eslint": "^8.56.0",
224
224
  "eslint-plugin-react": "^7.37.0",
225
225
  "eslint-plugin-react-hooks": "^5.0.0",
226
226
  "happy-dom": "^20.8.9",
227
227
  "isomorphic-git": "^1.37.4",
228
228
  "memfs": "^4.56.11",
229
- "openai": "^6.29.0",
229
+ "openai": "^7.9.0",
230
230
  "typescript": "^5.0.0",
231
231
  "zod": "^4.3.6",
232
- "@ggui-ai/design": "0.13.0",
233
- "@ggui-ai/gadgets": "0.13.0",
234
- "@ggui-ai/ui-visual-tester": "0.13.0",
235
- "@ggui-ai/sandbox": "0.13.0",
236
- "@ggui-ai/wire": "0.13.0"
232
+ "@ggui-ai/gadgets": "0.15.0",
233
+ "@ggui-ai/sandbox": "0.15.0",
234
+ "@ggui-ai/ui-visual-tester": "0.15.0",
235
+ "@ggui-ai/wire": "0.15.0",
236
+ "@ggui-ai/design": "0.15.0"
237
237
  },
238
238
  "devDependencies": {
239
- "@google/adk": "^0.5.0",
240
- "@openai/agents": "^0.7.1",
239
+ "@google/adk": "^2.0.0",
240
+ "@openai/agents": "^0.17.0",
241
241
  "@sparticuz/chromium": "^149.0.0",
242
242
  "@types/eslint": "^9.6.1",
243
243
  "@types/node": "^22.0.0",
@@ -248,12 +248,12 @@
248
248
  "react-dom": "^19.0.0",
249
249
  "tsup": "^8.5.0",
250
250
  "vitest": "^3.2.6",
251
- "@ggui-ai/mcp-server-core": "0.13.0",
252
- "@ggui-ai/project-config": "0.13.0",
253
- "@ggui-ai/protocol": "0.13.0"
251
+ "@ggui-ai/mcp-server-core": "0.15.0",
252
+ "@ggui-ai/project-config": "0.15.0",
253
+ "@ggui-ai/protocol": "0.15.0"
254
254
  },
255
255
  "engines": {
256
- "node": ">=20.0.0"
256
+ "node": ">=22.0.0"
257
257
  },
258
258
  "author": "ggui contributors <hello@ggui.ai>",
259
259
  "scripts": {
@@ -30,6 +30,15 @@ export function getBedrockModelId(model: string): string {
30
30
  'anthropic/claude-haiku-4-5': 'us.anthropic.claude-haiku-4-5-20251001-v1:0',
31
31
  'anthropic/claude-sonnet-4-6': 'us.anthropic.claude-sonnet-4-6',
32
32
  'anthropic/claude-opus-4-6': 'us.anthropic.claude-opus-4-6-v1:0',
33
+ // Claude 5-family (ggui#706, strings verified 2026-09-02 against
34
+ // platform.claude.com; Bedrock base form `anthropic.<id>`, upcast to
35
+ // the cross-region profile). Explicit so the ids are pinned by test,
36
+ // not merely reachable through the generic fallback below.
37
+ 'anthropic/claude-fable-5-1': 'us.anthropic.claude-fable-5-1',
38
+ 'anthropic/claude-opus-5': 'us.anthropic.claude-opus-5',
39
+ 'anthropic/claude-sonnet-5': 'us.anthropic.claude-sonnet-5',
40
+ // Legacy (still available; registry state `legacy`, not deprecated).
41
+ 'anthropic/claude-fable-5': 'us.anthropic.claude-fable-5',
33
42
  };
34
43
  // Normalize the Bedrock dot-form opt-in (e.g. `anthropic.claude-haiku-4-5`,
35
44
  // produced when cloud-pod's `resolvePoolRoute` strips a `bedrock/` opt-in
@@ -574,17 +574,17 @@ MANDATORY:
574
574
  1. NEVER use hardcoded hex colors like \`#7c3aed\` — ONLY \`var(--ggui-color-*)\` tokens.
575
575
  2. NEVER use CSS gradients with custom colors. If you need a gradient: \`linear-gradient(to bottom, var(--ggui-color-primary-500), var(--ggui-color-primary-700))\`.
576
576
  3. NEVER invent your own palette. The system provides primary, neutral, success, warning, error, and info — use ONLY these.
577
- 4. NEVER include literal fallback values in color tokens — write \`var(--ggui-color-primary-600)\` bare, never \`var(--ggui-color-primary-600, #0284c7)\`. The runtime injects every token; a literal fallback paints the wrong brand exactly when the operator's theme matters most.
577
+ 4. NEVER include literal fallback values in ANY token reference — colors, spacing, typography, radius, shadows alike. Write \`var(--ggui-spacing-4)\` bare, never \`var(--ggui-spacing-4, 16px)\`; \`var(--ggui-color-primary-600)\` bare, never \`var(--ggui-color-primary-600, #0284c7)\`. The runtime injects every token; a literal fallback paints the wrong value exactly when the operator's theme matters most.
578
578
 
579
579
  Token categories:
580
580
  - Brand: \`var(--ggui-color-primary-600)\`, \`var(--ggui-color-primary-50)\`
581
581
  - Text: \`var(--ggui-color-onSurface)\`, \`var(--ggui-color-onSurfaceVariant)\`
582
582
  - Backgrounds: \`var(--ggui-color-surface)\`, \`var(--ggui-color-surfaceVariant)\`
583
583
  - Borders: \`var(--ggui-color-outline)\`
584
- - Spacing: \`var(--ggui-spacing-4, 16px)\`, \`var(--ggui-spacing-6, 24px)\`
585
- - Typography: \`var(--ggui-font-size-sm, 14px)\`, \`var(--ggui-font-weight-semibold, 600)\`
584
+ - Spacing: \`var(--ggui-spacing-4)\`, \`var(--ggui-spacing-6)\`
585
+ - Typography: \`var(--ggui-font-size-sm)\`, \`var(--ggui-font-weight-semibold)\`
586
586
  - Shadows: \`var(--ggui-shape-shadow-sm)\`, \`var(--ggui-shape-shadow-md)\`, \`var(--ggui-shape-shadow-lg)\`
587
- - Radius: \`var(--ggui-shape-radius-md, 8px)\`, \`var(--ggui-shape-radius-lg, 12px)\`
587
+ - Radius: \`var(--ggui-shape-radius-md)\`, \`var(--ggui-shape-radius-lg)\`
588
588
 
589
589
  Prefer primitives' built-in styling props over inline styles when possible.
590
590
 
@@ -15,6 +15,7 @@
15
15
  import ts from 'typescript';
16
16
  import type { DataContract } from '@ggui-ai/protocol';
17
17
  import { HOOK_NAME_RE, listContractGadgets } from '@ggui-ai/protocol';
18
+ import { consumedTokenManifest } from '@ggui-ai/design/themes';
18
19
  import { validateAllContracts, type ContractIssue } from './contract-validation.js';
19
20
  import { typecheck } from './type-checker.js';
20
21
  import { lintReactHooks, type ReactLintDiagnostic } from './react-linter.js';
@@ -260,6 +261,17 @@ function detectCertainDoubleWiredActions(
260
261
  return issues;
261
262
  }
262
263
 
264
+ /**
265
+ * The consumed-token manifest as a closed set (ggui#613 residual 3 —
266
+ * the generation-time closure leg of the #598 record). Every theme
267
+ * MUST define exactly these names (the coverage gate); a generated
268
+ * component may therefore consume ONLY these names. Anything else
269
+ * is a reference no theme is obligated to define, and under the s4
270
+ * fallback ban it paints nothing.
271
+ */
272
+ const MANIFEST_TOKENS: ReadonlySet<string> = new Set(consumedTokenManifest);
273
+ const TOKEN_REF_RE = /var\(\s*(--ggui-[a-zA-Z0-9-]+)/g;
274
+
263
275
  /** Map wire kind to the matching `@ggui-ai/wire` hook name. */
264
276
  const HOOK_NAME_FOR: Readonly<Record<WireKind, string>> = {
265
277
  action: 'useAction',
@@ -507,30 +519,45 @@ export async function runTier0Checks(
507
519
  // patterns flowing untouched. Asset-fixed brand colors get a
508
520
  // typed escape on Box (`assetColor` + `assetSemantic`) — see
509
521
  // T-4 follow-up.
522
+ // Literal fallback inside ANY token reference (ggui#598 slice 4
523
+ // origin-kill, generalized by ggui#613 residual 4 from color to
524
+ // every token): the runtime injects every token — colors, spacing,
525
+ // typography, radius, shadows — so a literal fallback can only
526
+ // ever paint default-brand pixels under a failed injection, which
527
+ // must be visible (unset) rather than silently default. The old
528
+ // hex-only exemption plus a remediation that suggested
529
+ // `var(--x, ${hex})` actively taught the model to launder invented
530
+ // values INTO fallbacks (adversarial-cycle find). A token-to-token
531
+ // fallback (`var(--a, var(--b))`) is not a literal — it stays on
532
+ // the manifest check alone.
533
+ const literalFallbackMatch = line.match(
534
+ /var\(\s*(--ggui-[a-zA-Z0-9-]+)\s*,\s*([^)]+)\)/,
535
+ );
536
+ const literalFallback =
537
+ literalFallbackMatch !== null &&
538
+ !literalFallbackMatch[2]!.trim().startsWith('var(')
539
+ ? { token: literalFallbackMatch[1]!, literal: literalFallbackMatch[2]!.trim() }
540
+ : null;
541
+ if (literalFallback !== null && !assetEscapedLines.has(lineNum)) {
542
+ issues.push({
543
+ tier: 0,
544
+ result: 'fail',
545
+ category: 'tokens',
546
+ subcategory: 'token-fallback',
547
+ severity: 'critical',
548
+ description: `Literal fallback "${literalFallback.literal}" inside a token reference — the runtime injects every token, and the fallback paints the wrong value exactly when the theme matters most.`,
549
+ fix: `Drop the fallback: write var(${literalFallback.token}) bare. If the token might not exist, that is a vocabulary problem to fix by choosing a token from the design-system docs, never by a literal.`,
550
+ line: lineNum,
551
+ });
552
+ }
553
+
510
554
  const hexMatch = line.match(/#[0-9a-fA-F]{3,8}\b/);
511
555
  if (hexMatch && !assetEscapedLines.has(lineNum)) {
512
- // ggui#598 slice 4 (origin-kill): hex inside a `var(--ggui-*, #hex)`
513
- // fallback is ALSO a violation now — the runtime injects every
514
- // token, so a literal fallback can only ever paint default-brand
515
- // pixels under a failed injection, which must be visible (unset)
516
- // rather than silently sky-blue under someone's brand. The old
517
- // exemption plus a remediation text that suggested
518
- // `var(--x, ${hex})` actively taught the model to launder its
519
- // invented hex INTO fallbacks (adversarial-cycle find).
520
- const fallbackVarMatch = line.match(
521
- /var\((--ggui-[a-zA-Z0-9-]+)\s*,[^)]*#[0-9a-fA-F]{3,8}/,
522
- );
523
- if (fallbackVarMatch) {
524
- issues.push({
525
- tier: 0,
526
- result: 'fail',
527
- category: 'tokens',
528
- subcategory: 'token-fallback',
529
- severity: 'critical',
530
- description: `Literal fallback "${hexMatch[0]}" inside a token reference — the runtime injects every token, and the fallback paints the wrong brand exactly when the theme matters most.`,
531
- fix: `Drop the fallback: write var(${fallbackVarMatch[1]}) bare. If the token might not exist, that is a vocabulary problem to fix by choosing a token from the design-system docs, never by a literal.`,
532
- line: lineNum,
533
- });
556
+ // A hex that lives inside a token fallback is already reported
557
+ // above as `token-fallback` — don't double-report it as
558
+ // `hex-color`.
559
+ if (literalFallback !== null && literalFallback.literal.includes(hexMatch[0])) {
560
+ // reported above
534
561
  } else if (!line.includes('// fallback')) {
535
562
  issues.push({
536
563
  tier: 0,
@@ -569,6 +596,31 @@ export async function runTier0Checks(
569
596
  });
570
597
  }
571
598
 
599
+ // Off-manifest token references (ggui#613 residual 3): a
600
+ // `var(--ggui-*)` whose name is outside the consumed-token
601
+ // manifest. The manifest is the closed vocabulary the prompt
602
+ // teaches AND the set every theme must define — so a name outside
603
+ // it is invented, and with literal fallbacks banned it renders
604
+ // UNSET. Fail, like the other token-contract violations; one issue
605
+ // per distinct name per line.
606
+ const offManifest = new Set<string>();
607
+ for (const m of line.matchAll(TOKEN_REF_RE)) {
608
+ const name = m[1]!;
609
+ if (!MANIFEST_TOKENS.has(name)) offManifest.add(name);
610
+ }
611
+ for (const name of offManifest) {
612
+ issues.push({
613
+ tier: 0,
614
+ result: 'fail',
615
+ category: 'tokens',
616
+ subcategory: 'off-manifest-token',
617
+ severity: 'critical',
618
+ description: `Token "${name}" is not in the design system's token manifest — no theme defines it, so it renders unset.`,
619
+ fix: `Choose a token that exists: the closed vocabulary is the manifest the design-system docs list (colors: var(--ggui-color-primary-500), var(--ggui-color-onSurface), …; spacing: var(--ggui-spacing-4); type: var(--ggui-font-size-sm)). Never invent a name and never add a literal fallback.`,
620
+ line: lineNum,
621
+ });
622
+ }
623
+
572
624
  // Raw CSS length on a spacing / radius prop — bypasses the design
573
625
  // scale. Same reasoning as the hardcoded-color checks above: a
574
626
  // literal `gap="8px"` defeats the theme's spacing scale exactly as
@@ -746,6 +798,41 @@ export async function runTier0Checks(
746
798
  );
747
799
  }
748
800
 
801
+ // ── Connection suppression (ggui#670 Phase 2) ────────────────
802
+ // `useRender().isConnected` is a LIVE read of the relay latch, and
803
+ // the ATTEMPT is the runtime's self-heal sensor (#440/#443
804
+ // attempt-always): a dispatch fired while the latch stands is how the
805
+ // runtime discovers the host can relay again. Binding `disabled` to
806
+ // the connection state strips the handler and blinds that sensor —
807
+ // the control stays dead after recovery. The runtime owns the dead
808
+ // zone's presentation (ggui#670 Phase 3: standing notice, then a
809
+ // per-gesture cue) — generated code never suppresses on it. Direct
810
+ // binding only, and only when the source reads
811
+ // `useRender` at all — a same-named `props.isConnected` (a device-
812
+ // status card) is not the connection state. Whole-source scan so a
813
+ // multi-line attribute is caught; the issue reports the attribute's
814
+ // line. The lookbehind keeps `aria-disabled={…}` (the idiom) out.
815
+ if (/\buseRender\s*\(/.test(sourceCode)) {
816
+ const suppressionRe = /(?<![\w-])disabled\s*=\s*\{([^}]*)\}/gs;
817
+ for (const m of sourceCode.matchAll(suppressionRe)) {
818
+ const expr = m[1]!;
819
+ if (!/\bisConnected\b/.test(expr)) continue;
820
+ const attrLine = sourceCode.slice(0, m.index ?? 0).split('\n').length;
821
+ issues.push({
822
+ tier: 0,
823
+ result: 'fail',
824
+ category: 'interactivity',
825
+ subcategory: 'connection-suppression',
826
+ severity: 'critical',
827
+ description:
828
+ `\`disabled\` is bound to the connection state (\`${expr.trim().replace(/\s+/g, ' ')}\`) — that strips the click handler, and the click is how the runtime detects that the host can relay actions again. The control stays dead after recovery.`,
829
+ fix:
830
+ 'Leave the control enabled — never bind `disabled` to the connection state. The runtime presents the dead zone itself (a standing notice, then a per-gesture cue) and the attempt is how it detects recovery; a suppressed control stays dead after the host recovers.',
831
+ line: attrLine,
832
+ });
833
+ }
834
+ }
835
+
749
836
  // ── Optional props accessed without guard ──────────────────
750
837
  // Check if Props interface has optional fields that are accessed without ?. or ?? or &&
751
838
  const propsInterfaceMatch = sourceCode.match(/interface Props\s*\{([^}]+)\}/);
@@ -316,10 +316,18 @@ async function loadVfs(): Promise<Map<string, VfsEntry>> {
316
316
  /**
317
317
  * Try to resolve a relative import within the VFS.
318
318
  * Given `containingFile` (e.g. `node_modules/@ggui-ai/design/primitives/index.d.ts`)
319
- * and `importPath` (e.g. `./Card`), try:
319
+ * and `importPath` (e.g. `./Card` or, as every published declaration says
320
+ * since 0.15.0, `./Card.js`), try:
320
321
  * - <resolved>.d.ts
321
322
  * - <resolved>/index.d.ts
322
323
  * - <resolved>.ts
324
+ * A `.js` / `.mjs` / `.cjs` extension on the specifier is the ESM
325
+ * convention for "the module that compiles to this file" — TypeScript's
326
+ * NodeNext maps `./Card.js` to `./Card.d.ts`, and so does this: the
327
+ * extension is stripped before the candidates are tried. Without that, a
328
+ * `.js`-suffixed re-export resolved to nothing, the imported names typed
329
+ * as `any`, and the checker reported zero errors on code it must reject
330
+ * (ggui 0.15.0 publish-gate, type-checker.test.ts).
323
331
  */
324
332
  function resolveRelativeInVfs(
325
333
  vfs: Map<string, VfsEntry>,
@@ -329,7 +337,9 @@ function resolveRelativeInVfs(
329
337
  // Strip leading '/' from containingFile — TS may prepend getCurrentDirectory()
330
338
  const normalized = containingFile.startsWith('/') ? containingFile.slice(1) : containingFile;
331
339
  const dir = path.posix.dirname(normalized);
332
- const resolved = path.posix.normalize(`${dir}/${importPath}`);
340
+ // `./x.js` (`.mjs`, `.cjs`) names the declaration `./x.d.ts` — strip it.
341
+ const specifier = importPath.replace(/\.(?:m|c)?js$/, '');
342
+ const resolved = path.posix.normalize(`${dir}/${specifier}`);
333
343
 
334
344
  // Try .d.ts extension
335
345
  const withDts = `${resolved}.d.ts`;
@@ -23,6 +23,8 @@ export type {
23
23
  EvalCategory,
24
24
  EvalResult,
25
25
  EvalCriterion,
26
+ CriterionCoverage,
27
+ CriterionRunStatus,
26
28
  QualityMode,
27
29
  QualityConfig,
28
30
  AxisCheck,
@@ -30,6 +32,8 @@ export type {
30
32
  } from './types-public.js';
31
33
  export {
32
34
  CRITERIA,
35
+ LLM_EVAL_STATIC_CRITERIA,
36
+ notApplicableCoverage,
33
37
  DEFAULT_QUALITY_CONFIG,
34
38
  matches,
35
39
  priorityForIssue,
@@ -14,7 +14,14 @@
14
14
  // Caching: Anthropic ephemeral cache, OpenAI auto-cache, Google explicit cache.
15
15
  // Same system prompt across all 7 calls → first call caches, 6 remaining hit cache.
16
16
 
17
- import type { EvalIssue, EvalResult, EvalCategory } from './types-public.js';
17
+ import type {
18
+ EvalIssue,
19
+ EvalResult,
20
+ EvalCategory,
21
+ CriterionCoverage,
22
+ CriterionRunStatus,
23
+ } from './types-public.js';
24
+ import { LLM_EVAL_STATIC_CRITERIA } from './types-public.js';
18
25
  import { createAgent } from '../harness/llm-router';
19
26
  import type { AgentConfig, LLMToolDef, LLMAgent } from '../harness/llm-router';
20
27
  import type { JsonObject } from '@ggui-ai/protocol';
@@ -518,13 +525,26 @@ interface CriterionResult {
518
525
  tier: 1 | 2;
519
526
  issues: EvalIssue[];
520
527
  pass: boolean;
528
+ /**
529
+ * Did this criterion actually produce a verdict? The evaluator
530
+ * fail-opens (`pass: true`) on LLM errors and missing tool calls so a
531
+ * flaky call never blocks generation — this field is what keeps that
532
+ * fail-open honest: `skipped` means `pass` carries zero evidence.
533
+ */
534
+ coverage: CriterionRunStatus;
535
+ /** Populated for `skipped` — why the criterion produced no verdict. */
536
+ skipReason?: string;
521
537
  tokens: { in: number; out: number };
522
538
  durationMs: number;
523
539
  }
524
540
 
525
- /** All valid categories for LLM evaluation */
526
- const TIER_1_CRITERIA = ['functionality', 'crash'] as const;
527
- const TIER_2_CRITERIA = ['interactivity', 'accessibility', 'layout', 'loading', 'visual'] as const;
541
+ /**
542
+ * All valid categories for LLM evaluation — derived from the ONE static
543
+ * list in types-public so the coverage stamp here and the harness's
544
+ * bypass stamp (`notApplicableCoverage`) can never disagree on the set.
545
+ */
546
+ const TIER_1_CRITERIA = LLM_EVAL_STATIC_CRITERIA.filter((c) => c.tier === 1).map((c) => c.criterion);
547
+ const TIER_2_CRITERIA = LLM_EVAL_STATIC_CRITERIA.filter((c) => c.tier === 2).map((c) => c.criterion);
528
548
 
529
549
  async function evalCriterion(
530
550
  criterion: string,
@@ -557,6 +577,8 @@ async function evalCriterion(
557
577
  tier,
558
578
  issues: [],
559
579
  pass: true,
580
+ coverage: 'skipped',
581
+ skipReason: 'no tool call returned',
560
582
  tokens: { in: response.inputTokens, out: response.outputTokens },
561
583
  durationMs,
562
584
  };
@@ -614,18 +636,25 @@ async function evalCriterion(
614
636
  tier,
615
637
  issues,
616
638
  pass,
639
+ coverage: 'ran',
617
640
  tokens: { in: response.inputTokens, out: response.outputTokens },
618
641
  durationMs,
619
642
  };
620
643
  } catch (e) {
621
644
  const durationMs = Date.now() - start;
622
- console.error(`[eval] ${criterion}: error — ${e instanceof Error ? e.message : e}`);
623
- // On error, treat as pass to avoid blocking on flaky LLM calls
645
+ const message = e instanceof Error ? e.message : String(e);
646
+ console.error(`[eval] ${criterion}: error — ${message}`);
647
+ // On error, treat as pass to avoid blocking on flaky LLM calls.
648
+ // `coverage: 'skipped'` discloses the fail-open: this criterion
649
+ // produced NO verdict, and consumers must not read its `pass`
650
+ // membership as evidence.
624
651
  return {
625
652
  criterion,
626
653
  tier,
627
654
  issues: [],
628
655
  pass: true,
656
+ coverage: 'skipped',
657
+ skipReason: message,
629
658
  tokens: { in: 0, out: 0 },
630
659
  durationMs,
631
660
  };
@@ -809,22 +838,37 @@ export async function runLLMEvaluation(
809
838
 
810
839
  const wallMs = Date.now() - evalStart;
811
840
 
841
+ // Per-criterion coverage — one row per criterion, ALWAYS all of them,
842
+ // so a consumer can tell "ran clean" from "silently didn't run"
843
+ // (the fail-open paths above return pass:true with zero evidence).
844
+ const criteriaCoverage: CriterionCoverage[] = results.map((r) => ({
845
+ criterion: r.criterion,
846
+ tier: r.tier,
847
+ status: r.coverage,
848
+ ...(r.skipReason !== undefined ? { reason: r.skipReason } : {}),
849
+ }));
850
+ const skippedCount = criteriaCoverage.filter((c) => c.status === 'skipped').length;
851
+
812
852
  // Log per-criterion results
813
853
  for (const r of results) {
814
854
  const issueCount = r.issues.length;
815
- const status = r.pass ? 'PASS' : r.tier === 1 ? 'FAIL' : (r.issues.some(i => i.result === 'fail') ? 'FAIL' : 'WARN');
855
+ const status = r.coverage === 'skipped'
856
+ ? 'SKIPPED'
857
+ : r.pass ? 'PASS' : r.tier === 1 ? 'FAIL' : (r.issues.some(i => i.result === 'fail') ? 'FAIL' : 'WARN');
816
858
  const issueLabel = issueCount > 0 ? ` (${issueCount} issue${issueCount > 1 ? 's' : ''})` : '';
817
859
  console.log(`[eval] ${r.criterion}: ${status}${issueLabel} ${(r.durationMs / 1000).toFixed(1)}s`);
818
860
  }
819
861
  console.log(
820
862
  `[eval] total: ${(wallMs / 1000).toFixed(1)}s wall (${totalCriteriaCount} parallel) | ` +
821
863
  `in=${totalIn} out=${totalOut} cached~=${cachedTokensEstimate} | ` +
822
- `issues=${allIssues.length} pass=${passCategories.length}`,
864
+ `issues=${allIssues.length} pass=${passCategories.length}` +
865
+ (skippedCount > 0 ? ` skipped=${skippedCount}` : ''),
823
866
  );
824
867
 
825
868
  return {
826
869
  issues: allIssues,
827
870
  pass: passCategories,
871
+ criteriaCoverage,
828
872
  inputTokens: totalIn,
829
873
  outputTokens: totalOut,
830
874
  };
@@ -87,9 +87,88 @@ export interface RuntimeProbeMeta {
87
87
  readonly reason?: string;
88
88
  }
89
89
 
90
+ /**
91
+ * Per-criterion execution status — three-valued so "the criterion
92
+ * silently produced no verdict" is never conflated with "the criterion
93
+ * ran clean", and neither is conflated with "not evaluated by design".
94
+ * Same doctrine and vocabulary family as `RuntimeProbeStatus` above.
95
+ *
96
+ * - `ran` — the LLM call completed and returned a verdict
97
+ * for this criterion; its findings (possibly
98
+ * none) are in `issues`.
99
+ * - `skipped` — the call errored or returned no tool call. The
100
+ * criterion carries ZERO evidence; the evaluator
101
+ * fail-opens it into `pass` to avoid blocking
102
+ * generation on flaky LLM calls, so consumers
103
+ * MUST consult this field before treating `pass`
104
+ * membership as a verdict. Counts AGAINST
105
+ * coverage.
106
+ * - `not-applicable` — the evaluator was bypassed by design for this
107
+ * cell (e.g. the same-image low-risk bypass:
108
+ * axis checks clean, tier-1/2 never invoked).
109
+ * `reason` names the bypass. Excluded from
110
+ * coverage denominators — it is neither evidence
111
+ * nor a silent absence.
112
+ */
113
+ export type CriterionRunStatus = "ran" | "skipped" | "not-applicable";
114
+
115
+ /** Coverage row for one evaluation criterion of one eval run. */
116
+ export interface CriterionCoverage {
117
+ readonly criterion: string;
118
+ readonly tier: 1 | 2;
119
+ readonly status: CriterionRunStatus;
120
+ /** Populated for `skipped` / `not-applicable` — why no verdict. */
121
+ readonly reason?: string;
122
+ }
123
+
124
+ /**
125
+ * The static criteria `runLLMEvaluation` runs, in stamp order — the
126
+ * ONE list both the evaluator's coverage stamp and the harness's
127
+ * bypass stamp derive from, so the two can never disagree on the set.
128
+ * Tier 1 = binary pass/fail gates; tier 2 = fail/warn/pass quality.
129
+ */
130
+ export const LLM_EVAL_STATIC_CRITERIA: ReadonlyArray<{ readonly criterion: string; readonly tier: 1 | 2 }> = [
131
+ { criterion: "functionality", tier: 1 },
132
+ { criterion: "crash", tier: 1 },
133
+ { criterion: "interactivity", tier: 2 },
134
+ { criterion: "accessibility", tier: 2 },
135
+ { criterion: "layout", tier: 2 },
136
+ { criterion: "loading", tier: 2 },
137
+ { criterion: "visual", tier: 2 },
138
+ ];
139
+
140
+ /**
141
+ * Coverage rows for a cell where the evaluator was BYPASSED BY DESIGN
142
+ * (never invoked) — one `not-applicable` row per static criterion, all
143
+ * carrying `reason`. Harness exits that skip tier-1/2 evaluation stamp
144
+ * this so the cell is neither a silent absence (field missing) nor
145
+ * evidence. Same shape contract as `runLLMEvaluation`'s stamp: always
146
+ * every criterion. Lives here (not in llm-evaluator) because the bypass
147
+ * exit runs before the evaluator module is lazily loaded.
148
+ */
149
+ export function notApplicableCoverage(reason: string): CriterionCoverage[] {
150
+ return LLM_EVAL_STATIC_CRITERIA.map(({ criterion, tier }) => ({
151
+ criterion,
152
+ tier,
153
+ status: "not-applicable",
154
+ reason,
155
+ }));
156
+ }
157
+
90
158
  export interface EvalResult {
91
159
  issues: EvalIssue[];
92
160
  pass: string[];
161
+ /**
162
+ * Per-criterion coverage for this eval run — one row per static
163
+ * criterion, ALWAYS all of them when present. Stamped by
164
+ * `runLLMEvaluation` (`ran` / `skipped`) and by the harness on
165
+ * evaluator-bypass exits (`not-applicable` + reason). The harness
166
+ * carries it unchanged through every later re-stamp of this result.
167
+ * Absent = coverage UNKNOWN (paths that predate the instrument, e.g.
168
+ * `parseEvalResponse` replays) — never assume `ran`, never assume
169
+ * bypass.
170
+ */
171
+ criteriaCoverage?: CriterionCoverage[];
93
172
  /**
94
173
  * Runtime-render probe execution meta. Stamped by the exit-decision
95
174
  * probe runner; absent on eval paths that never invoke the probe.
@@ -12,6 +12,7 @@
12
12
  // poor visual hierarchy, missing whitespace.
13
13
 
14
14
  import { build } from 'esbuild';
15
+ import { getCssTokens } from '@ggui-ai/design/rendering';
15
16
  import { resolve, dirname } from 'path';
16
17
  import { fileURLToPath } from 'url';
17
18
  import { writeFileSync, mkdirSync, unlinkSync } from 'fs';
@@ -69,6 +70,22 @@ export interface VisualEvalContext {
69
70
  * Produces a self-contained IIFE that only needs React + ReactDOM from CDN.
70
71
  * Resolves @ggui-ai/design/* imports from the local monorepo packages.
71
72
  */
73
+ /**
74
+ * The on-disk root of `@ggui-ai/design`, for the esbuild aliases that
75
+ * let JUDGED components resolve the same design source production
76
+ * ships. Exported for the ggui#613 regression pin: the previous inline
77
+ * `'..','..','..','packages','design'` resolved a pre-`oss/`-migration
78
+ * path (`oss/packages/packages/design`) that exists from NEITHER the
79
+ * src nor the dist layout — the visual leg's bundle broke silently at
80
+ * the migration. Three ups from `<pkg>/src/evaluation` (or
81
+ * `<pkg>/dist/evaluation`) is `oss/packages/`; the design package is
82
+ * its direct child.
83
+ */
84
+ export function resolveDesignPackageDir(): string {
85
+ const selfDir = dirname(fileURLToPath(import.meta.url));
86
+ return resolve(selfDir, '..', '..', '..', 'design');
87
+ }
88
+
72
89
  async function bundleForRendering(
73
90
  compiledCode: string,
74
91
  sampleProps: Record<string, unknown>,
@@ -79,10 +96,7 @@ async function bundleForRendering(
79
96
  const componentFile = resolve(tmpDir, 'component.tsx');
80
97
  const entryFile = resolve(tmpDir, 'entry.tsx');
81
98
 
82
- // Find the design package root — navigate from this file to packages/design/
83
- const __filename = fileURLToPath(import.meta.url);
84
- const __dirname = dirname(__filename);
85
- const designPkgDir = resolve(__dirname, '..', '..', '..', 'packages', 'design');
99
+ const designPkgDir = resolveDesignPackageDir();
86
100
 
87
101
  // Write the compiled component as a separate module
88
102
  writeFileSync(componentFile, compiledCode);
@@ -138,13 +152,21 @@ function buildRenderHTML(
138
152
  bundledCode: string,
139
153
  cssTokens?: string,
140
154
  ): string {
155
+ // Default to the design tokens production's no-theme branch injects
156
+ // (getCssTokens → default theme, light) — ggui#613: under the s4
157
+ // fallback ban the generated component's `var(--ggui-*)` references
158
+ // carry no literal fallbacks, so an un-injected judged render shows
159
+ // unset colors and the multimodal judge grades broken output. An
160
+ // explicit caller value still wins (theme-aware callers pass their
161
+ // own).
162
+ const tokens = cssTokens ?? getCssTokens();
141
163
  return `<!DOCTYPE html>
142
164
  <html>
143
165
  <head>
144
166
  <meta charset="utf-8">
145
167
  <meta name="viewport" content="width=device-width, initial-scale=1">
146
168
  <style>
147
- ${cssTokens ?? ''}
169
+ ${tokens}
148
170
  *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
149
171
  body {
150
172
  font-family: var(--ggui-font-family-sans, system-ui, -apple-system, sans-serif);