@ggui-ai/ui-gen 0.21.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/dist/adapters/generation-dispatch.d.ts +1 -1
  2. package/dist/adapters/generation-dispatch.js +198 -59
  3. package/dist/adapters/generation-dispatch.js.map +1 -1
  4. package/dist/adapters/index.js +35 -10
  5. package/dist/adapters/index.js.map +1 -1
  6. package/dist/advanced/index.d.ts +2 -2
  7. package/dist/advanced/index.js +381 -58
  8. package/dist/advanced/index.js.map +1 -1
  9. package/dist/boilerplate.js.map +1 -1
  10. package/dist/check/index.d.ts +1 -1
  11. package/dist/check/index.js.map +1 -1
  12. package/dist/{create-ui-generator-D7kOXNjJ.d.ts → create-ui-generator-BEkzEtNz.d.ts} +1 -1
  13. package/dist/{dispatch-DGvhmARp.d.ts → dispatch-DJNpQ7DI.d.ts} +1 -1
  14. package/dist/evaluation/axis-checks/index.d.ts +3 -3
  15. package/dist/evaluation/axis-checks/index.js.map +1 -1
  16. package/dist/evaluation/axis-checks/registry.d.ts +1 -1
  17. package/dist/evaluation/index.d.ts +3 -3
  18. package/dist/evaluation/index.js +37 -20
  19. package/dist/evaluation/index.js.map +1 -1
  20. package/dist/harness/check/runtime-render/index.d.ts +1 -1
  21. package/dist/harness/index.d.ts +1 -1
  22. package/dist/harness/index.js +34 -9
  23. package/dist/harness/index.js.map +1 -1
  24. package/dist/harness/prompts.js +1 -1
  25. package/dist/harness/prompts.js.map +1 -1
  26. package/dist/harness/result-types.d.ts +15 -2
  27. package/dist/harness/result-types.js.map +1 -1
  28. package/dist/harness/runtime.d.ts +1 -1
  29. package/dist/harness/runtime.js.map +1 -1
  30. package/dist/harness/types-public.d.ts +1 -1
  31. package/dist/harness/types-public.js.map +1 -1
  32. package/dist/harness/validator-trace-sink.d.ts +1 -1
  33. package/dist/index.d.ts +22 -6
  34. package/dist/index.js +381 -58
  35. package/dist/index.js.map +1 -1
  36. package/dist/patch.d.ts +1 -1
  37. package/dist/{types-public-CzYqZclY.d.ts → types-public-mM_-1Yk3.d.ts} +16 -1
  38. package/dist/workflows.d.ts +1 -1
  39. package/package.json +12 -12
  40. package/src/adapters/claude/claude-code-login.ts +62 -7
  41. package/src/adapters/claude/sdk.ts +6 -5
  42. package/src/boilerplate/templates/layouts/chat-mobile.tsx.tmpl +7 -3
  43. package/src/boilerplate/templates/layouts/chat-universal.tsx.tmpl +7 -3
  44. package/src/contract-context.ts +1 -1
  45. package/src/create-ui-generator.ts +30 -0
  46. package/src/evaluation/index.ts +1 -0
  47. package/src/evaluation/llm-evaluator.ts +40 -13
  48. package/src/evaluation/types-public.ts +16 -0
  49. package/src/evaluation/visual-evaluator.ts +9 -6
  50. package/src/generator-build.ts +158 -0
  51. package/src/harness/claude-code-login-query.ts +5 -4
  52. package/src/harness/coding/assemble-result.ts +8 -1
  53. package/src/harness/coding/generate-task-runner.ts +75 -36
  54. package/src/harness/coding/run-coding-turn.ts +9 -0
  55. package/src/harness/coding/run-eval-round.ts +160 -5
  56. package/src/harness/result-types.ts +14 -0
  57. package/src/index.ts +2 -1
package/dist/patch.d.ts CHANGED
@@ -2,7 +2,7 @@ import { PatchFn } from './harness/types-public.js';
2
2
  import '@ggui-ai/protocol';
3
3
  import './styling-profile-CxfvV48g.js';
4
4
  import './axes-CzLEMDeB.js';
5
- import './types-public-CzYqZclY.js';
5
+ import './types-public-mM_-1Yk3.js';
6
6
  import './design-mode-DLufhrNp.js';
7
7
  import './types-BOvHNG7K.js';
8
8
  import './llm.js';
@@ -120,6 +120,13 @@ declare const LLM_EVAL_STATIC_CRITERIA: ReadonlyArray<{
120
120
  * exit runs before the evaluator module is lazily loaded.
121
121
  */
122
122
  declare function notApplicableCoverage(reason: string): CriterionCoverage[];
123
+ /** One contract-feedback round (ggui#1261): what bought it, and the source it was fed back on. */
124
+ interface ContractFeedbackRecord {
125
+ /** The exit-probe issue subcategories that bought the round, e.g. `runtime:prop-sensitivity:currentUser`. */
126
+ readonly firedOn: readonly string[];
127
+ /** The component source as it stood when the model was told — the round's BEFORE. */
128
+ readonly sourceBefore: string;
129
+ }
123
130
  interface EvalResult {
124
131
  issues: EvalIssue[];
125
132
  pass: string[];
@@ -139,6 +146,14 @@ interface EvalResult {
139
146
  * probe runner; absent on eval paths that never invoke the probe.
140
147
  */
141
148
  runtimeProbe?: RuntimeProbeMeta;
149
+ /**
150
+ * The contract-feedback round, when one fired this generation (ggui#1261):
151
+ * which exit-probe findings bought it and the source as it stood when the
152
+ * model was told. `runtimeProbe` and the `runtime:*` issues on this result
153
+ * are the probe's verdict AFTER that round; `sourceBefore` beside the final
154
+ * source is the round's before/after. Absent when no such round fired.
155
+ */
156
+ contractFeedback?: ContractFeedbackRecord;
142
157
  /**
143
158
  * Per-canvas visual verdicts. Stamped by the harness ONLY when the
144
159
  * visual leg ran with `visualEvaluation.canvases` set (arm-neutral —
@@ -344,4 +359,4 @@ declare function getLLMCriteria(designMode?: DesignMode): EvalCriterion[];
344
359
  */
345
360
  declare function buildCodingCriteriaSummary(designMode?: DesignMode): string;
346
361
 
347
- export { type AxisCheck as A, type CanvasJudgeRecord as C, DEFAULT_QUALITY_CONFIG as D, type EvalIssue as E, LLM_EVAL_STATIC_CRITERIA as L, type Priority as P, type QualityConfig as Q, type RuntimeProbeStatus as R, type VisualEvalSummary as V, type AxisCheckInput as a, type EvalResult as b, type EvalTier as c, type VisualCoverage as d, CRITERIA as e, type CanvasVisualSummary as f, type CriterionCoverage as g, type CriterionRunStatus as h, type EvalCategory as i, type EvalCriterion as j, type EvalOutcome as k, type QualityMode as l, matches as m, buildCodingCriteriaSummary as n, criteriaFor as o, getActionableIssues as p, getCriteriaByPriority as q, getCriterionById as r, getLLMCriteria as s, isBlocked as t, notApplicableCoverage as u, priorityForIssue as v };
362
+ export { type AxisCheck as A, type CanvasJudgeRecord as C, DEFAULT_QUALITY_CONFIG as D, type EvalIssue as E, LLM_EVAL_STATIC_CRITERIA as L, type Priority as P, type QualityConfig as Q, type RuntimeProbeStatus as R, type VisualEvalSummary as V, type AxisCheckInput as a, type EvalResult as b, type EvalTier as c, type VisualCoverage as d, CRITERIA as e, type CanvasVisualSummary as f, type ContractFeedbackRecord as g, type CriterionCoverage as h, type CriterionRunStatus as i, type EvalCategory as j, type EvalCriterion as k, type EvalOutcome as l, matches as m, type QualityMode as n, buildCodingCriteriaSummary as o, criteriaFor as p, getActionableIssues as q, getCriteriaByPriority as r, getCriterionById as s, getLLMCriteria as t, isBlocked as u, notApplicableCoverage as v, priorityForIssue as w };
@@ -2,7 +2,7 @@ import { C as Classification } from './axes-CzLEMDeB.js';
2
2
  import { Workflow } from './harness/types-public.js';
3
3
  import '@ggui-ai/protocol';
4
4
  import './styling-profile-CxfvV48g.js';
5
- import './types-public-CzYqZclY.js';
5
+ import './types-public-mM_-1Yk3.js';
6
6
  import './design-mode-DLufhrNp.js';
7
7
  import './types-BOvHNG7K.js';
8
8
  import './llm.js';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ggui-ai/ui-gen",
3
- "version": "0.21.0",
3
+ "version": "0.22.0",
4
4
  "description": "UI generation harness for ggui — the LLM system prompt, multi-turn coding loop, and evaluation engine that turn a DataContract into a React component. Implements the UiGenerator contract from @ggui-ai/mcp-server-core; consumed by @ggui-ai/mcp-server.",
5
5
  "license": "Apache-2.0",
6
6
  "keywords": [
@@ -189,9 +189,9 @@
189
189
  "puppeteer-core": "^24.40.0 || ^25.0.0",
190
190
  "react": "^18.0.0 || ^19.0.0",
191
191
  "react-dom": "^18.0.0 || ^19.0.0",
192
- "@ggui-ai/protocol": "0.21.0",
193
- "@ggui-ai/mcp-server-core": "0.21.0",
194
- "@ggui-ai/project-config": "0.21.0"
192
+ "@ggui-ai/mcp-server-core": "0.22.0",
193
+ "@ggui-ai/project-config": "0.22.0",
194
+ "@ggui-ai/protocol": "0.22.0"
195
195
  },
196
196
  "peerDependenciesMeta": {
197
197
  "@google/adk": {
@@ -233,11 +233,11 @@
233
233
  "openai": "^7.9.0",
234
234
  "typescript": "^5.0.0",
235
235
  "zod": "^4.3.6",
236
- "@ggui-ai/design": "0.21.0",
237
- "@ggui-ai/gadgets": "0.21.0",
238
- "@ggui-ai/sandbox": "0.21.0",
239
- "@ggui-ai/ui-visual-tester": "0.21.0",
240
- "@ggui-ai/wire": "0.21.0"
236
+ "@ggui-ai/design": "0.22.0",
237
+ "@ggui-ai/sandbox": "0.22.0",
238
+ "@ggui-ai/gadgets": "0.22.0",
239
+ "@ggui-ai/ui-visual-tester": "0.22.0",
240
+ "@ggui-ai/wire": "0.22.0"
241
241
  },
242
242
  "devDependencies": {
243
243
  "@google/adk": "^2.0.0",
@@ -252,9 +252,9 @@
252
252
  "react-dom": "^19.0.0",
253
253
  "tsup": "^8.5.0",
254
254
  "vitest": "^3.2.6",
255
- "@ggui-ai/mcp-server-core": "0.21.0",
256
- "@ggui-ai/project-config": "0.21.0",
257
- "@ggui-ai/protocol": "0.21.0"
255
+ "@ggui-ai/mcp-server-core": "0.22.0",
256
+ "@ggui-ai/project-config": "0.22.0",
257
+ "@ggui-ai/protocol": "0.22.0"
258
258
  },
259
259
  "engines": {
260
260
  "node": ">=22.0.0"
@@ -9,9 +9,12 @@
9
9
  * 1. {@link CLAUDE_CODE_LOGIN_CREDENTIAL} — the sentinel a `ProviderKeyRef`
10
10
  * carries in place of a key, the same shape `bedrock-iam` already uses
11
11
  * for "authenticate with ambient credentials, not a key I hold".
12
- * 2. {@link stripProviderKeyEnv} — no provider key may reach the spawned
13
- * process: the binary prefers an env key over its login, so a stale
14
- * `ANTHROPIC_API_KEY` would silently take the run off the login path.
12
+ * 2. {@link loginChildEnv} — the spawned process sees an ALLOWLIST of the
13
+ * parent's env, never the parent's env minus a few names: no provider
14
+ * key (the binary prefers an env key over its login, so a stale
15
+ * `ANTHROPIC_API_KEY` would silently take the run off the login path),
16
+ * no other secret the parent merely holds, and no switch that would
17
+ * change what the binary does (ggui#1278).
15
18
  * 3. {@link claudeCodeLoginQueryOptions} — the SDK options that keep the
16
19
  * run bounded: no built-in tool (`tools: []`), no `~/.claude` settings,
17
20
  * hooks or MCP servers leaking in (`settingSources: []`). Not
@@ -40,14 +43,66 @@ export const PROVIDER_KEY_ENV_NAMES = [
40
43
  "ANTHROPIC_BASE_URL",
41
44
  ] as const;
42
45
 
43
- /** A copy of `env` without any provider key or endpoint override. */
44
- export function stripProviderKeyEnv(
46
+ /**
47
+ * The only env names the login child inherits (ggui#1278), plus every
48
+ * `LC_*` locale name. The spawned binary sees a property of this code, not
49
+ * whatever the parent happened to load: a parent that sourced a whole
50
+ * `.env` holds every provider's key, and a denylist would pass all of them
51
+ * on, together with switches such as `CLAUDE_CODE_USE_BEDROCK` that would
52
+ * silently re-route the run.
53
+ *
54
+ * - Process basics: `PATH` (the binary runs helpers, e.g. the macOS
55
+ * keychain tool), `HOME`/`USER`/`LOGNAME` (where and whose the login
56
+ * store is), `SHELL`, temp dirs, `TERM`, `TZ`, locale.
57
+ * - Where the login lives: `CLAUDE_CONFIG_DIR`, the XDG base dirs.
58
+ * - `CLAUDE_CODE_OAUTH_TOKEN`: the login ITSELF when a headless host
59
+ * supplies it by env (`claude setup-token`). It is the one credential
60
+ * this path may use; every provider key stays out
61
+ * ({@link PROVIDER_KEY_ENV_NAMES} is never on this list).
62
+ * - Network reach: proxy variables in both spellings, and the extra-CA
63
+ * and certificate paths a corporate network needs.
64
+ */
65
+ export const LOGIN_CHILD_ENV_NAMES = [
66
+ "PATH",
67
+ "HOME",
68
+ "USER",
69
+ "LOGNAME",
70
+ "SHELL",
71
+ "TMPDIR",
72
+ "TMP",
73
+ "TEMP",
74
+ "TERM",
75
+ "TZ",
76
+ "LANG",
77
+ "CLAUDE_CONFIG_DIR",
78
+ "CLAUDE_CODE_OAUTH_TOKEN",
79
+ "XDG_CONFIG_HOME",
80
+ "XDG_DATA_HOME",
81
+ "XDG_CACHE_HOME",
82
+ "XDG_STATE_HOME",
83
+ "XDG_RUNTIME_DIR",
84
+ "HTTPS_PROXY",
85
+ "https_proxy",
86
+ "HTTP_PROXY",
87
+ "http_proxy",
88
+ "NO_PROXY",
89
+ "no_proxy",
90
+ "ALL_PROXY",
91
+ "all_proxy",
92
+ "NODE_EXTRA_CA_CERTS",
93
+ "SSL_CERT_FILE",
94
+ "SSL_CERT_DIR",
95
+ ] as const;
96
+
97
+ const LOGIN_CHILD_ENV: ReadonlySet<string> = new Set(LOGIN_CHILD_ENV_NAMES);
98
+
99
+ /** The env the login child is spawned with: the allowlisted names present in `env`, nothing else. */
100
+ export function loginChildEnv(
45
101
  env: Readonly<Record<string, string | undefined>>
46
102
  ): Record<string, string> {
47
103
  const out: Record<string, string> = {};
48
- const drop: ReadonlySet<string> = new Set(PROVIDER_KEY_ENV_NAMES);
49
104
  for (const [k, v] of Object.entries(env)) {
50
- if (v !== undefined && !drop.has(k)) out[k] = v;
105
+ if (v !== undefined && (LOGIN_CHILD_ENV.has(k) || k.startsWith("LC_"))) out[k] = v;
51
106
  }
52
107
  return out;
53
108
  }
@@ -13,7 +13,7 @@ import type { ClaudeSdkConfig, GenerateParams } from '../base';
13
13
  import type { AdapterResult, ProviderName, AdapterMode } from '../types';
14
14
  import { createArtifacts, processMessage } from './message-parser';
15
15
  import { createToolMcpServer } from './tool-bridge';
16
- import { claudeCodeLoginQueryOptions, stripProviderKeyEnv } from './claude-code-login';
16
+ import { claudeCodeLoginQueryOptions, loginChildEnv } from './claude-code-login';
17
17
 
18
18
  export class ClaudeSdkAdapter extends GeneratorAdapter {
19
19
  readonly provider: ProviderName = 'claude';
@@ -41,11 +41,12 @@ export class ClaudeSdkAdapter extends GeneratorAdapter {
41
41
  const { query } = await import('@anthropic-ai/claude-agent-sdk');
42
42
 
43
43
  // Build environment from config or process.env
44
- // ggui#1185 — on the login path no provider key may reach the binary:
45
- // it prefers an env key over its login, so a stale one would take the
46
- // run off this path without a word.
44
+ // ggui#1185/#1278 — on the login path the binary sees an allowlisted
45
+ // env: no provider key (it prefers an env key over its login, so a stale
46
+ // one would take the run off this path without a word), and nothing
47
+ // else the parent merely holds.
47
48
  const login = cfg.claudeCodeLogin === true;
48
- const env = login ? stripProviderKeyEnv(buildEnv(cfg.env)) : buildEnv(cfg.env);
49
+ const env = login ? loginChildEnv(buildEnv(cfg.env)) : buildEnv(cfg.env);
49
50
 
50
51
  const artifacts = createArtifacts();
51
52
  const onProgress = cfg.onProgress;
@@ -1,3 +1,7 @@
1
- <Card padding="sm" shadow="sm">
2
- {/* implement your UI here */}
3
- </Card>
1
+ <Box padding="md">
2
+ {/* implement your UI here — an inline card: the chat bubble around it
3
+ provides the chrome, so add no shadow or outer radius of your own.
4
+ Compact and natural height; stack vertically. This root fills the
5
+ bubble — wrap PROSE in <Container maxWidth="sm"> to cap the reading
6
+ measure (inert at bubble width, composed on a wider frame). */}
7
+ </Box>
@@ -1,3 +1,7 @@
1
- <Card padding="sm" shadow="sm">
2
- {/* implement your UI here */}
3
- </Card>
1
+ <Box padding="md">
2
+ {/* implement your UI here — an inline card: the chat bubble around it
3
+ provides the chrome, so add no shadow or outer radius of your own.
4
+ Compact and natural height; stack vertically. This root fills the
5
+ bubble — wrap PROSE in <Container maxWidth="sm"> to cap the reading
6
+ measure (inert at bubble width, composed on a wider frame). */}
7
+ </Box>
@@ -59,7 +59,7 @@ const SHELL_HINTS: Record<string, string> = {
59
59
  - Padding: use var(--ggui-spacing-2) for inner elements, var(--ggui-spacing-4) for sections. Tight.
60
60
  - Scrolling: parent scrolls (chat feed) — component should NOT have internal scroll
61
61
  - Layout: single column, compact. No sidebar. Stack everything vertically.
62
- - Width: width: 100% (fills parent bubble). Do NOT set max-width.`,
62
+ - Width: the root is width: 100% (fills the bubble) — never cap the ROOT. A prose column inside it carries the reading cap (<Container maxWidth="sm">), which is inert at bubble width.`,
63
63
 
64
64
  fullscreen: `**Fullscreen Shell — Takes over entire viewport**
65
65
  - Container: the whole viewport, edge to edge — the frame stretches your root to its full height (you never set a viewport height yourself)
@@ -56,6 +56,7 @@ import {
56
56
  } from '@ggui-ai/mcp-server-core';
57
57
  import { createGeneratorTools } from './adapters/index.js';
58
58
  import { dispatchGeneration } from './adapters/generation-dispatch.js';
59
+ import { generatorBuild } from './generator-build.js';
59
60
  import { hasProfile } from './boilerplate/styling-profile.js';
60
61
  import { effortDials } from './effort.js';
61
62
  import type { ProviderName } from './adapters/types.js';
@@ -68,6 +69,7 @@ import {
68
69
  } from './contract-context.js';
69
70
  import type { RenderingContext } from './contract-context.js';
70
71
  import type { DesignMode } from './design-mode.js';
72
+ import { DEFAULT_DESIGN_MODE } from './design-mode.js';
71
73
  import { resolveRoute, applyRouteToEnv } from './adapters/provider-router.js';
72
74
  import type { QualityConfig } from './evaluation/types-public.js';
73
75
  import type { EvaluationConfig } from './evaluation/types.js';
@@ -339,6 +341,7 @@ export function createUiGenerator(
339
341
  // Ignored by the constrained prompt (its shell/screen
340
342
  // descriptors stay byte-identical).
341
343
  const canvas = rendering ? canvasForRenderingContext(rendering) : undefined;
344
+ const shellType = harnessShellForRendering(rendering);
342
345
  const promptWithVariance = injectVariance(
343
346
  promptWithRendering,
344
347
  input.variance,
@@ -405,6 +408,8 @@ export function createUiGenerator(
405
408
  ...(onRetry !== undefined ? { onRetry } : {}),
406
409
  ...(designMode !== undefined ? { designMode } : {}),
407
410
  ...(canvas !== undefined ? { canvas } : {}),
411
+ // ggui#1279 — the shell the declared rendering composes for (chat only; see harnessShellForRendering).
412
+ ...(shellType !== undefined ? { shellType } : {}),
408
413
  ...(input.profile !== undefined ? { profile: input.profile } : {}),
409
414
  // Fixture props reach the in-loop probe + visual round only (never the prompt, never identity).
410
415
  ...(input.fixtureProps !== undefined ? { fixtureProps: input.fixtureProps } : {}),
@@ -421,6 +426,8 @@ export function createUiGenerator(
421
426
  latencyMs: Date.now() - startedAt,
422
427
  cacheHit: false,
423
428
  attempts: result.turnsUsed,
429
+ // ggui#1280 — the build that made this generation (template keys, never per-request).
430
+ build: generatorBuild(designMode ?? DEFAULT_DESIGN_MODE),
424
431
  // Prompt-cache counters are provider-specific (Claude reports
425
432
  // them; others omit). Pass through truthfully — absent on the
426
433
  // adapter result stays absent here, never defaulted to 0.
@@ -461,6 +468,8 @@ export function createUiGenerator(
461
468
  outputTokens: 0,
462
469
  latencyMs: Date.now() - startedAt,
463
470
  cacheHit: false,
471
+ // ggui#1280 — a failed generation still names the build that failed.
472
+ build: generatorBuild(designMode ?? DEFAULT_DESIGN_MODE),
464
473
  },
465
474
  };
466
475
  } finally {
@@ -523,6 +532,27 @@ function mapLlmProviderToDispatchProvider(provider: LlmProvider): ProviderName {
523
532
  }
524
533
  }
525
534
 
535
+ /**
536
+ * ggui#1279 — the harness shell a declared rendering composes for. The canvas
537
+ * (`canvasForRenderingContext`) already follows the rendering; the WHAT leg's
538
+ * layout and the HOW leg's shell descriptor did not, so a declared chat card
539
+ * was scaffolded to "fill the frame" while the user prompt and the fit verdict
540
+ * treated it as an inline card. Only a declared CHAT shell changes anything.
541
+ * `fullscreen` IS the harness default, and `partial` has no layout of its own
542
+ * yet, so both pass no shell and those generations stay byte-identical — named
543
+ * here rather than guessed.
544
+ */
545
+ function harnessShellForRendering(rendering: RenderingContext | undefined): 'chat' | undefined {
546
+ if (rendering === undefined) return undefined;
547
+ switch (rendering.shell) {
548
+ case 'chat':
549
+ return 'chat';
550
+ case 'fullscreen':
551
+ case 'partial':
552
+ return undefined;
553
+ }
554
+ }
555
+
526
556
  function mapRendering(
527
557
  rendering: UiGenerateInput['rendering'],
528
558
  ): RenderingContext | undefined {
@@ -22,6 +22,7 @@ export type {
22
22
  EvalOutcome,
23
23
  EvalCategory,
24
24
  EvalResult,
25
+ ContractFeedbackRecord,
25
26
  VisualEvalSummary,
26
27
  CanvasJudgeRecord,
27
28
  CanvasVisualSummary,
@@ -25,7 +25,7 @@ import type {
25
25
  } from './types-public.js';
26
26
  import { LLM_EVAL_STATIC_CRITERIA } from './types-public.js';
27
27
  import { createAgent } from '../harness/llm-router';
28
- import type { AgentConfig, LLMToolDef, LLMAgent } from '../harness/llm-router';
28
+ import type { AgentConfig, LLMToolDef, LLMAgent, LLMToolCallResponse } from '../harness/llm-router';
29
29
  import type { JsonObject } from '@ggui-ai/protocol';
30
30
  import { getCriterionById } from './types-public.js';
31
31
  import { DEFAULT_DESIGN_MODE, type DesignMode } from '../design-mode.js';
@@ -596,10 +596,36 @@ interface CriterionResult {
596
596
  coverage: CriterionRunStatus;
597
597
  /** Populated for `skipped` — why the criterion produced no verdict. */
598
598
  skipReason?: string;
599
- tokens: { in: number; out: number };
599
+ /** This call's token accounting. The cache counts are present only when the
600
+ * provider reported them — absent means unreported, never zero (ggui#1281). */
601
+ tokens: { in: number; out: number; cacheRead?: number; cacheCreation?: number };
600
602
  durationMs: number;
601
603
  }
602
604
 
605
+ /** A call's token accounting, carrying the prompt-cache counts only when the
606
+ * provider reported them (the router's rule: absent is "unreported"). */
607
+ function callTokens(response: LLMToolCallResponse): CriterionResult['tokens'] {
608
+ return {
609
+ in: response.inputTokens,
610
+ out: response.outputTokens,
611
+ ...(response.cacheReadTokens !== undefined ? { cacheRead: response.cacheReadTokens } : {}),
612
+ ...(response.cacheCreationTokens !== undefined ? { cacheCreation: response.cacheCreationTokens } : {}),
613
+ };
614
+ }
615
+
616
+ /**
617
+ * The evaluation's summed token accounting. `inputTokens` is the NON-cached
618
+ * input (the router's convention); the cache counts are the sums of what the
619
+ * calls reported, and stay absent when no call reported one — so a caller can
620
+ * tell "no cache reads" from "the provider does not say" (ggui#1281).
621
+ */
622
+ export interface LLMEvalTokenUsage {
623
+ inputTokens: number;
624
+ outputTokens: number;
625
+ cacheReadTokens?: number;
626
+ cacheCreationTokens?: number;
627
+ }
628
+
603
629
  /**
604
630
  * All valid categories for LLM evaluation — derived from the ONE static
605
631
  * list in types-public so the coverage stamp here and the harness's
@@ -642,7 +668,7 @@ async function evalCriterion(
642
668
  pass: true,
643
669
  coverage: 'skipped',
644
670
  skipReason: 'no tool call returned',
645
- tokens: { in: response.inputTokens, out: response.outputTokens },
671
+ tokens: callTokens(response),
646
672
  durationMs,
647
673
  };
648
674
  }
@@ -700,7 +726,7 @@ async function evalCriterion(
700
726
  issues,
701
727
  pass,
702
728
  coverage: 'ran',
703
- tokens: { in: response.inputTokens, out: response.outputTokens },
729
+ tokens: callTokens(response),
704
730
  durationMs,
705
731
  };
706
732
  } catch (e) {
@@ -833,7 +859,7 @@ export async function runLLMEvaluation(
833
859
  context: LLMEvalContext,
834
860
  config: LLMEvalConfig,
835
861
  preWarmedContext?: PreWarmedEvalContext | null,
836
- ): Promise<EvalResult & { inputTokens: number; outputTokens: number }> {
862
+ ): Promise<EvalResult & LLMEvalTokenUsage> {
837
863
  const providerName = config.provider;
838
864
  const model = config.model || getDefaultEvalModel(providerName);
839
865
  const routerProvider =
@@ -887,7 +913,10 @@ export async function runLLMEvaluation(
887
913
  const passCategories: string[] = [];
888
914
  let totalIn = 0;
889
915
  let totalOut = 0;
890
- let cachedTokensEstimate = 0;
916
+ // Measured, not estimated: summed from what the calls reported; absent
917
+ // while no call has reported one (ggui#1281).
918
+ let cacheRead: number | undefined;
919
+ let cacheCreation: number | undefined;
891
920
 
892
921
  for (const r of results) {
893
922
  allIssues.push(...r.issues);
@@ -896,12 +925,8 @@ export async function runLLMEvaluation(
896
925
  }
897
926
  totalIn += r.tokens.in;
898
927
  totalOut += r.tokens.out;
899
- }
900
-
901
- // Estimate cached tokens (mother prompt sent N times, cached N-1 times)
902
- const firstResult = results[0];
903
- if (firstResult && firstResult.tokens.in > 0) {
904
- cachedTokensEstimate = firstResult.tokens.in * (totalCriteriaCount - 1);
928
+ if (r.tokens.cacheRead !== undefined) cacheRead = (cacheRead ?? 0) + r.tokens.cacheRead;
929
+ if (r.tokens.cacheCreation !== undefined) cacheCreation = (cacheCreation ?? 0) + r.tokens.cacheCreation;
905
930
  }
906
931
 
907
932
  const wallMs = Date.now() - evalStart;
@@ -928,7 +953,7 @@ export async function runLLMEvaluation(
928
953
  }
929
954
  console.log(
930
955
  `[eval] total: ${(wallMs / 1000).toFixed(1)}s wall (${totalCriteriaCount} parallel) | ` +
931
- `in=${totalIn} out=${totalOut} cached~=${cachedTokensEstimate} | ` +
956
+ `in=${totalIn} out=${totalOut} cacheRead=${cacheRead ?? 'unreported'} cacheWrite=${cacheCreation ?? 'unreported'} | ` +
932
957
  `issues=${allIssues.length} pass=${passCategories.length}` +
933
958
  (skippedCount > 0 ? ` skipped=${skippedCount}` : ''),
934
959
  );
@@ -939,5 +964,7 @@ export async function runLLMEvaluation(
939
964
  criteriaCoverage,
940
965
  inputTokens: totalIn,
941
966
  outputTokens: totalOut,
967
+ ...(cacheRead !== undefined ? { cacheReadTokens: cacheRead } : {}),
968
+ ...(cacheCreation !== undefined ? { cacheCreationTokens: cacheCreation } : {}),
942
969
  };
943
970
  }
@@ -175,6 +175,14 @@ export function notApplicableCoverage(reason: string): CriterionCoverage[] {
175
175
  }));
176
176
  }
177
177
 
178
+ /** One contract-feedback round (ggui#1261): what bought it, and the source it was fed back on. */
179
+ export interface ContractFeedbackRecord {
180
+ /** The exit-probe issue subcategories that bought the round, e.g. `runtime:prop-sensitivity:currentUser`. */
181
+ readonly firedOn: readonly string[];
182
+ /** The component source as it stood when the model was told — the round's BEFORE. */
183
+ readonly sourceBefore: string;
184
+ }
185
+
178
186
  export interface EvalResult {
179
187
  issues: EvalIssue[];
180
188
  pass: string[];
@@ -194,6 +202,14 @@ export interface EvalResult {
194
202
  * probe runner; absent on eval paths that never invoke the probe.
195
203
  */
196
204
  runtimeProbe?: RuntimeProbeMeta;
205
+ /**
206
+ * The contract-feedback round, when one fired this generation (ggui#1261):
207
+ * which exit-probe findings bought it and the source as it stood when the
208
+ * model was told. `runtimeProbe` and the `runtime:*` issues on this result
209
+ * are the probe's verdict AFTER that round; `sourceBefore` beside the final
210
+ * source is the round's before/after. Absent when no such round fired.
211
+ */
212
+ contractFeedback?: ContractFeedbackRecord;
197
213
  /**
198
214
  * Per-canvas visual verdicts. Stamped by the harness ONLY when the
199
215
  * visual leg ran with `visualEvaluation.canvases` set (arm-neutral —
@@ -19,7 +19,7 @@ import { judgeDesignIdentity, type JudgeDesignIdentity } from './design-identity
19
19
  import { resolve, dirname } from 'path';
20
20
  import { fileURLToPath } from 'url';
21
21
  import { createRequire } from 'node:module';
22
- import { existsSync, writeFileSync, mkdirSync, unlinkSync } from 'fs';
22
+ import { existsSync, writeFileSync, mkdtempSync, rmSync } from 'fs';
23
23
  import { createHash } from 'node:crypto';
24
24
  import { tmpdir } from 'os';
25
25
  import { createVisionAgent, type AgentConfig } from '../harness/llm-router';
@@ -346,9 +346,12 @@ async function bundleForRendering(
346
346
  sampleProps: Record<string, unknown>,
347
347
  designSrc: string = resolve(resolveDesignPackageDir(), 'src'),
348
348
  ): Promise<string> {
349
- // Write component + entry to temp files for esbuild
350
- const tmpDir = resolve(tmpdir(), 'ggui-visual-eval-' + Date.now());
351
- mkdirSync(tmpDir, { recursive: true });
349
+ // Write component + entry to temp files for esbuild — in a directory of THIS
350
+ // call's own. `mkdtempSync` is atomic and unique: a timestamp-named directory
351
+ // was shared by two bundles started in the same millisecond, so the first
352
+ // build read the second's component (a judge scoring the wrong generation)
353
+ // and the first's cleanup deleted the files the second was building.
354
+ const tmpDir = mkdtempSync(resolve(tmpdir(), 'ggui-visual-eval-'));
352
355
  const componentFile = resolve(tmpDir, 'component.tsx');
353
356
  const entryFile = resolve(tmpDir, 'entry.tsx');
354
357
 
@@ -439,8 +442,8 @@ try {
439
442
 
440
443
  return result.outputFiles[0]?.text ?? '';
441
444
  } finally {
442
- try { unlinkSync(entryFile); } catch { /* cleanup */ }
443
- try { unlinkSync(componentFile); } catch { /* cleanup */ }
445
+ // The whole per-call directory; `force` makes an already-absent path a no-op.
446
+ rmSync(tmpDir, { recursive: true, force: true });
444
447
  }
445
448
  }
446
449