@ggui-ai/ui-gen 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/generation-dispatch.d.ts +1 -1
- package/dist/adapters/generation-dispatch.js +198 -59
- package/dist/adapters/generation-dispatch.js.map +1 -1
- package/dist/adapters/index.js +35 -10
- package/dist/adapters/index.js.map +1 -1
- package/dist/advanced/index.d.ts +2 -2
- package/dist/advanced/index.js +381 -58
- package/dist/advanced/index.js.map +1 -1
- package/dist/boilerplate.js.map +1 -1
- package/dist/check/index.d.ts +1 -1
- package/dist/check/index.js.map +1 -1
- package/dist/{create-ui-generator-D7kOXNjJ.d.ts → create-ui-generator-BEkzEtNz.d.ts} +1 -1
- package/dist/{dispatch-DGvhmARp.d.ts → dispatch-DJNpQ7DI.d.ts} +1 -1
- package/dist/evaluation/axis-checks/index.d.ts +3 -3
- package/dist/evaluation/axis-checks/index.js.map +1 -1
- package/dist/evaluation/axis-checks/registry.d.ts +1 -1
- package/dist/evaluation/index.d.ts +3 -3
- package/dist/evaluation/index.js +37 -20
- package/dist/evaluation/index.js.map +1 -1
- package/dist/harness/check/runtime-render/index.d.ts +1 -1
- package/dist/harness/index.d.ts +1 -1
- package/dist/harness/index.js +34 -9
- package/dist/harness/index.js.map +1 -1
- package/dist/harness/prompts.js +1 -1
- package/dist/harness/prompts.js.map +1 -1
- package/dist/harness/result-types.d.ts +15 -2
- package/dist/harness/result-types.js.map +1 -1
- package/dist/harness/runtime.d.ts +1 -1
- package/dist/harness/runtime.js.map +1 -1
- package/dist/harness/types-public.d.ts +1 -1
- package/dist/harness/types-public.js.map +1 -1
- package/dist/harness/validator-trace-sink.d.ts +1 -1
- package/dist/index.d.ts +22 -6
- package/dist/index.js +381 -58
- package/dist/index.js.map +1 -1
- package/dist/patch.d.ts +1 -1
- package/dist/{types-public-CzYqZclY.d.ts → types-public-mM_-1Yk3.d.ts} +16 -1
- package/dist/workflows.d.ts +1 -1
- package/package.json +12 -12
- package/src/adapters/claude/claude-code-login.ts +62 -7
- package/src/adapters/claude/sdk.ts +6 -5
- package/src/boilerplate/templates/layouts/chat-mobile.tsx.tmpl +7 -3
- package/src/boilerplate/templates/layouts/chat-universal.tsx.tmpl +7 -3
- package/src/contract-context.ts +1 -1
- package/src/create-ui-generator.ts +30 -0
- package/src/evaluation/index.ts +1 -0
- package/src/evaluation/llm-evaluator.ts +40 -13
- package/src/evaluation/types-public.ts +16 -0
- package/src/evaluation/visual-evaluator.ts +9 -6
- package/src/generator-build.ts +158 -0
- package/src/harness/claude-code-login-query.ts +5 -4
- package/src/harness/coding/assemble-result.ts +8 -1
- package/src/harness/coding/generate-task-runner.ts +75 -36
- package/src/harness/coding/run-coding-turn.ts +9 -0
- package/src/harness/coding/run-eval-round.ts +160 -5
- package/src/harness/result-types.ts +14 -0
- package/src/index.ts +2 -1
package/dist/patch.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { PatchFn } from './harness/types-public.js';
|
|
|
2
2
|
import '@ggui-ai/protocol';
|
|
3
3
|
import './styling-profile-CxfvV48g.js';
|
|
4
4
|
import './axes-CzLEMDeB.js';
|
|
5
|
-
import './types-public-
|
|
5
|
+
import './types-public-mM_-1Yk3.js';
|
|
6
6
|
import './design-mode-DLufhrNp.js';
|
|
7
7
|
import './types-BOvHNG7K.js';
|
|
8
8
|
import './llm.js';
|
|
@@ -120,6 +120,13 @@ declare const LLM_EVAL_STATIC_CRITERIA: ReadonlyArray<{
|
|
|
120
120
|
* exit runs before the evaluator module is lazily loaded.
|
|
121
121
|
*/
|
|
122
122
|
declare function notApplicableCoverage(reason: string): CriterionCoverage[];
|
|
123
|
+
/** One contract-feedback round (ggui#1261): what bought it, and the source it was fed back on. */
|
|
124
|
+
interface ContractFeedbackRecord {
|
|
125
|
+
/** The exit-probe issue subcategories that bought the round, e.g. `runtime:prop-sensitivity:currentUser`. */
|
|
126
|
+
readonly firedOn: readonly string[];
|
|
127
|
+
/** The component source as it stood when the model was told — the round's BEFORE. */
|
|
128
|
+
readonly sourceBefore: string;
|
|
129
|
+
}
|
|
123
130
|
interface EvalResult {
|
|
124
131
|
issues: EvalIssue[];
|
|
125
132
|
pass: string[];
|
|
@@ -139,6 +146,14 @@ interface EvalResult {
|
|
|
139
146
|
* probe runner; absent on eval paths that never invoke the probe.
|
|
140
147
|
*/
|
|
141
148
|
runtimeProbe?: RuntimeProbeMeta;
|
|
149
|
+
/**
|
|
150
|
+
* The contract-feedback round, when one fired this generation (ggui#1261):
|
|
151
|
+
* which exit-probe findings bought it and the source as it stood when the
|
|
152
|
+
* model was told. `runtimeProbe` and the `runtime:*` issues on this result
|
|
153
|
+
* are the probe's verdict AFTER that round; `sourceBefore` beside the final
|
|
154
|
+
* source is the round's before/after. Absent when no such round fired.
|
|
155
|
+
*/
|
|
156
|
+
contractFeedback?: ContractFeedbackRecord;
|
|
142
157
|
/**
|
|
143
158
|
* Per-canvas visual verdicts. Stamped by the harness ONLY when the
|
|
144
159
|
* visual leg ran with `visualEvaluation.canvases` set (arm-neutral —
|
|
@@ -344,4 +359,4 @@ declare function getLLMCriteria(designMode?: DesignMode): EvalCriterion[];
|
|
|
344
359
|
*/
|
|
345
360
|
declare function buildCodingCriteriaSummary(designMode?: DesignMode): string;
|
|
346
361
|
|
|
347
|
-
export { type AxisCheck as A, type CanvasJudgeRecord as C, DEFAULT_QUALITY_CONFIG as D, type EvalIssue as E, LLM_EVAL_STATIC_CRITERIA as L, type Priority as P, type QualityConfig as Q, type RuntimeProbeStatus as R, type VisualEvalSummary as V, type AxisCheckInput as a, type EvalResult as b, type EvalTier as c, type VisualCoverage as d, CRITERIA as e, type CanvasVisualSummary as f, type
|
|
362
|
+
export { type AxisCheck as A, type CanvasJudgeRecord as C, DEFAULT_QUALITY_CONFIG as D, type EvalIssue as E, LLM_EVAL_STATIC_CRITERIA as L, type Priority as P, type QualityConfig as Q, type RuntimeProbeStatus as R, type VisualEvalSummary as V, type AxisCheckInput as a, type EvalResult as b, type EvalTier as c, type VisualCoverage as d, CRITERIA as e, type CanvasVisualSummary as f, type ContractFeedbackRecord as g, type CriterionCoverage as h, type CriterionRunStatus as i, type EvalCategory as j, type EvalCriterion as k, type EvalOutcome as l, matches as m, type QualityMode as n, buildCodingCriteriaSummary as o, criteriaFor as p, getActionableIssues as q, getCriteriaByPriority as r, getCriterionById as s, getLLMCriteria as t, isBlocked as u, notApplicableCoverage as v, priorityForIssue as w };
|
package/dist/workflows.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { C as Classification } from './axes-CzLEMDeB.js';
|
|
|
2
2
|
import { Workflow } from './harness/types-public.js';
|
|
3
3
|
import '@ggui-ai/protocol';
|
|
4
4
|
import './styling-profile-CxfvV48g.js';
|
|
5
|
-
import './types-public-
|
|
5
|
+
import './types-public-mM_-1Yk3.js';
|
|
6
6
|
import './design-mode-DLufhrNp.js';
|
|
7
7
|
import './types-BOvHNG7K.js';
|
|
8
8
|
import './llm.js';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ggui-ai/ui-gen",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.22.0",
|
|
4
4
|
"description": "UI generation harness for ggui — the LLM system prompt, multi-turn coding loop, and evaluation engine that turn a DataContract into a React component. Implements the UiGenerator contract from @ggui-ai/mcp-server-core; consumed by @ggui-ai/mcp-server.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"keywords": [
|
|
@@ -189,9 +189,9 @@
|
|
|
189
189
|
"puppeteer-core": "^24.40.0 || ^25.0.0",
|
|
190
190
|
"react": "^18.0.0 || ^19.0.0",
|
|
191
191
|
"react-dom": "^18.0.0 || ^19.0.0",
|
|
192
|
-
"@ggui-ai/
|
|
193
|
-
"@ggui-ai/
|
|
194
|
-
"@ggui-ai/
|
|
192
|
+
"@ggui-ai/mcp-server-core": "0.22.0",
|
|
193
|
+
"@ggui-ai/project-config": "0.22.0",
|
|
194
|
+
"@ggui-ai/protocol": "0.22.0"
|
|
195
195
|
},
|
|
196
196
|
"peerDependenciesMeta": {
|
|
197
197
|
"@google/adk": {
|
|
@@ -233,11 +233,11 @@
|
|
|
233
233
|
"openai": "^7.9.0",
|
|
234
234
|
"typescript": "^5.0.0",
|
|
235
235
|
"zod": "^4.3.6",
|
|
236
|
-
"@ggui-ai/design": "0.
|
|
237
|
-
"@ggui-ai/
|
|
238
|
-
"@ggui-ai/
|
|
239
|
-
"@ggui-ai/ui-visual-tester": "0.
|
|
240
|
-
"@ggui-ai/wire": "0.
|
|
236
|
+
"@ggui-ai/design": "0.22.0",
|
|
237
|
+
"@ggui-ai/sandbox": "0.22.0",
|
|
238
|
+
"@ggui-ai/gadgets": "0.22.0",
|
|
239
|
+
"@ggui-ai/ui-visual-tester": "0.22.0",
|
|
240
|
+
"@ggui-ai/wire": "0.22.0"
|
|
241
241
|
},
|
|
242
242
|
"devDependencies": {
|
|
243
243
|
"@google/adk": "^2.0.0",
|
|
@@ -252,9 +252,9 @@
|
|
|
252
252
|
"react-dom": "^19.0.0",
|
|
253
253
|
"tsup": "^8.5.0",
|
|
254
254
|
"vitest": "^3.2.6",
|
|
255
|
-
"@ggui-ai/mcp-server-core": "0.
|
|
256
|
-
"@ggui-ai/project-config": "0.
|
|
257
|
-
"@ggui-ai/protocol": "0.
|
|
255
|
+
"@ggui-ai/mcp-server-core": "0.22.0",
|
|
256
|
+
"@ggui-ai/project-config": "0.22.0",
|
|
257
|
+
"@ggui-ai/protocol": "0.22.0"
|
|
258
258
|
},
|
|
259
259
|
"engines": {
|
|
260
260
|
"node": ">=22.0.0"
|
|
@@ -9,9 +9,12 @@
|
|
|
9
9
|
* 1. {@link CLAUDE_CODE_LOGIN_CREDENTIAL} — the sentinel a `ProviderKeyRef`
|
|
10
10
|
* carries in place of a key, the same shape `bedrock-iam` already uses
|
|
11
11
|
* for "authenticate with ambient credentials, not a key I hold".
|
|
12
|
-
* 2. {@link
|
|
13
|
-
*
|
|
14
|
-
*
|
|
12
|
+
* 2. {@link loginChildEnv} — the spawned process sees an ALLOWLIST of the
|
|
13
|
+
* parent's env, never the parent's env minus a few names: no provider
|
|
14
|
+
* key (the binary prefers an env key over its login, so a stale
|
|
15
|
+
* `ANTHROPIC_API_KEY` would silently take the run off the login path),
|
|
16
|
+
* no other secret the parent merely holds, and no switch that would
|
|
17
|
+
* change what the binary does (ggui#1278).
|
|
15
18
|
* 3. {@link claudeCodeLoginQueryOptions} — the SDK options that keep the
|
|
16
19
|
* run bounded: no built-in tool (`tools: []`), no `~/.claude` settings,
|
|
17
20
|
* hooks or MCP servers leaking in (`settingSources: []`). Not
|
|
@@ -40,14 +43,66 @@ export const PROVIDER_KEY_ENV_NAMES = [
|
|
|
40
43
|
"ANTHROPIC_BASE_URL",
|
|
41
44
|
] as const;
|
|
42
45
|
|
|
43
|
-
/**
|
|
44
|
-
|
|
46
|
+
/**
|
|
47
|
+
* The only env names the login child inherits (ggui#1278), plus every
|
|
48
|
+
* `LC_*` locale name. The spawned binary sees a property of this code, not
|
|
49
|
+
* whatever the parent happened to load: a parent that sourced a whole
|
|
50
|
+
* `.env` holds every provider's key, and a denylist would pass all of them
|
|
51
|
+
* on, together with switches such as `CLAUDE_CODE_USE_BEDROCK` that would
|
|
52
|
+
* silently re-route the run.
|
|
53
|
+
*
|
|
54
|
+
* - Process basics: `PATH` (the binary runs helpers, e.g. the macOS
|
|
55
|
+
* keychain tool), `HOME`/`USER`/`LOGNAME` (where and whose the login
|
|
56
|
+
* store is), `SHELL`, temp dirs, `TERM`, `TZ`, locale.
|
|
57
|
+
* - Where the login lives: `CLAUDE_CONFIG_DIR`, the XDG base dirs.
|
|
58
|
+
* - `CLAUDE_CODE_OAUTH_TOKEN`: the login ITSELF when a headless host
|
|
59
|
+
* supplies it by env (`claude setup-token`). It is the one credential
|
|
60
|
+
* this path may use; every provider key stays out
|
|
61
|
+
* ({@link PROVIDER_KEY_ENV_NAMES} is never on this list).
|
|
62
|
+
* - Network reach: proxy variables in both spellings, and the extra-CA
|
|
63
|
+
* and certificate paths a corporate network needs.
|
|
64
|
+
*/
|
|
65
|
+
export const LOGIN_CHILD_ENV_NAMES = [
|
|
66
|
+
"PATH",
|
|
67
|
+
"HOME",
|
|
68
|
+
"USER",
|
|
69
|
+
"LOGNAME",
|
|
70
|
+
"SHELL",
|
|
71
|
+
"TMPDIR",
|
|
72
|
+
"TMP",
|
|
73
|
+
"TEMP",
|
|
74
|
+
"TERM",
|
|
75
|
+
"TZ",
|
|
76
|
+
"LANG",
|
|
77
|
+
"CLAUDE_CONFIG_DIR",
|
|
78
|
+
"CLAUDE_CODE_OAUTH_TOKEN",
|
|
79
|
+
"XDG_CONFIG_HOME",
|
|
80
|
+
"XDG_DATA_HOME",
|
|
81
|
+
"XDG_CACHE_HOME",
|
|
82
|
+
"XDG_STATE_HOME",
|
|
83
|
+
"XDG_RUNTIME_DIR",
|
|
84
|
+
"HTTPS_PROXY",
|
|
85
|
+
"https_proxy",
|
|
86
|
+
"HTTP_PROXY",
|
|
87
|
+
"http_proxy",
|
|
88
|
+
"NO_PROXY",
|
|
89
|
+
"no_proxy",
|
|
90
|
+
"ALL_PROXY",
|
|
91
|
+
"all_proxy",
|
|
92
|
+
"NODE_EXTRA_CA_CERTS",
|
|
93
|
+
"SSL_CERT_FILE",
|
|
94
|
+
"SSL_CERT_DIR",
|
|
95
|
+
] as const;
|
|
96
|
+
|
|
97
|
+
const LOGIN_CHILD_ENV: ReadonlySet<string> = new Set(LOGIN_CHILD_ENV_NAMES);
|
|
98
|
+
|
|
99
|
+
/** The env the login child is spawned with: the allowlisted names present in `env`, nothing else. */
|
|
100
|
+
export function loginChildEnv(
|
|
45
101
|
env: Readonly<Record<string, string | undefined>>
|
|
46
102
|
): Record<string, string> {
|
|
47
103
|
const out: Record<string, string> = {};
|
|
48
|
-
const drop: ReadonlySet<string> = new Set(PROVIDER_KEY_ENV_NAMES);
|
|
49
104
|
for (const [k, v] of Object.entries(env)) {
|
|
50
|
-
if (v !== undefined &&
|
|
105
|
+
if (v !== undefined && (LOGIN_CHILD_ENV.has(k) || k.startsWith("LC_"))) out[k] = v;
|
|
51
106
|
}
|
|
52
107
|
return out;
|
|
53
108
|
}
|
|
@@ -13,7 +13,7 @@ import type { ClaudeSdkConfig, GenerateParams } from '../base';
|
|
|
13
13
|
import type { AdapterResult, ProviderName, AdapterMode } from '../types';
|
|
14
14
|
import { createArtifacts, processMessage } from './message-parser';
|
|
15
15
|
import { createToolMcpServer } from './tool-bridge';
|
|
16
|
-
import { claudeCodeLoginQueryOptions,
|
|
16
|
+
import { claudeCodeLoginQueryOptions, loginChildEnv } from './claude-code-login';
|
|
17
17
|
|
|
18
18
|
export class ClaudeSdkAdapter extends GeneratorAdapter {
|
|
19
19
|
readonly provider: ProviderName = 'claude';
|
|
@@ -41,11 +41,12 @@ export class ClaudeSdkAdapter extends GeneratorAdapter {
|
|
|
41
41
|
const { query } = await import('@anthropic-ai/claude-agent-sdk');
|
|
42
42
|
|
|
43
43
|
// Build environment from config or process.env
|
|
44
|
-
// ggui#1185 — on the login path
|
|
45
|
-
// it prefers an env key over its login, so a stale
|
|
46
|
-
// run off this path without a word
|
|
44
|
+
// ggui#1185/#1278 — on the login path the binary sees an allowlisted
|
|
45
|
+
// env: no provider key (it prefers an env key over its login, so a stale
|
|
46
|
+
// one would take the run off this path without a word), and nothing
|
|
47
|
+
// else the parent merely holds.
|
|
47
48
|
const login = cfg.claudeCodeLogin === true;
|
|
48
|
-
const env = login ?
|
|
49
|
+
const env = login ? loginChildEnv(buildEnv(cfg.env)) : buildEnv(cfg.env);
|
|
49
50
|
|
|
50
51
|
const artifacts = createArtifacts();
|
|
51
52
|
const onProgress = cfg.onProgress;
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
-
<
|
|
2
|
-
{/* implement your UI here
|
|
3
|
-
|
|
1
|
+
<Box padding="md">
|
|
2
|
+
{/* implement your UI here — an inline card: the chat bubble around it
|
|
3
|
+
provides the chrome, so add no shadow or outer radius of your own.
|
|
4
|
+
Compact and natural height; stack vertically. This root fills the
|
|
5
|
+
bubble — wrap PROSE in <Container maxWidth="sm"> to cap the reading
|
|
6
|
+
measure (inert at bubble width, composed on a wider frame). */}
|
|
7
|
+
</Box>
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
-
<
|
|
2
|
-
{/* implement your UI here
|
|
3
|
-
|
|
1
|
+
<Box padding="md">
|
|
2
|
+
{/* implement your UI here — an inline card: the chat bubble around it
|
|
3
|
+
provides the chrome, so add no shadow or outer radius of your own.
|
|
4
|
+
Compact and natural height; stack vertically. This root fills the
|
|
5
|
+
bubble — wrap PROSE in <Container maxWidth="sm"> to cap the reading
|
|
6
|
+
measure (inert at bubble width, composed on a wider frame). */}
|
|
7
|
+
</Box>
|
package/src/contract-context.ts
CHANGED
|
@@ -59,7 +59,7 @@ const SHELL_HINTS: Record<string, string> = {
|
|
|
59
59
|
- Padding: use var(--ggui-spacing-2) for inner elements, var(--ggui-spacing-4) for sections. Tight.
|
|
60
60
|
- Scrolling: parent scrolls (chat feed) — component should NOT have internal scroll
|
|
61
61
|
- Layout: single column, compact. No sidebar. Stack everything vertically.
|
|
62
|
-
- Width: width: 100% (fills
|
|
62
|
+
- Width: the root is width: 100% (fills the bubble) — never cap the ROOT. A prose column inside it carries the reading cap (<Container maxWidth="sm">), which is inert at bubble width.`,
|
|
63
63
|
|
|
64
64
|
fullscreen: `**Fullscreen Shell — Takes over entire viewport**
|
|
65
65
|
- Container: the whole viewport, edge to edge — the frame stretches your root to its full height (you never set a viewport height yourself)
|
|
@@ -56,6 +56,7 @@ import {
|
|
|
56
56
|
} from '@ggui-ai/mcp-server-core';
|
|
57
57
|
import { createGeneratorTools } from './adapters/index.js';
|
|
58
58
|
import { dispatchGeneration } from './adapters/generation-dispatch.js';
|
|
59
|
+
import { generatorBuild } from './generator-build.js';
|
|
59
60
|
import { hasProfile } from './boilerplate/styling-profile.js';
|
|
60
61
|
import { effortDials } from './effort.js';
|
|
61
62
|
import type { ProviderName } from './adapters/types.js';
|
|
@@ -68,6 +69,7 @@ import {
|
|
|
68
69
|
} from './contract-context.js';
|
|
69
70
|
import type { RenderingContext } from './contract-context.js';
|
|
70
71
|
import type { DesignMode } from './design-mode.js';
|
|
72
|
+
import { DEFAULT_DESIGN_MODE } from './design-mode.js';
|
|
71
73
|
import { resolveRoute, applyRouteToEnv } from './adapters/provider-router.js';
|
|
72
74
|
import type { QualityConfig } from './evaluation/types-public.js';
|
|
73
75
|
import type { EvaluationConfig } from './evaluation/types.js';
|
|
@@ -339,6 +341,7 @@ export function createUiGenerator(
|
|
|
339
341
|
// Ignored by the constrained prompt (its shell/screen
|
|
340
342
|
// descriptors stay byte-identical).
|
|
341
343
|
const canvas = rendering ? canvasForRenderingContext(rendering) : undefined;
|
|
344
|
+
const shellType = harnessShellForRendering(rendering);
|
|
342
345
|
const promptWithVariance = injectVariance(
|
|
343
346
|
promptWithRendering,
|
|
344
347
|
input.variance,
|
|
@@ -405,6 +408,8 @@ export function createUiGenerator(
|
|
|
405
408
|
...(onRetry !== undefined ? { onRetry } : {}),
|
|
406
409
|
...(designMode !== undefined ? { designMode } : {}),
|
|
407
410
|
...(canvas !== undefined ? { canvas } : {}),
|
|
411
|
+
// ggui#1279 — the shell the declared rendering composes for (chat only; see harnessShellForRendering).
|
|
412
|
+
...(shellType !== undefined ? { shellType } : {}),
|
|
408
413
|
...(input.profile !== undefined ? { profile: input.profile } : {}),
|
|
409
414
|
// Fixture props reach the in-loop probe + visual round only (never the prompt, never identity).
|
|
410
415
|
...(input.fixtureProps !== undefined ? { fixtureProps: input.fixtureProps } : {}),
|
|
@@ -421,6 +426,8 @@ export function createUiGenerator(
|
|
|
421
426
|
latencyMs: Date.now() - startedAt,
|
|
422
427
|
cacheHit: false,
|
|
423
428
|
attempts: result.turnsUsed,
|
|
429
|
+
// ggui#1280 — the build that made this generation (template keys, never per-request).
|
|
430
|
+
build: generatorBuild(designMode ?? DEFAULT_DESIGN_MODE),
|
|
424
431
|
// Prompt-cache counters are provider-specific (Claude reports
|
|
425
432
|
// them; others omit). Pass through truthfully — absent on the
|
|
426
433
|
// adapter result stays absent here, never defaulted to 0.
|
|
@@ -461,6 +468,8 @@ export function createUiGenerator(
|
|
|
461
468
|
outputTokens: 0,
|
|
462
469
|
latencyMs: Date.now() - startedAt,
|
|
463
470
|
cacheHit: false,
|
|
471
|
+
// ggui#1280 — a failed generation still names the build that failed.
|
|
472
|
+
build: generatorBuild(designMode ?? DEFAULT_DESIGN_MODE),
|
|
464
473
|
},
|
|
465
474
|
};
|
|
466
475
|
} finally {
|
|
@@ -523,6 +532,27 @@ function mapLlmProviderToDispatchProvider(provider: LlmProvider): ProviderName {
|
|
|
523
532
|
}
|
|
524
533
|
}
|
|
525
534
|
|
|
535
|
+
/**
|
|
536
|
+
* ggui#1279 — the harness shell a declared rendering composes for. The canvas
|
|
537
|
+
* (`canvasForRenderingContext`) already follows the rendering; the WHAT leg's
|
|
538
|
+
* layout and the HOW leg's shell descriptor did not, so a declared chat card
|
|
539
|
+
* was scaffolded to "fill the frame" while the user prompt and the fit verdict
|
|
540
|
+
* treated it as an inline card. Only a declared CHAT shell changes anything.
|
|
541
|
+
* `fullscreen` IS the harness default, and `partial` has no layout of its own
|
|
542
|
+
* yet, so both pass no shell and those generations stay byte-identical — named
|
|
543
|
+
* here rather than guessed.
|
|
544
|
+
*/
|
|
545
|
+
function harnessShellForRendering(rendering: RenderingContext | undefined): 'chat' | undefined {
|
|
546
|
+
if (rendering === undefined) return undefined;
|
|
547
|
+
switch (rendering.shell) {
|
|
548
|
+
case 'chat':
|
|
549
|
+
return 'chat';
|
|
550
|
+
case 'fullscreen':
|
|
551
|
+
case 'partial':
|
|
552
|
+
return undefined;
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
|
|
526
556
|
function mapRendering(
|
|
527
557
|
rendering: UiGenerateInput['rendering'],
|
|
528
558
|
): RenderingContext | undefined {
|
package/src/evaluation/index.ts
CHANGED
|
@@ -25,7 +25,7 @@ import type {
|
|
|
25
25
|
} from './types-public.js';
|
|
26
26
|
import { LLM_EVAL_STATIC_CRITERIA } from './types-public.js';
|
|
27
27
|
import { createAgent } from '../harness/llm-router';
|
|
28
|
-
import type { AgentConfig, LLMToolDef, LLMAgent } from '../harness/llm-router';
|
|
28
|
+
import type { AgentConfig, LLMToolDef, LLMAgent, LLMToolCallResponse } from '../harness/llm-router';
|
|
29
29
|
import type { JsonObject } from '@ggui-ai/protocol';
|
|
30
30
|
import { getCriterionById } from './types-public.js';
|
|
31
31
|
import { DEFAULT_DESIGN_MODE, type DesignMode } from '../design-mode.js';
|
|
@@ -596,10 +596,36 @@ interface CriterionResult {
|
|
|
596
596
|
coverage: CriterionRunStatus;
|
|
597
597
|
/** Populated for `skipped` — why the criterion produced no verdict. */
|
|
598
598
|
skipReason?: string;
|
|
599
|
-
|
|
599
|
+
/** This call's token accounting. The cache counts are present only when the
|
|
600
|
+
* provider reported them — absent means unreported, never zero (ggui#1281). */
|
|
601
|
+
tokens: { in: number; out: number; cacheRead?: number; cacheCreation?: number };
|
|
600
602
|
durationMs: number;
|
|
601
603
|
}
|
|
602
604
|
|
|
605
|
+
/** A call's token accounting, carrying the prompt-cache counts only when the
|
|
606
|
+
* provider reported them (the router's rule: absent is "unreported"). */
|
|
607
|
+
function callTokens(response: LLMToolCallResponse): CriterionResult['tokens'] {
|
|
608
|
+
return {
|
|
609
|
+
in: response.inputTokens,
|
|
610
|
+
out: response.outputTokens,
|
|
611
|
+
...(response.cacheReadTokens !== undefined ? { cacheRead: response.cacheReadTokens } : {}),
|
|
612
|
+
...(response.cacheCreationTokens !== undefined ? { cacheCreation: response.cacheCreationTokens } : {}),
|
|
613
|
+
};
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
/**
|
|
617
|
+
* The evaluation's summed token accounting. `inputTokens` is the NON-cached
|
|
618
|
+
* input (the router's convention); the cache counts are the sums of what the
|
|
619
|
+
* calls reported, and stay absent when no call reported one — so a caller can
|
|
620
|
+
* tell "no cache reads" from "the provider does not say" (ggui#1281).
|
|
621
|
+
*/
|
|
622
|
+
export interface LLMEvalTokenUsage {
|
|
623
|
+
inputTokens: number;
|
|
624
|
+
outputTokens: number;
|
|
625
|
+
cacheReadTokens?: number;
|
|
626
|
+
cacheCreationTokens?: number;
|
|
627
|
+
}
|
|
628
|
+
|
|
603
629
|
/**
|
|
604
630
|
* All valid categories for LLM evaluation — derived from the ONE static
|
|
605
631
|
* list in types-public so the coverage stamp here and the harness's
|
|
@@ -642,7 +668,7 @@ async function evalCriterion(
|
|
|
642
668
|
pass: true,
|
|
643
669
|
coverage: 'skipped',
|
|
644
670
|
skipReason: 'no tool call returned',
|
|
645
|
-
tokens:
|
|
671
|
+
tokens: callTokens(response),
|
|
646
672
|
durationMs,
|
|
647
673
|
};
|
|
648
674
|
}
|
|
@@ -700,7 +726,7 @@ async function evalCriterion(
|
|
|
700
726
|
issues,
|
|
701
727
|
pass,
|
|
702
728
|
coverage: 'ran',
|
|
703
|
-
tokens:
|
|
729
|
+
tokens: callTokens(response),
|
|
704
730
|
durationMs,
|
|
705
731
|
};
|
|
706
732
|
} catch (e) {
|
|
@@ -833,7 +859,7 @@ export async function runLLMEvaluation(
|
|
|
833
859
|
context: LLMEvalContext,
|
|
834
860
|
config: LLMEvalConfig,
|
|
835
861
|
preWarmedContext?: PreWarmedEvalContext | null,
|
|
836
|
-
): Promise<EvalResult &
|
|
862
|
+
): Promise<EvalResult & LLMEvalTokenUsage> {
|
|
837
863
|
const providerName = config.provider;
|
|
838
864
|
const model = config.model || getDefaultEvalModel(providerName);
|
|
839
865
|
const routerProvider =
|
|
@@ -887,7 +913,10 @@ export async function runLLMEvaluation(
|
|
|
887
913
|
const passCategories: string[] = [];
|
|
888
914
|
let totalIn = 0;
|
|
889
915
|
let totalOut = 0;
|
|
890
|
-
|
|
916
|
+
// Measured, not estimated: summed from what the calls reported; absent
|
|
917
|
+
// while no call has reported one (ggui#1281).
|
|
918
|
+
let cacheRead: number | undefined;
|
|
919
|
+
let cacheCreation: number | undefined;
|
|
891
920
|
|
|
892
921
|
for (const r of results) {
|
|
893
922
|
allIssues.push(...r.issues);
|
|
@@ -896,12 +925,8 @@ export async function runLLMEvaluation(
|
|
|
896
925
|
}
|
|
897
926
|
totalIn += r.tokens.in;
|
|
898
927
|
totalOut += r.tokens.out;
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
// Estimate cached tokens (mother prompt sent N times, cached N-1 times)
|
|
902
|
-
const firstResult = results[0];
|
|
903
|
-
if (firstResult && firstResult.tokens.in > 0) {
|
|
904
|
-
cachedTokensEstimate = firstResult.tokens.in * (totalCriteriaCount - 1);
|
|
928
|
+
if (r.tokens.cacheRead !== undefined) cacheRead = (cacheRead ?? 0) + r.tokens.cacheRead;
|
|
929
|
+
if (r.tokens.cacheCreation !== undefined) cacheCreation = (cacheCreation ?? 0) + r.tokens.cacheCreation;
|
|
905
930
|
}
|
|
906
931
|
|
|
907
932
|
const wallMs = Date.now() - evalStart;
|
|
@@ -928,7 +953,7 @@ export async function runLLMEvaluation(
|
|
|
928
953
|
}
|
|
929
954
|
console.log(
|
|
930
955
|
`[eval] total: ${(wallMs / 1000).toFixed(1)}s wall (${totalCriteriaCount} parallel) | ` +
|
|
931
|
-
`in=${totalIn} out=${totalOut}
|
|
956
|
+
`in=${totalIn} out=${totalOut} cacheRead=${cacheRead ?? 'unreported'} cacheWrite=${cacheCreation ?? 'unreported'} | ` +
|
|
932
957
|
`issues=${allIssues.length} pass=${passCategories.length}` +
|
|
933
958
|
(skippedCount > 0 ? ` skipped=${skippedCount}` : ''),
|
|
934
959
|
);
|
|
@@ -939,5 +964,7 @@ export async function runLLMEvaluation(
|
|
|
939
964
|
criteriaCoverage,
|
|
940
965
|
inputTokens: totalIn,
|
|
941
966
|
outputTokens: totalOut,
|
|
967
|
+
...(cacheRead !== undefined ? { cacheReadTokens: cacheRead } : {}),
|
|
968
|
+
...(cacheCreation !== undefined ? { cacheCreationTokens: cacheCreation } : {}),
|
|
942
969
|
};
|
|
943
970
|
}
|
|
@@ -175,6 +175,14 @@ export function notApplicableCoverage(reason: string): CriterionCoverage[] {
|
|
|
175
175
|
}));
|
|
176
176
|
}
|
|
177
177
|
|
|
178
|
+
/** One contract-feedback round (ggui#1261): what bought it, and the source it was fed back on. */
|
|
179
|
+
export interface ContractFeedbackRecord {
|
|
180
|
+
/** The exit-probe issue subcategories that bought the round, e.g. `runtime:prop-sensitivity:currentUser`. */
|
|
181
|
+
readonly firedOn: readonly string[];
|
|
182
|
+
/** The component source as it stood when the model was told — the round's BEFORE. */
|
|
183
|
+
readonly sourceBefore: string;
|
|
184
|
+
}
|
|
185
|
+
|
|
178
186
|
export interface EvalResult {
|
|
179
187
|
issues: EvalIssue[];
|
|
180
188
|
pass: string[];
|
|
@@ -194,6 +202,14 @@ export interface EvalResult {
|
|
|
194
202
|
* probe runner; absent on eval paths that never invoke the probe.
|
|
195
203
|
*/
|
|
196
204
|
runtimeProbe?: RuntimeProbeMeta;
|
|
205
|
+
/**
|
|
206
|
+
* The contract-feedback round, when one fired this generation (ggui#1261):
|
|
207
|
+
* which exit-probe findings bought it and the source as it stood when the
|
|
208
|
+
* model was told. `runtimeProbe` and the `runtime:*` issues on this result
|
|
209
|
+
* are the probe's verdict AFTER that round; `sourceBefore` beside the final
|
|
210
|
+
* source is the round's before/after. Absent when no such round fired.
|
|
211
|
+
*/
|
|
212
|
+
contractFeedback?: ContractFeedbackRecord;
|
|
197
213
|
/**
|
|
198
214
|
* Per-canvas visual verdicts. Stamped by the harness ONLY when the
|
|
199
215
|
* visual leg ran with `visualEvaluation.canvases` set (arm-neutral —
|
|
@@ -19,7 +19,7 @@ import { judgeDesignIdentity, type JudgeDesignIdentity } from './design-identity
|
|
|
19
19
|
import { resolve, dirname } from 'path';
|
|
20
20
|
import { fileURLToPath } from 'url';
|
|
21
21
|
import { createRequire } from 'node:module';
|
|
22
|
-
import { existsSync, writeFileSync,
|
|
22
|
+
import { existsSync, writeFileSync, mkdtempSync, rmSync } from 'fs';
|
|
23
23
|
import { createHash } from 'node:crypto';
|
|
24
24
|
import { tmpdir } from 'os';
|
|
25
25
|
import { createVisionAgent, type AgentConfig } from '../harness/llm-router';
|
|
@@ -346,9 +346,12 @@ async function bundleForRendering(
|
|
|
346
346
|
sampleProps: Record<string, unknown>,
|
|
347
347
|
designSrc: string = resolve(resolveDesignPackageDir(), 'src'),
|
|
348
348
|
): Promise<string> {
|
|
349
|
-
// Write component + entry to temp files for esbuild
|
|
350
|
-
|
|
351
|
-
|
|
349
|
+
// Write component + entry to temp files for esbuild — in a directory of THIS
|
|
350
|
+
// call's own. `mkdtempSync` is atomic and unique: a timestamp-named directory
|
|
351
|
+
// was shared by two bundles started in the same millisecond, so the first
|
|
352
|
+
// build read the second's component (a judge scoring the wrong generation)
|
|
353
|
+
// and the first's cleanup deleted the files the second was building.
|
|
354
|
+
const tmpDir = mkdtempSync(resolve(tmpdir(), 'ggui-visual-eval-'));
|
|
352
355
|
const componentFile = resolve(tmpDir, 'component.tsx');
|
|
353
356
|
const entryFile = resolve(tmpDir, 'entry.tsx');
|
|
354
357
|
|
|
@@ -439,8 +442,8 @@ try {
|
|
|
439
442
|
|
|
440
443
|
return result.outputFiles[0]?.text ?? '';
|
|
441
444
|
} finally {
|
|
442
|
-
|
|
443
|
-
|
|
445
|
+
// The whole per-call directory; `force` makes an already-absent path a no-op.
|
|
446
|
+
rmSync(tmpDir, { recursive: true, force: true });
|
|
444
447
|
}
|
|
445
448
|
}
|
|
446
449
|
|