@ggui-ai/ui-gen 0.24.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{adapter-Du7P_5Fx.d.ts → adapter-BJwn74rx.d.ts} +49 -1
- package/dist/adapters/generation-dispatch.d.ts +32 -7
- package/dist/adapters/generation-dispatch.js +1997 -216
- package/dist/adapters/generation-dispatch.js.map +1 -1
- package/dist/adapters/index.js +29 -6
- package/dist/adapters/index.js.map +1 -1
- package/dist/advanced/index.d.ts +15 -12
- package/dist/advanced/index.js +2013 -270
- package/dist/advanced/index.js.map +1 -1
- package/dist/{axes-CzLEMDeB.d.ts → axes-CKVCzR_x.d.ts} +1 -1
- package/dist/bank-Ctdp3FUA.d.ts +118 -0
- package/dist/blueprint-validator.js +416 -92
- package/dist/blueprint-validator.js.map +1 -1
- package/dist/boilerplate.d.ts +2 -2
- package/dist/boilerplate.js +34 -15
- package/dist/boilerplate.js.map +1 -1
- package/dist/check/index.d.ts +13 -5
- package/dist/check/index.js +56 -3
- package/dist/check/index.js.map +1 -1
- package/dist/classifier/index.d.ts +2 -2
- package/dist/compose.d.ts +2 -2
- package/dist/{create-ui-generator-GIhzx08x.d.ts → create-ui-generator-N4gRXpcL.d.ts} +11 -4
- package/dist/{design-mode-DLufhrNp.d.ts → design-mode-DGWN_Gkm.d.ts} +1 -1
- package/dist/{dispatch-BnAH56p4.d.ts → dispatch-BUoXRj_r.d.ts} +6 -4
- package/dist/evaluation/axis-checks/index.d.ts +7 -6
- package/dist/evaluation/axis-checks/index.js +90 -2
- package/dist/evaluation/axis-checks/index.js.map +1 -1
- package/dist/evaluation/axis-checks/registry.d.ts +6 -5
- package/dist/evaluation/axis-checks/registry.js +88 -1
- package/dist/evaluation/axis-checks/registry.js.map +1 -1
- package/dist/evaluation/index.d.ts +189 -344
- package/dist/evaluation/index.js +1272 -59
- package/dist/evaluation/index.js.map +1 -1
- package/dist/evaluation/types.d.ts +26 -1
- package/dist/evaluation/types.js.map +1 -1
- package/dist/fragments/index.d.ts +3 -3
- package/dist/{generate-9IeNT4ZY.d.ts → generate-tSO_shQ0.d.ts} +1 -1
- package/dist/harness/check/runtime-render/index.d.ts +7 -7
- package/dist/harness/check/runtime-render/index.js +416 -92
- package/dist/harness/check/runtime-render/index.js.map +1 -1
- package/dist/harness/check/runtime-render/render-check-worker.js +393 -88
- package/dist/harness/check/runtime-render/render-check-worker.js.map +1 -1
- package/dist/harness/index.d.ts +5 -6
- package/dist/harness/index.js +69 -20
- package/dist/harness/index.js.map +1 -1
- package/dist/harness/prompts.d.ts +2 -2
- package/dist/harness/prompts.js +10 -1
- package/dist/harness/prompts.js.map +1 -1
- package/dist/harness/result-types.d.ts +19 -5
- package/dist/harness/result-types.js.map +1 -1
- package/dist/harness/runtime.d.ts +21 -7
- package/dist/harness/runtime.js +36 -17
- package/dist/harness/runtime.js.map +1 -1
- package/dist/harness/types-public.d.ts +6 -4
- package/dist/harness/types-public.js +43 -18
- package/dist/harness/types-public.js.map +1 -1
- package/dist/harness/validator-trace-sink.d.ts +6 -5
- package/dist/hash.d.ts +1 -1
- package/dist/index.d.ts +64 -13
- package/dist/index.js +2023 -270
- package/dist/index.js.map +1 -1
- package/dist/patch.d.ts +5 -5
- package/dist/policy.d.ts +1 -1
- package/dist/{prompts-YU9piOgs.d.ts → prompts-BhNUuRoZ.d.ts} +1 -1
- package/dist/tools/get-primitives-ts.d.ts +1 -1
- package/dist/tools/get-primitives-ts.js +1 -1
- package/dist/tools/get-primitives-ts.js.map +1 -1
- package/dist/tools/render-check-worker.js +8 -1
- package/dist/tools/render-check-worker.js.map +1 -1
- package/dist/{types-BOvHNG7K.d.ts → types-Dh1s4K9j.d.ts} +1 -1
- package/dist/types-public-B7wn4ce5.d.ts +959 -0
- package/dist/validation/index.d.ts +1 -1
- package/dist/validation/index.js +1 -1
- package/dist/validation/index.js.map +1 -1
- package/dist/visual-evaluator-C9L-o6ap.d.ts +639 -0
- package/dist/workflows.d.ts +5 -5
- package/package.json +12 -12
- package/src/adapters/generation-dispatch.ts +38 -2
- package/src/adapters/tools.ts +2 -1
- package/src/boilerplate/free-design-prompt.ts +4 -4
- package/src/boilerplate/generate.ts +26 -1
- package/src/boilerplate/hard-sections.ts +14 -14
- package/src/boilerplate/system-prompt.ts +1 -1
- package/src/boilerplate/templates/layouts/chat-mobile.tsx.tmpl +3 -1
- package/src/boilerplate/templates/layouts/chat-universal.tsx.tmpl +3 -1
- package/src/boilerplate/templates/layouts/fullscreen-desktop.tsx.tmpl +3 -1
- package/src/boilerplate/templates/layouts/fullscreen-universal.tsx.tmpl +3 -1
- package/src/check/index.ts +1 -0
- package/src/check/run-tier0.ts +6 -3
- package/src/check/sample-props.ts +63 -0
- package/src/coding-agent/tools.ts +4 -1
- package/src/contract-context.ts +1 -1
- package/src/create-harness.ts +3 -3
- package/src/create-ui-generator.ts +20 -1
- package/src/evaluation/__fixtures__/png.ts +100 -0
- package/src/evaluation/axis-checks/checks/state-merge.ts +1 -1
- package/src/evaluation/axis-checks/checks/universal.ts +111 -0
- package/src/evaluation/axis-checks/dispatch.ts +4 -1
- package/src/evaluation/cost-tracker.ts +54 -6
- package/src/evaluation/criteria/bank.ts +85 -0
- package/src/evaluation/criteria/context.ts +52 -0
- package/src/evaluation/criteria/edge-probe.ts +286 -0
- package/src/evaluation/criteria/edge.ts +202 -0
- package/src/evaluation/criteria/resolve.ts +207 -0
- package/src/evaluation/criteria/select.ts +79 -0
- package/src/evaluation/index.ts +27 -0
- package/src/evaluation/ink-extent.ts +200 -0
- package/src/evaluation/judge-spend.ts +48 -0
- package/src/evaluation/theme-css.ts +12 -5
- package/src/evaluation/types-public.ts +221 -3
- package/src/evaluation/types.ts +19 -0
- package/src/evaluation/visual-evaluator.ts +853 -51
- package/src/harness/check/runtime-render/adapter.ts +18 -0
- package/src/harness/check/runtime-render/probe-pending.ts +111 -0
- package/src/harness/check/runtime-render/render-check-host.ts +10 -6
- package/src/harness/check/runtime-render/render-check-worker.ts +6 -3
- package/src/harness/check/runtime-render/render-check.ts +417 -133
- package/src/harness/check/runtime-render/walk.ts +200 -0
- package/src/harness/coding/assemble-result.ts +2 -0
- package/src/harness/coding/generate-task-runner.ts +30 -0
- package/src/harness/coding/init-session.ts +5 -3
- package/src/harness/coding/run-coding-turn.ts +12 -4
- package/src/harness/coding/run-eval-round.ts +82 -3
- package/src/harness/in-loop-evaluation.ts +39 -0
- package/src/harness/llm-router.ts +58 -1
- package/src/harness/result-types.ts +9 -0
- package/src/harness/runtime.ts +14 -0
- package/src/harness/types-public.ts +10 -1
- package/src/harness-tool-names.ts +38 -0
- package/src/index.ts +17 -0
- package/src/internal/verdict-line.ts +33 -0
- package/src/run-check.ts +2 -0
- package/src/taught-tokens.ts +32 -0
- package/src/tools/get-primitives-ts.ts +1 -1
- package/src/tools/get-wire.ts +1 -1
- package/src/tools/render-check-worker.ts +2 -1
- package/src/tools/render-check.ts +8 -67
- package/src/tools.ts +11 -1
- package/src/validation/primitives.ts +1 -1
- package/dist/llm-router-DwwUK1bz.d.ts +0 -265
- package/dist/types-public-Bk6d1WLN.d.ts +0 -451
|
@@ -0,0 +1,959 @@
|
|
|
1
|
+
import { JsonObject, DataContract } from '@ggui-ai/protocol';
|
|
2
|
+
import { LLMToolDef } from './llm.js';
|
|
3
|
+
import { GenerationRuntimeProbeStatus, GenerationRuntimeProbeOutcome } from '@ggui-ai/mcp-server-core';
|
|
4
|
+
import { C as CanvasClass, D as DesignMode } from './design-mode-DGWN_Gkm.js';
|
|
5
|
+
import { R as RiskTier, a as RenderShape, S as StateShape, W as WriteShape, F as FetchShape, L as LayoutShape, C as Classification, A as AxisVector } from './axes-CKVCzR_x.js';
|
|
6
|
+
import { A as AxisKey } from './types-Dh1s4K9j.js';
|
|
7
|
+
|
|
8
|
+
interface AgentConfig {
|
|
9
|
+
provider: 'anthropic' | 'openai' | 'google' | 'openrouter';
|
|
10
|
+
model: string;
|
|
11
|
+
/**
|
|
12
|
+
* Sampling temperature. When defined it is threaded into the provider
|
|
13
|
+
* request; when undefined the provider default is used (unchanged
|
|
14
|
+
* behavior for all existing callers).
|
|
15
|
+
*
|
|
16
|
+
* Capability is the ROUTER's: Anthropic Opus 4.7+ and the 5-family
|
|
17
|
+
* (`claude-opus-4-7*`, `claude-fable-5*`, `claude-opus-5*`,
|
|
18
|
+
* `claude-sonnet-5*`, incl. Bedrock ids) return HTTP 400 on a
|
|
19
|
+
* non-default `temperature`/`top_p`/`top_k`, so the Anthropic adapter
|
|
20
|
+
* STRIPS the value for those ids (one log line naming model + param)
|
|
21
|
+
* and reports the effective sampling on `LLMResponse.sampling`. Callers
|
|
22
|
+
* keep expressing intent; disclosure reads the applied value.
|
|
23
|
+
*/
|
|
24
|
+
temperature?: number;
|
|
25
|
+
/**
|
|
26
|
+
* Explicit per-call credentials/routing, bypassing `process.env`
|
|
27
|
+
* entirely when supplied. Absent (default) preserves today's
|
|
28
|
+
* behavior — each agent falls back to reading its provider's env
|
|
29
|
+
* var(s) directly, so existing callers (including self-hosters who
|
|
30
|
+
* set env vars once at process start) are unaffected. Use this to
|
|
31
|
+
* run concurrent generations safely in one process: two simultaneous
|
|
32
|
+
* calls with different `routeOverride`s never race on shared
|
|
33
|
+
* `process.env` state.
|
|
34
|
+
*/
|
|
35
|
+
routeOverride?: {
|
|
36
|
+
readonly apiKey?: string;
|
|
37
|
+
/** Anthropic-only: force the Bedrock-IAM client instead of the direct API client. */
|
|
38
|
+
readonly useBedrock?: boolean;
|
|
39
|
+
/**
|
|
40
|
+
* Anthropic-only (ggui#1185): run on the machine's own Claude Code
|
|
41
|
+
* LOGIN instead of a key — the route decision `auth: 'claude-code-login'`
|
|
42
|
+
* lands here, and `createAgent` / `createVisionAgent` select
|
|
43
|
+
* `ClaudeCodeLoginAgent` (one `query()` per call) instead of the raw
|
|
44
|
+
* API client. Every agent the harness builds — coding, evaluation,
|
|
45
|
+
* visual judge — takes the same decision. Off unless set; a real key
|
|
46
|
+
* never coexists with it (the router refuses a non-anthropic route).
|
|
47
|
+
*/
|
|
48
|
+
readonly claudeCodeLogin?: boolean;
|
|
49
|
+
};
|
|
50
|
+
/**
|
|
51
|
+
* Observer invoked once per retried attempt when `apiCall()` (#489)
|
|
52
|
+
* retries a rate-limited (HTTP 429) provider call. Fires AFTER the
|
|
53
|
+
* retry decision is made and BEFORE the delay is awaited — never on
|
|
54
|
+
* the terminal failure/success of the whole call. Purely an
|
|
55
|
+
* observability hook: it never influences whether or how long
|
|
56
|
+
* `apiCall()` retries. If the observer itself throws, `apiCall()`
|
|
57
|
+
* catches it, logs a `console.warn`, and proceeds to the retry
|
|
58
|
+
* delay unaffected — a broken observer cannot turn a recoverable
|
|
59
|
+
* 429 into an unrelated hard failure. Absent (default) is a no-op;
|
|
60
|
+
* `apiCall()` still logs every retry via `console.warn` regardless.
|
|
61
|
+
*/
|
|
62
|
+
onRetry?: (info: ProviderRetryInfo) => void;
|
|
63
|
+
}
|
|
64
|
+
/** One retried attempt's context — passed to `AgentConfig.onRetry`. */
|
|
65
|
+
interface ProviderRetryInfo {
|
|
66
|
+
readonly provider: AgentConfig['provider'];
|
|
67
|
+
/** 1-based index of this retry (1 = first retry, 2 = second, …). */
|
|
68
|
+
readonly attempt: number;
|
|
69
|
+
/** Total retries `apiCall()` will attempt before giving up. */
|
|
70
|
+
readonly maxAttempts: number;
|
|
71
|
+
/** HTTP status of the failed call, when the caught error carries one. */
|
|
72
|
+
readonly status: number | undefined;
|
|
73
|
+
/** Provider-supplied `Retry-After` value in seconds, when present and within the cap. */
|
|
74
|
+
readonly retryAfterSec: number | undefined;
|
|
75
|
+
/** Delay actually awaited before the next attempt. */
|
|
76
|
+
readonly delayMs: number;
|
|
77
|
+
/** One-line error summary (same shape as the existing `console.error` line). */
|
|
78
|
+
readonly message: string;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* What sampling the provider ACTUALLY received for a text call — so a
|
|
82
|
+
* caller that asked for `temperature: 0` (a disclosed reproducibility
|
|
83
|
+
* property, e.g. the benchmark judge panel) can record the effective
|
|
84
|
+
* value instead of the requested one. `'provider-default'` means no
|
|
85
|
+
* temperature reached the provider: either the caller passed none, or
|
|
86
|
+
* the adapter stripped it because the model rejects sampling params
|
|
87
|
+
* (Anthropic Opus 4.7+ / 5-family → HTTP 400 on non-default
|
|
88
|
+
* `temperature`/`top_p`/`top_k`); `strippedReason` names the second case.
|
|
89
|
+
*/
|
|
90
|
+
interface AppliedSampling {
|
|
91
|
+
readonly temperature: number | 'provider-default';
|
|
92
|
+
readonly strippedReason?: string;
|
|
93
|
+
}
|
|
94
|
+
interface LLMResponse {
|
|
95
|
+
text: string;
|
|
96
|
+
inputTokens: number;
|
|
97
|
+
outputTokens: number;
|
|
98
|
+
/**
|
|
99
|
+
* Effective sampling for this call (see {@link AppliedSampling}).
|
|
100
|
+
* Every provider's `callText` sets it; absent only on paths that
|
|
101
|
+
* predate the field — treat absence as unknown, never as "as
|
|
102
|
+
* requested".
|
|
103
|
+
*/
|
|
104
|
+
sampling?: AppliedSampling;
|
|
105
|
+
/**
|
|
106
|
+
* Why the provider stopped, normalized (ggui#1127): `length` = it hit the output cap, `stop` = it ended its
|
|
107
|
+
* answer, `content-filter` = a safety stop, `other` = anything else it reported. Set by the vision calls of
|
|
108
|
+
* providers that report one (Anthropic `stop_reason`, Google `finishReason`); absent when the provider or route
|
|
109
|
+
* said nothing — read absence as unknown, never as `stop`.
|
|
110
|
+
*/
|
|
111
|
+
finishReason?: VisionFinishReason;
|
|
112
|
+
/**
|
|
113
|
+
* The model that answered, in the provider's own words (ggui#1687): Anthropic's `model`, Google's `modelVersion`.
|
|
114
|
+
* The request names an alias or a resolved id; this is what the provider says served it. Absent when it said
|
|
115
|
+
* nothing — read absence as unknown, never as the requested model.
|
|
116
|
+
*/
|
|
117
|
+
servedModel?: string;
|
|
118
|
+
}
|
|
119
|
+
/** The normalized stop reason a vision call reports (ggui#1127). */
|
|
120
|
+
type VisionFinishReason = 'stop' | 'length' | 'content-filter' | 'other';
|
|
121
|
+
interface LLMTool {
|
|
122
|
+
name: string;
|
|
123
|
+
description: string;
|
|
124
|
+
parameters: JsonObject;
|
|
125
|
+
handler: (args: JsonObject) => Promise<{
|
|
126
|
+
content: Array<{
|
|
127
|
+
text: string;
|
|
128
|
+
}>;
|
|
129
|
+
isError?: boolean;
|
|
130
|
+
}>;
|
|
131
|
+
}
|
|
132
|
+
interface LLMWithToolsResponse {
|
|
133
|
+
text: string;
|
|
134
|
+
inputTokens: number;
|
|
135
|
+
outputTokens: number;
|
|
136
|
+
turnsUsed: number;
|
|
137
|
+
/** Prompt-cache reads summed over the loop's turns, when the provider reports them (absent = unreported, never zero). */
|
|
138
|
+
cacheReadTokens?: number;
|
|
139
|
+
}
|
|
140
|
+
interface LLMToolCall {
|
|
141
|
+
/** Provider-specific call ID (for sendToolResult) */
|
|
142
|
+
id?: string;
|
|
143
|
+
name: string;
|
|
144
|
+
input: JsonObject;
|
|
145
|
+
}
|
|
146
|
+
interface LLMToolCallResponse {
|
|
147
|
+
toolCalls: LLMToolCall[];
|
|
148
|
+
inputTokens: number;
|
|
149
|
+
outputTokens: number;
|
|
150
|
+
/**
|
|
151
|
+
* The tool-choice mode the provider ACTUALLY received. Callers ask
|
|
152
|
+
* for `'required'` (Anthropic `tool_choice: any`); Claude Fable 5.1 and
|
|
153
|
+
* Opus 5.5 reject forced tool choice with HTTP 400, so the Anthropic
|
|
154
|
+
* and OpenRouter adapters downgrade to `'auto'` for those families and
|
|
155
|
+
* report it here. A
|
|
156
|
+
* downgraded call may legitimately return zero tool calls — the
|
|
157
|
+
* evaluator's `criteriaCoverage` records that as `skipped`, never as
|
|
158
|
+
* a pass. Absent = predates the field (treat as unknown).
|
|
159
|
+
*/
|
|
160
|
+
appliedToolChoice?: 'required' | 'auto';
|
|
161
|
+
/**
|
|
162
|
+
* Tokens read from / written to the prompt cache, when the provider
|
|
163
|
+
* reports them. Optional because not every provider exposes
|
|
164
|
+
* prompt-cache accounting — absent means "unreported", never zero.
|
|
165
|
+
*/
|
|
166
|
+
cacheReadTokens?: number;
|
|
167
|
+
cacheCreationTokens?: number;
|
|
168
|
+
}
|
|
169
|
+
/** Result of executing a tool — passed to sendToolResult to close the API contract. */
|
|
170
|
+
interface LLMToolResult {
|
|
171
|
+
/** Tool call ID from the response (for providers that need it) */
|
|
172
|
+
callId?: string;
|
|
173
|
+
/** Tool name */
|
|
174
|
+
name: string;
|
|
175
|
+
/** Text result of executing the tool */
|
|
176
|
+
result: string;
|
|
177
|
+
/** Whether the tool execution failed */
|
|
178
|
+
isError?: boolean;
|
|
179
|
+
}
|
|
180
|
+
declare abstract class LLMAgent {
|
|
181
|
+
abstract readonly provider: AgentConfig['provider'];
|
|
182
|
+
private client;
|
|
183
|
+
protected lastSessionId: string | undefined;
|
|
184
|
+
/**
|
|
185
|
+
* Explicit per-call routing override — see `AgentConfig.routeOverride`.
|
|
186
|
+
* Absent when constructed via the bare `new XAgent()` form (every
|
|
187
|
+
* existing call site), so `createClient()`/`resolveModel()` fall
|
|
188
|
+
* back to `process.env` exactly as before.
|
|
189
|
+
*/
|
|
190
|
+
protected readonly routeOverride: AgentConfig['routeOverride'];
|
|
191
|
+
/** See `AgentConfig.onRetry` (#489). Absent is a no-op. */
|
|
192
|
+
protected readonly onRetry: AgentConfig['onRetry'];
|
|
193
|
+
constructor(routeOverride?: AgentConfig['routeOverride'], onRetry?: AgentConfig['onRetry']);
|
|
194
|
+
protected abstract resolveModel(model: string): string;
|
|
195
|
+
protected abstract createClient(): Promise<unknown>;
|
|
196
|
+
protected getClient<T>(): Promise<T>;
|
|
197
|
+
/** Text-only call — no tools */
|
|
198
|
+
abstract callText(model: string, systemPrompt: string, userPrompt: string, maxTokens?: number, temperature?: number): Promise<LLMResponse>;
|
|
199
|
+
/**
|
|
200
|
+
* Single-turn function calling — returns tool calls without executing them.
|
|
201
|
+
* Each provider uses its native function/tool calling:
|
|
202
|
+
* - Anthropic: tool_use blocks
|
|
203
|
+
* - OpenAI: function_call output items
|
|
204
|
+
* - Google: functionCall parts
|
|
205
|
+
* SDK handles JSON escaping — safe for code, diffs, and other content.
|
|
206
|
+
*/
|
|
207
|
+
abstract callTools(model: string, systemPrompt: string, userPrompt: string, tools: LLMToolDef[], toolChoice?: 'required' | 'auto',
|
|
208
|
+
/**
|
|
209
|
+
* Optional scoped fallback tools. If the primary tools fail with a
|
|
210
|
+
* transport-class error (e.g. `malformed_tool_call` on Gemini after
|
|
211
|
+
* retry exhaustion), the provider may retry once with these narrower
|
|
212
|
+
* tools before throwing. Universal signal — not provider-gated.
|
|
213
|
+
*/
|
|
214
|
+
scopedTools?: LLMToolDef[]): Promise<LLMToolCallResponse>;
|
|
215
|
+
/** Multi-turn agentic loop — executes tools internally */
|
|
216
|
+
abstract callWithTools(model: string, systemPrompt: string, userPrompt: string, tools: LLMTool[], maxTurns: number): Promise<LLMWithToolsResponse>;
|
|
217
|
+
/**
|
|
218
|
+
* Pre-warm cache for repeated callTools() calls with the same system prompt + tools.
|
|
219
|
+
* Override in providers that support server-side context caching (e.g., Google).
|
|
220
|
+
* No-op by default (Anthropic/OpenAI handle caching automatically per-request).
|
|
221
|
+
*/
|
|
222
|
+
warmCache(_model: string, _systemPrompt: string, _tools: LLMToolDef[], _toolChoice?: 'required' | 'auto'): Promise<void>;
|
|
223
|
+
/** Cleanup any cached resources. Call after generation completes. No-op by default. */
|
|
224
|
+
cleanup(): Promise<void>;
|
|
225
|
+
/** Reset session state between independent generation runs. */
|
|
226
|
+
resetSession(): void;
|
|
227
|
+
/**
|
|
228
|
+
* Send tool execution results back to the provider to close the API contract.
|
|
229
|
+
* Call this after executing tools from callTools() and before the next callTools().
|
|
230
|
+
*
|
|
231
|
+
* For providers with server-side state (Google, OpenAI), this sends the
|
|
232
|
+
* function results so the next callTools() can chain properly.
|
|
233
|
+
* For stateless providers (Anthropic), this is a no-op.
|
|
234
|
+
*
|
|
235
|
+
* Override in providers that need it.
|
|
236
|
+
*/
|
|
237
|
+
sendToolResult(_results: LLMToolResult[]): Promise<void>;
|
|
238
|
+
/**
|
|
239
|
+
* Execute an API call, retrying once or twice on HTTP 429
|
|
240
|
+
* (rate-limited) before giving up (#489).
|
|
241
|
+
*
|
|
242
|
+
* Policy: on a 429, honor the provider's `Retry-After` header when
|
|
243
|
+
* present (capped at `RETRY_AFTER_CAP_SEC` — a provider asking for a
|
|
244
|
+
* longer wait is treated as "don't retry", not "wait longer": making
|
|
245
|
+
* the caller sit through 15+ more seconds on a single failed call is
|
|
246
|
+
* a bad experience regardless of what's driving the request);
|
|
247
|
+
* otherwise fall back to exponential backoff with jitter, capped at
|
|
248
|
+
* `DEFAULT_BACKOFF_MAX_MS` per attempt. Stops after
|
|
249
|
+
* `MAX_RETRY_ATTEMPTS` retries OR once the cumulative delay would
|
|
250
|
+
* exceed `MAX_TOTAL_RETRY_DELAY_MS`, whichever comes first. Any
|
|
251
|
+
* non-429 error, or a 429 with no attempts left, is logged and
|
|
252
|
+
* re-thrown immediately — unchanged from the pre-#489 behavior.
|
|
253
|
+
*
|
|
254
|
+
* **Budget scope (final-review correction):** `MAX_TOTAL_RETRY_DELAY_MS`
|
|
255
|
+
* bounds a single `apiCall()` invocation, not a whole generation. A
|
|
256
|
+
* generation issues many sequential `apiCall()` calls (coding turns,
|
|
257
|
+
* tool-result round-trips, eval rounds); under sustained throttling
|
|
258
|
+
* each one can independently add up to `MAX_TOTAL_RETRY_DELAY_MS`, so
|
|
259
|
+
* a generation with k rate-limited calls can add up to roughly `k *
|
|
260
|
+
* MAX_TOTAL_RETRY_DELAY_MS` to how long it holds its caller's
|
|
261
|
+
* resources (e.g. the caller's concurrency slot) — not a flat 20s
|
|
262
|
+
* ceiling. No shared per-generation budget is threaded here by
|
|
263
|
+
* design (accepted ruling, not an oversight): the caller's own
|
|
264
|
+
* queue/admission backstop is what bounds how long other
|
|
265
|
+
* queued work waits, independent of any one generation's retry
|
|
266
|
+
* total.
|
|
267
|
+
*/
|
|
268
|
+
protected apiCall<T>(fn: () => Promise<T>): Promise<T>;
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Create a fresh agent instance for the given provider.
|
|
272
|
+
* Each call returns a new instance — no shared state between callers.
|
|
273
|
+
*
|
|
274
|
+
* Accepts either a bare provider string (every pre-existing call
|
|
275
|
+
* site — the agent falls back to reading its provider's env var(s)
|
|
276
|
+
* directly) or a full {@link AgentConfig}, whose `routeOverride`
|
|
277
|
+
* (when present) is threaded into the constructed agent so it never
|
|
278
|
+
* touches `process.env`.
|
|
279
|
+
*/
|
|
280
|
+
declare function createAgent(provider: AgentConfig['provider']): LLMAgent;
|
|
281
|
+
declare function createAgent(config: AgentConfig): LLMAgent;
|
|
282
|
+
/** Providers whose agent implements {@link VisionAgent}. */
|
|
283
|
+
type VisionProvider = 'anthropic' | 'google';
|
|
284
|
+
declare function callLLM(config: AgentConfig, systemPrompt: string, userPrompt: string, maxTokens?: number): Promise<LLMResponse>;
|
|
285
|
+
|
|
286
|
+
type Priority = "P0" | "P1" | "P2";
|
|
287
|
+
|
|
288
|
+
type EvalTier = 0 | 1 | 2;
|
|
289
|
+
type EvalOutcome = "fail" | "warn" | "pass";
|
|
290
|
+
type EvalCategory = "compile" | "security" | "contract" | "types" | "imports" | "tokens" | "mode" | "functionality" | "crash" | "interactivity" | "accessibility" | "layout" | "loading" | "visual";
|
|
291
|
+
interface EvalIssue {
|
|
292
|
+
tier: EvalTier;
|
|
293
|
+
result: EvalOutcome;
|
|
294
|
+
category: EvalCategory;
|
|
295
|
+
/**
|
|
296
|
+
* Priority tier — threaded through to the retry formatter so the LLM can
|
|
297
|
+
* rank its next patch against the prompt's P0/P1/P2 priority schema.
|
|
298
|
+
* Derived from (category, result) via {@link priorityForIssue} when not
|
|
299
|
+
* explicitly set. Optional on construction; check-runners typically
|
|
300
|
+
* populate it before returning.
|
|
301
|
+
*/
|
|
302
|
+
priority?: Priority;
|
|
303
|
+
subcategory?: string;
|
|
304
|
+
severity?: "critical" | "major";
|
|
305
|
+
description: string;
|
|
306
|
+
fix: string;
|
|
307
|
+
line?: number;
|
|
308
|
+
/**
|
|
309
|
+
* The visual leg's provenance (ggui#1545): `judge` (the model's finding) or `instrument` (a deterministic
|
|
310
|
+
* measurement). Absent on every other leg's issues. `visualEvaluation.actOn` acts on `judge` only.
|
|
311
|
+
*/
|
|
312
|
+
origin?: "judge" | "instrument";
|
|
313
|
+
}
|
|
314
|
+
/**
|
|
315
|
+
* Runtime-render probe execution status — four-valued so "the probe
|
|
316
|
+
* never ran" is never conflated with "the probe ran clean". ONE union: the
|
|
317
|
+
* same values a generation's metadata reports (`GenerationRuntimeProbeStatus`
|
|
318
|
+
* in `@ggui-ai/mcp-server-core`), declared there and named here.
|
|
319
|
+
*
|
|
320
|
+
* - `ran` — probe executed; its findings (possibly none) are
|
|
321
|
+
* in the issue stream under `runtime:*` subcategories.
|
|
322
|
+
* - `infra-skipped` — probe could NOT execute (environment failure:
|
|
323
|
+
* happy-dom load, user-event document capture,
|
|
324
|
+
* ESM/CJS interop). Carries ZERO probe evidence;
|
|
325
|
+
* consumers MUST NOT count it as a pass.
|
|
326
|
+
* - `not-applicable`— nothing to probe (no compiled code, no contract
|
|
327
|
+
* surface, or no runtimeRender check configured).
|
|
328
|
+
* - `timed-out` — probe started but did not finish inside its
|
|
329
|
+
* wall-clock bound (ggui#1299). Carries ZERO
|
|
330
|
+
* evidence about the component: on a contended
|
|
331
|
+
* host an ordinary component crosses the bound.
|
|
332
|
+
* Consumers MUST NOT count it as a pass, and MUST
|
|
333
|
+
* NOT count it as a crash either. `elapsedMs` and
|
|
334
|
+
* `hostLoad` say how long it ran and how busy the
|
|
335
|
+
* host was, so a reader can tell the two cases apart.
|
|
336
|
+
*/
|
|
337
|
+
type RuntimeProbeStatus = GenerationRuntimeProbeStatus;
|
|
338
|
+
/**
|
|
339
|
+
* The host's 1-minute load average at a probe's start and end, beside the
|
|
340
|
+
* number of CPUs the process can use (ggui#1299). A load well above
|
|
341
|
+
* `cores` means the probe competed for CPU; a wall-clock bound crossed
|
|
342
|
+
* there says more about the host than about the component.
|
|
343
|
+
*/
|
|
344
|
+
interface ProbeHostLoad {
|
|
345
|
+
readonly start: number;
|
|
346
|
+
readonly end: number;
|
|
347
|
+
readonly cores: number;
|
|
348
|
+
}
|
|
349
|
+
/**
|
|
350
|
+
* ggui#1398 — whether the controls the probe clicked showed they were
|
|
351
|
+
* working once their action was pending. Report-only: it is read by the
|
|
352
|
+
* benchmark and never counted into the probe's warnings or fed back to
|
|
353
|
+
* the coding loop.
|
|
354
|
+
*/
|
|
355
|
+
interface ProbePendingAffordance {
|
|
356
|
+
/** Controls the pending phase marked pending: those whose click dispatched their action (click or submit wiring) and that were still in the page. `visible + missing.length === dispatched`. */
|
|
357
|
+
readonly dispatched: number;
|
|
358
|
+
/** Of those, how many visibly changed while pending (disabled, busy, text, or replaced). */
|
|
359
|
+
readonly visible: number;
|
|
360
|
+
/** The actions whose control did not visibly change, including one whose card threw while pending. */
|
|
361
|
+
readonly missing: readonly string[];
|
|
362
|
+
/** Actions whose control was no longer in the page when its turn came (the card replaced it, or blanked after a throw): never marked, not counted as dispatched. */
|
|
363
|
+
readonly gone: readonly string[];
|
|
364
|
+
}
|
|
365
|
+
/**
|
|
366
|
+
* The engine's own fields on a probe's meta, beside the outcome a
|
|
367
|
+
* generation's metadata reports: why a probe did not finish, its render
|
|
368
|
+
* time, the host's load around it. Present on every arm of
|
|
369
|
+
* {@link RuntimeProbeMeta}; each is absent when the probe had nothing to
|
|
370
|
+
* say about it.
|
|
371
|
+
*/
|
|
372
|
+
interface RuntimeProbeMetaDetail {
|
|
373
|
+
/** Populated for `infra-skipped` / `not-applicable` / `timed-out` — why the probe didn't finish. */
|
|
374
|
+
readonly reason?: string;
|
|
375
|
+
/** `ran` only: the check's own render time, inside `elapsedMs`. */
|
|
376
|
+
readonly renderMs?: number;
|
|
377
|
+
/** Host load around the probe, when it ran isolated (absent for an in-process probe). */
|
|
378
|
+
readonly hostLoad?: ProbeHostLoad;
|
|
379
|
+
/** Time the check waited for a probe slot before it started, ms; present only when > 0 (ggui#1380 C2b). */
|
|
380
|
+
readonly queuedMs?: number;
|
|
381
|
+
/**
|
|
382
|
+
* `ran` only: the pending-affordance walk (ggui#1398). Absent when no
|
|
383
|
+
* action walk ran, so "no data" and "0 of 0" read differently.
|
|
384
|
+
*/
|
|
385
|
+
readonly pendingAffordance?: ProbePendingAffordance;
|
|
386
|
+
}
|
|
387
|
+
/**
|
|
388
|
+
* Probe execution meta stamped onto an `EvalResult` at the exit-decision
|
|
389
|
+
* points that invoke the probe. Absence of this field means the eval
|
|
390
|
+
* path never reached a probe invocation at all — consumers treat that
|
|
391
|
+
* the same as not-run, never as a pass.
|
|
392
|
+
*
|
|
393
|
+
* ONE union with the generation metadata's probe outcome
|
|
394
|
+
* (`GenerationRuntimeProbeOutcome` in `@ggui-ai/mcp-server-core`, declared
|
|
395
|
+
* there and carried here with {@link RuntimeProbeMetaDetail} on every arm),
|
|
396
|
+
* so the round's stamp and the metadata's record cannot say different
|
|
397
|
+
* things (ggui#1380):
|
|
398
|
+
*
|
|
399
|
+
* - `ran` + `verdict: 'pass'` — the probe executed and no check failed;
|
|
400
|
+
* - `ran` + `verdict: 'fail'` — ANY check failed; `failChecks` lists the
|
|
401
|
+
* distinct check kinds that did (`RenderCheckKind`, the engine's name
|
|
402
|
+
* for the same union), never empty, in the order `RENDER_CHECK_KINDS`
|
|
403
|
+
* declares them, parsed from the probe's own
|
|
404
|
+
* `runtime:<check>[:<subject>]` issues;
|
|
405
|
+
* - `timed-out` / `infra-skipped` / `not-applicable` — no verdict, and the
|
|
406
|
+
* type carries none. `elapsedMs` (on every status that reached the
|
|
407
|
+
* check) and `hostLoad` say how long it ran and how busy the host was.
|
|
408
|
+
*
|
|
409
|
+
* The verdict and the repair trigger are two different questions. The
|
|
410
|
+
* verdict is `fail` on any failing check. The one repair turn a probe-only
|
|
411
|
+
* round buys fires on a `render-no-throw` fail the engine recognises
|
|
412
|
+
* (`isRecoverableRenderCrash`) and on nothing else: on the PROBE-ONLY round a
|
|
413
|
+
* `fail` verdict whose checks are `prop-sensitivity`, `action-wiring` or any
|
|
414
|
+
* other kind is recorded here and never fed back. The same meta is stamped on
|
|
415
|
+
* the EVALUATION lane, where a `prop-sensitivity` fail buys that lane's own
|
|
416
|
+
* contract-feedback round (ggui#1261, `FEEDBACK_PROBE_CHECKS`) — a different
|
|
417
|
+
* turn under a different rule, not the repair turn.
|
|
418
|
+
*/
|
|
419
|
+
type RuntimeProbeMeta = GenerationRuntimeProbeOutcome & RuntimeProbeMetaDetail;
|
|
420
|
+
/**
|
|
421
|
+
* The one repair turn a probe-only round buys (ggui#1380). A serving
|
|
422
|
+
* deployment that wires the runtime probe and configures no evaluator runs
|
|
423
|
+
* the probe once after the coding turns; a render crash the engine
|
|
424
|
+
* recognises buys exactly one repair turn, then one re-probe. This records
|
|
425
|
+
* what came of it:
|
|
426
|
+
* - `compiled: false` — the repair turn's code did not pass self-check, so
|
|
427
|
+
* no re-probe ran (there is no `after`) and the pre-repair card is the
|
|
428
|
+
* one served;
|
|
429
|
+
* - `compiled: true` — the re-probe ran and `after` is its whole meta:
|
|
430
|
+
* its status, its verdict when it ran, and the checks that still fail.
|
|
431
|
+
* Whether the crash class the turn was bought for is still there is
|
|
432
|
+
* `after.verdict === 'fail' && after.failChecks.includes('render-no-throw')`.
|
|
433
|
+
* The two arms are exclusive by type.
|
|
434
|
+
*/
|
|
435
|
+
type RuntimeProbeRepair = {
|
|
436
|
+
readonly attempted: true;
|
|
437
|
+
readonly compiled: false;
|
|
438
|
+
readonly trigger?: never;
|
|
439
|
+
} | {
|
|
440
|
+
readonly attempted: true;
|
|
441
|
+
readonly compiled: true;
|
|
442
|
+
readonly trigger: RuntimeProbeMeta;
|
|
443
|
+
};
|
|
444
|
+
/**
|
|
445
|
+
* Per-criterion execution status — three-valued so "the criterion
|
|
446
|
+
* silently produced no verdict" is never conflated with "the criterion
|
|
447
|
+
* ran clean", and neither is conflated with "not evaluated by design".
|
|
448
|
+
* Same doctrine and vocabulary family as `RuntimeProbeStatus` above.
|
|
449
|
+
*
|
|
450
|
+
* - `ran` — the LLM call completed and returned a verdict
|
|
451
|
+
* for this criterion; its findings (possibly
|
|
452
|
+
* none) are in `issues`.
|
|
453
|
+
* - `skipped` — the call errored or returned no tool call. The
|
|
454
|
+
* criterion carries ZERO evidence; the evaluator
|
|
455
|
+
* fail-opens it into `pass` to avoid blocking
|
|
456
|
+
* generation on flaky LLM calls, so consumers
|
|
457
|
+
* MUST consult this field before treating `pass`
|
|
458
|
+
* membership as a verdict. Counts AGAINST
|
|
459
|
+
* coverage.
|
|
460
|
+
* - `not-applicable` — the evaluator was bypassed by design for this
|
|
461
|
+
* cell (e.g. the same-image low-risk bypass:
|
|
462
|
+
* axis checks clean, tier-1/2 never invoked).
|
|
463
|
+
* `reason` names the bypass. Excluded from
|
|
464
|
+
* coverage denominators — it is neither evidence
|
|
465
|
+
* nor a silent absence.
|
|
466
|
+
*/
|
|
467
|
+
type CriterionRunStatus = "ran" | "skipped" | "not-applicable";
|
|
468
|
+
/** Coverage row for one evaluation criterion of one eval run. */
|
|
469
|
+
interface CriterionCoverage {
|
|
470
|
+
readonly criterion: string;
|
|
471
|
+
readonly tier: 1 | 2;
|
|
472
|
+
readonly status: CriterionRunStatus;
|
|
473
|
+
/** Populated for `skipped` / `not-applicable` — why no verdict. */
|
|
474
|
+
readonly reason?: string;
|
|
475
|
+
}
|
|
476
|
+
/**
|
|
477
|
+
* Coverage of the screenshot JUDGE leg for one eval round (ggui#1221) — a
|
|
478
|
+
* different surface from the `visual` CRITERION in `criteriaCoverage` (the
|
|
479
|
+
* LLM evaluator's read of the source). `ran` = a frame was captured and
|
|
480
|
+
* judged; `skipped` = the leg was configured but produced no verdict
|
|
481
|
+
* (no browser, a launch or in-page failure, an unbundlable component, an
|
|
482
|
+
* unparsable judge answer) with `reason` naming why; `not-applicable` =
|
|
483
|
+
* the round had no visual leg configured. Before this row a skipped leg
|
|
484
|
+
* left NO trace on the result — `{ issues: [] }` read exactly like a
|
|
485
|
+
* clean run, and every bootstrap mint on a browserless runtime "passed"
|
|
486
|
+
* its visual round that way.
|
|
487
|
+
*/
|
|
488
|
+
interface VisualCoverage {
|
|
489
|
+
readonly status: CriterionRunStatus;
|
|
490
|
+
/** Populated for `skipped` / `not-applicable` — why no verdict. */
|
|
491
|
+
readonly reason?: string;
|
|
492
|
+
}
|
|
493
|
+
/**
|
|
494
|
+
* The static criteria `runLLMEvaluation` runs, in stamp order — the
|
|
495
|
+
* ONE list both the evaluator's coverage stamp and the harness's
|
|
496
|
+
* bypass stamp derive from, so the two can never disagree on the set.
|
|
497
|
+
* Tier 1 = binary pass/fail gates; tier 2 = fail/warn/pass quality.
|
|
498
|
+
*/
|
|
499
|
+
declare const LLM_EVAL_STATIC_CRITERIA: ReadonlyArray<{
|
|
500
|
+
readonly criterion: string;
|
|
501
|
+
readonly tier: 1 | 2;
|
|
502
|
+
}>;
|
|
503
|
+
/**
|
|
504
|
+
* Coverage rows for a cell where the evaluator was BYPASSED BY DESIGN
|
|
505
|
+
* (never invoked) — one `not-applicable` row per static criterion, all
|
|
506
|
+
* carrying `reason`. Harness exits that skip tier-1/2 evaluation stamp
|
|
507
|
+
* this so the cell is neither a silent absence (field missing) nor
|
|
508
|
+
* evidence. Same shape contract as `runLLMEvaluation`'s stamp: always
|
|
509
|
+
* every criterion. Lives here (not in llm-evaluator) because the bypass
|
|
510
|
+
* exit runs before the evaluator module is lazily loaded.
|
|
511
|
+
*/
|
|
512
|
+
declare function notApplicableCoverage(reason: string): CriterionCoverage[];
|
|
513
|
+
/** One contract-feedback round (ggui#1261): what bought it, and the source it was fed back on. */
|
|
514
|
+
interface ContractFeedbackRecord {
|
|
515
|
+
/** The exit-probe issue subcategories that bought the round, e.g. `runtime:prop-sensitivity:currentUser`. */
|
|
516
|
+
readonly firedOn: readonly string[];
|
|
517
|
+
/** The component source as it stood when the model was told — the round's BEFORE. */
|
|
518
|
+
readonly sourceBefore: string;
|
|
519
|
+
}
|
|
520
|
+
interface EvalResult {
|
|
521
|
+
issues: EvalIssue[];
|
|
522
|
+
pass: string[];
|
|
523
|
+
/**
|
|
524
|
+
* Per-criterion coverage for this eval run — one row per static
|
|
525
|
+
* criterion, ALWAYS all of them when present. Stamped by
|
|
526
|
+
* `runLLMEvaluation` (`ran` / `skipped`) and by the harness on
|
|
527
|
+
* evaluator-bypass exits (`not-applicable` + reason). The harness
|
|
528
|
+
* carries it unchanged through every later re-stamp of this result.
|
|
529
|
+
* Absent = coverage UNKNOWN (paths that predate the instrument, e.g.
|
|
530
|
+
* `parseEvalResponse` replays) — never assume `ran`, never assume
|
|
531
|
+
* bypass.
|
|
532
|
+
*/
|
|
533
|
+
criteriaCoverage?: CriterionCoverage[];
|
|
534
|
+
/**
|
|
535
|
+
* Runtime-render probe execution meta. Stamped by the exit-decision
|
|
536
|
+
* probe runner; absent on eval paths that never invoke the probe.
|
|
537
|
+
*/
|
|
538
|
+
runtimeProbe?: RuntimeProbeMeta;
|
|
539
|
+
/**
|
|
540
|
+
* The probe-only round's repair record (ggui#1380) — present only when a
|
|
541
|
+
* recoverable render crash bought the one repair turn. See
|
|
542
|
+
* {@link RuntimeProbeRepair}. `runtimeProbe` beside it is the re-probe's
|
|
543
|
+
* verdict when the repair compiled, else the pre-repair probe's.
|
|
544
|
+
*/
|
|
545
|
+
runtimeProbeRepair?: RuntimeProbeRepair;
|
|
546
|
+
/**
|
|
547
|
+
* The contract-feedback round, when one fired this generation (ggui#1261):
|
|
548
|
+
* which exit-probe findings bought it and the source as it stood when the
|
|
549
|
+
* model was told. `runtimeProbe` and the `runtime:*` issues on this result
|
|
550
|
+
* are the probe's verdict AFTER that round; `sourceBefore` beside the final
|
|
551
|
+
* source is the round's before/after. Absent when no such round fired.
|
|
552
|
+
*/
|
|
553
|
+
contractFeedback?: ContractFeedbackRecord;
|
|
554
|
+
/**
|
|
555
|
+
* Per-canvas visual verdicts. Stamped by the harness ONLY when the
|
|
556
|
+
* visual leg ran with `visualEvaluation.canvases` set (arm-neutral —
|
|
557
|
+
* the same judge in every design mode); absent on the single-shot
|
|
558
|
+
* visual path and whenever the visual leg did not run. PNGs stay on
|
|
559
|
+
* `runVisualEvaluation`'s own result (`VisualEvaluationResult.canvases[*].screenshotPng`),
|
|
560
|
+
* never here — this object is serialized into reports.
|
|
561
|
+
*/
|
|
562
|
+
visual?: VisualEvalSummary;
|
|
563
|
+
/**
|
|
564
|
+
* Whether the screenshot judge leg ran, was skipped (with the reason) or
|
|
565
|
+
* was not configured — ALWAYS stamped by the harness's eval round
|
|
566
|
+
* (ggui#1221), so a consumer can tell a clean visual run from a leg
|
|
567
|
+
* that silently could not run. See {@link VisualCoverage}.
|
|
568
|
+
*/
|
|
569
|
+
visualCoverage?: VisualCoverage;
|
|
570
|
+
}
|
|
571
|
+
/** One canvas's visual verdict, PNG-free — see `EvalResult.visual`. */
|
|
572
|
+
/**
|
|
573
|
+
* How one canvas's `score` was reached (ggui#1072): `k` vision calls on the
|
|
574
|
+
* SAME captured frame, aggregated by `rule`. Always present — `k: 1` ⇒
|
|
575
|
+
* `samples: [score]`, `sigma: 0`, `notes: [critique]` — so one shape serves
|
|
576
|
+
* a single judgement and a median of many. `samples`/`notes` carry only the
|
|
577
|
+
* calls the judge could parse (in call order); `k` stays the requested count,
|
|
578
|
+
* so a short `samples` is visible on the row rather than filled in.
|
|
579
|
+
*/
|
|
580
|
+
interface CanvasJudgeRecord {
|
|
581
|
+
readonly k: number;
|
|
582
|
+
readonly rule: 'median';
|
|
583
|
+
readonly samples: number[];
|
|
584
|
+
/** Population σ of `samples` (0 when one sample). */
|
|
585
|
+
readonly sigma: number;
|
|
586
|
+
/** The judge's per-canvas critique per sample — the words beside a swing. */
|
|
587
|
+
readonly notes: string[];
|
|
588
|
+
/**
|
|
589
|
+
* ggui#1127 — the cause of each scoring sample whose answer failed to parse and was SALVAGED: its four dimensions
|
|
590
|
+
* were recovered from the closed prefix, its issues kept only as far as they closed, its critique lost. One entry
|
|
591
|
+
* per salvaged sample; absent when every sample parsed. The salvaged sample's score still counts.
|
|
592
|
+
*/
|
|
593
|
+
readonly salvaged?: readonly JudgeSalvageCause[];
|
|
594
|
+
/**
|
|
595
|
+
* ggui#1127 — the criteria call's answer was salvaged. A salvage keeps no criteria, so that frame's judge rows read
|
|
596
|
+
* "not answered" with this cause rather than a clean `n/a`. Absent when the call parsed or was not asked.
|
|
597
|
+
*/
|
|
598
|
+
readonly criteriaSalvaged?: JudgeSalvageCause;
|
|
599
|
+
/**
|
|
600
|
+
* ggui#1687 — the model asked for, and every model the provider said answered this canvas's calls (scoring and
|
|
601
|
+
* criteria), deduplicated. `served` is empty when no provider named one: unknown, never "the requested model".
|
|
602
|
+
*/
|
|
603
|
+
readonly models?: {
|
|
604
|
+
readonly requested: string;
|
|
605
|
+
readonly served: readonly string[];
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
/**
|
|
609
|
+
* ggui#1687 — what the criteria call's answer held, before resolution. A row the judge left unanswered reads `n/a`
|
|
610
|
+
* either way; this says whether the call came back empty, cut, or with entries the parser had to drop.
|
|
611
|
+
*/
|
|
612
|
+
interface CriteriaAnswerCounts {
|
|
613
|
+
/** The answer carried a `criteria` array. `false`: absent or not an array, so no row was read from it. */
|
|
614
|
+
readonly array: boolean;
|
|
615
|
+
/** Entries in the array. */
|
|
616
|
+
readonly received: number;
|
|
617
|
+
/** Entries kept: an object with a string id and a verdict of pass, fail or n/a. */
|
|
618
|
+
readonly kept: number;
|
|
619
|
+
/** Entries dropped, by why. */
|
|
620
|
+
readonly dropped: {
|
|
621
|
+
readonly notObject: number;
|
|
622
|
+
readonly badId: number;
|
|
623
|
+
readonly badVerdict: number;
|
|
624
|
+
};
|
|
625
|
+
}
|
|
626
|
+
/**
|
|
627
|
+
* ggui#1127 — why a judge answer that failed to parse had to be salvaged. `output-cap`: the provider reported it
|
|
628
|
+
* stopped at the output cap. `malformed`: the provider reported a normal stop, so the answer ended but did not
|
|
629
|
+
* parse. `stopped`: the provider stopped it for another reason it named (a safety stop, or one it reports as
|
|
630
|
+
* other), so it is cut, not malformed. `unknown`: the provider or route reported no stop reason. Never inferred
|
|
631
|
+
* from the answer's length.
|
|
632
|
+
*/
|
|
633
|
+
type JudgeSalvageCause = "output-cap" | "malformed" | "stopped" | "unknown";
|
|
634
|
+
/** ggui#1127 — salvaged judge answers across a canvas run, per leg, as N of M. */
|
|
635
|
+
interface JudgeSalvageCounts {
|
|
636
|
+
/** Scoring answers: `answers` = samples that parsed or were salvaged, `salvaged` = those salvaged. */
|
|
637
|
+
readonly scoring: {
|
|
638
|
+
readonly salvaged: number;
|
|
639
|
+
readonly answers: number;
|
|
640
|
+
};
|
|
641
|
+
/** Criteria calls: `answered` = calls that parsed or were salvaged, `salvaged` = those salvaged (criteria lost). */
|
|
642
|
+
readonly criteria: {
|
|
643
|
+
readonly salvaged: number;
|
|
644
|
+
readonly answered: number;
|
|
645
|
+
};
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
type CriteriaSeverity = "must" | "should";
|
|
649
|
+
type CriteriaVerdict = "pass" | "fail" | "n/a";
|
|
650
|
+
/** The method that produces a verdict: an instrument's measurement, the vision judge off the frame, or a person's eye. */
|
|
651
|
+
type CriteriaMethod = "instrument" | "judge" | "human";
|
|
652
|
+
/** The method's readiness, as the bank's author wrote it — the only source of an `n/a` that is not the judge's own. */
|
|
653
|
+
type CriteriaStatus = "live" | "reference" | "planned" | "manual";
|
|
654
|
+
/**
|
|
655
|
+
* How a criterion entered the selection: bank-level (`static`, an empty scope) or by the selector's
|
|
656
|
+
* rules for this card (`context`). A model PROPOSING a criterion at judge time is RESERVED and not a
|
|
657
|
+
* member until a consumer exists — it would be `should`-only, recorded verbatim, never a gate.
|
|
658
|
+
*/
|
|
659
|
+
type CriteriaSource = "static" | "context";
|
|
660
|
+
type CriteriaChroma = "achromatic" | "chromatic" | "unknown";
|
|
661
|
+
/** The classifier's shape axes, as hashed into the set id. */
|
|
662
|
+
interface CriteriaAxes {
|
|
663
|
+
render: RenderShape;
|
|
664
|
+
state: StateShape;
|
|
665
|
+
writes: WriteShape;
|
|
666
|
+
fetch: FetchShape;
|
|
667
|
+
layout: LayoutShape;
|
|
668
|
+
}
|
|
669
|
+
/** The selector's inputs, stored with the verdict so a reader recomputes `criteriaSetId` from the record alone. */
|
|
670
|
+
interface CriteriaContext {
|
|
671
|
+
canvas: CanvasClass;
|
|
672
|
+
hasActions: boolean;
|
|
673
|
+
riskTier: RiskTier;
|
|
674
|
+
axes: CriteriaAxes;
|
|
675
|
+
chroma: CriteriaChroma;
|
|
676
|
+
profilePresent: boolean;
|
|
677
|
+
/** The harness's shell as named; `unknown` when the round carries none. */
|
|
678
|
+
shell: string;
|
|
679
|
+
/** The caller's item kind (a bootstrap mint's `hello`); hashed when present. */
|
|
680
|
+
kind?: string;
|
|
681
|
+
/** The styling preset in force (the profile's aesthetic id); a recipe is selected only under its preset. */
|
|
682
|
+
preset?: string;
|
|
683
|
+
}
|
|
684
|
+
/** What was asked and why — recorded BEFORE the judge runs. Omission is the selector's decision. */
|
|
685
|
+
interface CriteriaSelection {
|
|
686
|
+
id: string;
|
|
687
|
+
source: CriteriaSource;
|
|
688
|
+
reason: string;
|
|
689
|
+
}
|
|
690
|
+
/** One criterion's verdict; `n/a` is the judge's answer ("cannot read this off the frame"), an unanswered id, a tie, or a method whose `status` is not live. */
|
|
691
|
+
interface CriterionVerdict {
|
|
692
|
+
id: string;
|
|
693
|
+
severity: CriteriaSeverity;
|
|
694
|
+
method: CriteriaMethod;
|
|
695
|
+
status: CriteriaStatus;
|
|
696
|
+
source: CriteriaSource;
|
|
697
|
+
verdict: CriteriaVerdict;
|
|
698
|
+
evidence: string;
|
|
699
|
+
/**
|
|
700
|
+
* ggui#1663 — a measured reading beside a judge row, REPORT-ONLY: the row's `verdict` stays the judge's, so this
|
|
701
|
+
* changes no verdict, score or bar. Present only on rows an instrument answers beside the judge (`space.edge`).
|
|
702
|
+
*/
|
|
703
|
+
instrument?: {
|
|
704
|
+
verdict: CriteriaVerdict;
|
|
705
|
+
evidence: string;
|
|
706
|
+
};
|
|
707
|
+
}
|
|
708
|
+
/** ggui#1687 — the criteria call as it came back: its answer's counts, ids it answered outside the bank, its stop. */
|
|
709
|
+
interface CriteriaCallRecord {
|
|
710
|
+
readonly answer: CriteriaAnswerCounts;
|
|
711
|
+
/** Kept entries whose id names no row of the bank. */
|
|
712
|
+
readonly unknownIds: number;
|
|
713
|
+
/** The provider's normalized stop reason (`length` = the output cap); absent when it reported none. */
|
|
714
|
+
readonly finishReason?: VisionFinishReason;
|
|
715
|
+
}
|
|
716
|
+
interface CriteriaBlock {
|
|
717
|
+
/** sha256(bank version | selector version | canonical context), 16 hex — the block's own digest. */
|
|
718
|
+
criteriaSetId: string;
|
|
719
|
+
bankVersion: string;
|
|
720
|
+
selectorVersion: string;
|
|
721
|
+
context: CriteriaContext;
|
|
722
|
+
selection: CriteriaSelection[];
|
|
723
|
+
verdicts: CriterionVerdict[];
|
|
724
|
+
/** ggui#1687 — the call that answered this block, when one did. Absent for a block resolved with no call. */
|
|
725
|
+
call?: CriteriaCallRecord;
|
|
726
|
+
}
|
|
727
|
+
/** The roll-up across canvases; a `must` at `n/a` is listed, never counted as a pass. */
|
|
728
|
+
interface CriteriaRollup {
|
|
729
|
+
mustFailed: string[];
|
|
730
|
+
shouldFailed: string[];
|
|
731
|
+
mustNa: string[];
|
|
732
|
+
}
|
|
733
|
+
/** ggui#1492 — a host"s frame round the inline card, as data. Colours are `#rrggbb`; the ring"s `alpha` is 0..1. */
|
|
734
|
+
interface HostInlineFrame {
|
|
735
|
+
/** The frame"s surface — what shows behind a transparent card, and under the floor"s residual. */
|
|
736
|
+
readonly surface: string;
|
|
737
|
+
/** The room the frame sits in (the page ground round it). */
|
|
738
|
+
readonly ground: string;
|
|
739
|
+
readonly ring: {
|
|
740
|
+
readonly widthPx: number;
|
|
741
|
+
readonly color: string;
|
|
742
|
+
readonly alpha: number;
|
|
743
|
+
};
|
|
744
|
+
readonly radiusPx: number;
|
|
745
|
+
/** The host"s minimum frame height: a card shorter than it sits in a frame this tall, and the visitor sees the rest as void. */
|
|
746
|
+
readonly minHeightPx?: number;
|
|
747
|
+
}
|
|
748
|
+
/** ggui#1492 — how a host presents one canvas: the label the judge is told, the page ground, and (the inline card only) its frame. */
|
|
749
|
+
interface CanvasHostPresentation {
|
|
750
|
+
readonly label: string;
|
|
751
|
+
/** The page ground under a fill canvas"s panel; the inline card"s ground lives on its {@link HostInlineFrame}. */
|
|
752
|
+
readonly ground?: string;
|
|
753
|
+
readonly frame?: HostInlineFrame;
|
|
754
|
+
}
|
|
755
|
+
/** ggui#1492 — why the judge did not draw a canvas's host presentation. The codes are the lane reader's own. */
|
|
756
|
+
type HostPresentationIgnoredReason = "label_missing" | "color_not_hex" | "frame_off_inline" | "frame_malformed" | "ground_on_inline";
|
|
757
|
+
/**
|
|
758
|
+
* ggui#1492 — what the judge DID with a canvas's host presentation, read from the judge itself: `applied` carries
|
|
759
|
+
* the presentation it drew, `ignored` the reason it drew none. Absent when the caller supplied no presentation.
|
|
760
|
+
*/
|
|
761
|
+
type CanvasPresentationOutcome = {
|
|
762
|
+
readonly status: "applied";
|
|
763
|
+
readonly applied: CanvasHostPresentation;
|
|
764
|
+
} | {
|
|
765
|
+
readonly status: "ignored";
|
|
766
|
+
readonly reason: HostPresentationIgnoredReason;
|
|
767
|
+
};
|
|
768
|
+
interface CanvasVisualSummary {
|
|
769
|
+
canvas: CanvasClass;
|
|
770
|
+
viewport: {
|
|
771
|
+
width: number;
|
|
772
|
+
height: number;
|
|
773
|
+
};
|
|
774
|
+
/** The judge's weighted score at this canvas (0-100). */
|
|
775
|
+
score: number;
|
|
776
|
+
/** `score >= passThreshold`, and on the inline card: no overflow (ggui#1027). */
|
|
777
|
+
passed: boolean;
|
|
778
|
+
/** The document's scroll height at this canvas, CSS px; `null` when unmeasurable (ggui#1027). */
|
|
779
|
+
contentHeight: number | null;
|
|
780
|
+
/** `contentHeight > viewport.height` — measured on every canvas, judged per `canvasFitPolicy`. */
|
|
781
|
+
overflow: boolean;
|
|
782
|
+
/**
|
|
783
|
+
* ggui#1120 — how far down the capture anything was painted: the last inked row over the measured
|
|
784
|
+
* region's height (inside a drawn host panel's chrome on md/lg/xl), read from the screenshot's
|
|
785
|
+
* pixels against their dominant colour. `0` is the blank the judge fails (`canvas-blank`); every
|
|
786
|
+
* other value is reported, never scored. `null` when the capture could not be read.
|
|
787
|
+
*/
|
|
788
|
+
inkRatio: number | null;
|
|
789
|
+
/** How `score` was reached — always present (ggui#1072). */
|
|
790
|
+
judge: CanvasJudgeRecord;
|
|
791
|
+
/** ggui#1436 — the typed criteria block, when a bank was configured; report-only. */
|
|
792
|
+
criteria?: CriteriaBlock;
|
|
793
|
+
/** ggui#1492 — what the judge did with this canvas's host presentation, when one was supplied. */
|
|
794
|
+
presentation?: CanvasPresentationOutcome;
|
|
795
|
+
/**
|
|
796
|
+
* How the judge composed the mount (ggui#1100): `'fill'` on every
|
|
797
|
+
* fullscreen canvas — the served runtime's fit, the root stretched to
|
|
798
|
+
* the frame; absent on the inline card, which keeps its natural height.
|
|
799
|
+
*/
|
|
800
|
+
fit?: 'fill';
|
|
801
|
+
}
|
|
802
|
+
/**
|
|
803
|
+
* ggui#1195 — the fit reading on the canvas whose policy FAILS an overflow
|
|
804
|
+
* (the inline chat card): the box it was judged at (`ceiling` — the order's
|
|
805
|
+
* declared viewport when one was carried, else the class box), whether that
|
|
806
|
+
* box was declared, and how many px of content sit below it (0 = fits).
|
|
807
|
+
* Present only when such a canvas was judged with a measurable height, so
|
|
808
|
+
* the bar and the ledger read one number for "does the hello fit".
|
|
809
|
+
*/
|
|
810
|
+
interface VisualFitStamp {
|
|
811
|
+
readonly canvas: CanvasClass;
|
|
812
|
+
readonly ceiling: {
|
|
813
|
+
readonly width: number;
|
|
814
|
+
readonly height: number;
|
|
815
|
+
};
|
|
816
|
+
readonly declared: boolean;
|
|
817
|
+
readonly overflowPx: number;
|
|
818
|
+
}
|
|
819
|
+
/** The visual leg's per-canvas summary — see `EvalResult.visual`. */
|
|
820
|
+
interface VisualEvalSummary {
|
|
821
|
+
/** Mean of the canvas scores (rounded). */
|
|
822
|
+
score: number;
|
|
823
|
+
/** Every canvas passed. */
|
|
824
|
+
passed: boolean;
|
|
825
|
+
canvases: CanvasVisualSummary[];
|
|
826
|
+
/** ggui#1195 — see {@link VisualFitStamp}; absent when no fail-policy canvas was judged. */
|
|
827
|
+
fit?: VisualFitStamp;
|
|
828
|
+
/** The design tree the judge painted with (ggui#1042): `src` + `srcSha256` — `design@judge`. */
|
|
829
|
+
design?: {
|
|
830
|
+
readonly src: string;
|
|
831
|
+
readonly srcSha256: string;
|
|
832
|
+
};
|
|
833
|
+
/** The mode the judge's tokens were composed in, when the caller said (ggui#1076). */
|
|
834
|
+
themeMode?: 'light' | 'dark';
|
|
835
|
+
/** ggui#1436 — the criteria roll-up across canvases, when any canvas carries a block; report-only. */
|
|
836
|
+
criteria?: CriteriaRollup;
|
|
837
|
+
}
|
|
838
|
+
type QualityMode = "fast" | "auto-improve" | "high-quality";
|
|
839
|
+
interface QualityConfig {
|
|
840
|
+
quality: QualityMode;
|
|
841
|
+
visualEval: boolean;
|
|
842
|
+
maxCostPerGeneration: number;
|
|
843
|
+
model?: {
|
|
844
|
+
provider?: string;
|
|
845
|
+
model?: string;
|
|
846
|
+
};
|
|
847
|
+
}
|
|
848
|
+
declare const DEFAULT_QUALITY_CONFIG: QualityConfig;
|
|
849
|
+
|
|
850
|
+
interface AxisCheckInput {
|
|
851
|
+
sourceCode: string;
|
|
852
|
+
compiledCode: string | null;
|
|
853
|
+
contract?: DataContract;
|
|
854
|
+
originalPrompt: string;
|
|
855
|
+
/** Full classification — checks may read sibling axes. */
|
|
856
|
+
classification: Classification;
|
|
857
|
+
/**
|
|
858
|
+
* Which triad produced the source. Checks that assert a design-package
|
|
859
|
+
* component (e.g. `layout.multi_step.stepper_adopted`) read it and
|
|
860
|
+
* stand down in `free` mode, where the primitive is optional. Absent
|
|
861
|
+
* means `constrained`.
|
|
862
|
+
*/
|
|
863
|
+
designMode?: DesignMode;
|
|
864
|
+
/**
|
|
865
|
+
* The rendering canvas the source is judged FOR (ggui#1117). A width cap on
|
|
866
|
+
* the outermost element is a defect on a fullscreen canvas and a no-op in a
|
|
867
|
+
* chat bubble, so a check that reads width must know the surface. Populated
|
|
868
|
+
* where the harness is built (`Harness.canvas`); ABSENT for callers with no
|
|
869
|
+
* rendering context — a check handles `undefined`, it never assumes a
|
|
870
|
+
* default (a silent `lg` would be the same guess with a nicer face).
|
|
871
|
+
*/
|
|
872
|
+
canvas?: CanvasClass;
|
|
873
|
+
/**
|
|
874
|
+
* The props the probe and the judge RENDERED the source with (ggui#1285) — a bootstrap order's
|
|
875
|
+
* `sampleProps`, the bench commit's fixture. They are judging data, not the request: a literal in
|
|
876
|
+
* the source that equals one of these values, and that the request never stated, is baked copy a
|
|
877
|
+
* visitor sees whenever the agent leaves the prop out. Populated on the served loop and at
|
|
878
|
+
* auto-commit; ABSENT where the caller has none — the check stands down, it never guesses.
|
|
879
|
+
*/
|
|
880
|
+
fixtureProps?: JsonObject;
|
|
881
|
+
}
|
|
882
|
+
/**
|
|
883
|
+
* A gated check. Runs only when the classification's axis value matches
|
|
884
|
+
* one of the gate's accepted values. Multiple gates (implicit AND) support
|
|
885
|
+
* cross-axis combinations.
|
|
886
|
+
*/
|
|
887
|
+
interface AxisCheck {
|
|
888
|
+
/** Stable id used in issue subcategories. */
|
|
889
|
+
id: string;
|
|
890
|
+
/** Primary axis this check gates on. */
|
|
891
|
+
axis: AxisKey;
|
|
892
|
+
/** Which values of that axis activate the check. */
|
|
893
|
+
values: readonly string[];
|
|
894
|
+
/** Optional extra gate on a sibling axis (e.g., only when state=merge AND writes=per-item). */
|
|
895
|
+
and?: {
|
|
896
|
+
axis: AxisKey;
|
|
897
|
+
values: readonly string[];
|
|
898
|
+
};
|
|
899
|
+
/** Execute the check and emit zero or more issues. */
|
|
900
|
+
run(input: AxisCheckInput): EvalIssue[];
|
|
901
|
+
}
|
|
902
|
+
/**
|
|
903
|
+
* Whether a check's gate(s) match the given axis vector. Pure function,
|
|
904
|
+
* no issue emission.
|
|
905
|
+
*/
|
|
906
|
+
declare function matches(vector: AxisVector, check: AxisCheck): boolean;
|
|
907
|
+
/**
|
|
908
|
+
* Map a tier-0 issue category to the canonical P0/P1/P2 priority.
|
|
909
|
+
*
|
|
910
|
+
* Sourced from the criteria priority assignments:
|
|
911
|
+
* - P0 (must): compile, security, imports, contract, types, mode
|
|
912
|
+
* - P1 (safety): tokens, crash, functionality
|
|
913
|
+
* - P2 (quality): interactivity, accessibility, layout, loading, visual
|
|
914
|
+
*/
|
|
915
|
+
declare function priorityForIssue(category: EvalCategory): Priority;
|
|
916
|
+
/** Whether any issue blocks shipping (has result = 'fail'). */
|
|
917
|
+
declare function isBlocked(result: EvalResult): boolean;
|
|
918
|
+
/**
|
|
919
|
+
* Return the issues the agent should act on, depending on quality mode.
|
|
920
|
+
* - fast: only fails (blocking issues)
|
|
921
|
+
* - auto-improve / high-quality: fails + warns
|
|
922
|
+
*/
|
|
923
|
+
declare function getActionableIssues(result: EvalResult, mode: QualityMode): EvalIssue[];
|
|
924
|
+
interface EvalCriterion {
|
|
925
|
+
/** Unique identifier matching the eval tool name (e.g., 'functionality') */
|
|
926
|
+
id: string;
|
|
927
|
+
/** Human-readable name shown to both agents */
|
|
928
|
+
name: string;
|
|
929
|
+
/** Priority tier — P0 must, P1 should, P2 nice */
|
|
930
|
+
priority: Priority;
|
|
931
|
+
/** What the coding agent should DO to satisfy this criterion */
|
|
932
|
+
codingGuidance: string;
|
|
933
|
+
/** What the eval agent should CHECK — the evaluation prompt */
|
|
934
|
+
evalInstruction: string;
|
|
935
|
+
/** Tier 0 = programmatic, Tier 1 = LLM critical, Tier 2 = LLM quality */
|
|
936
|
+
tier: 0 | 1 | 2;
|
|
937
|
+
/** Eval outcome when this criterion fails */
|
|
938
|
+
failOutcome: "fail" | "warn";
|
|
939
|
+
}
|
|
940
|
+
declare const CRITERIA: readonly EvalCriterion[];
|
|
941
|
+
/**
|
|
942
|
+
* The criteria registry for a design mode. `constrained` IS `CRITERIA`
|
|
943
|
+
* (same array, byte-identical summary); `free` swaps in the two
|
|
944
|
+
* overrides above by id, preserving order.
|
|
945
|
+
*/
|
|
946
|
+
declare function criteriaFor(designMode?: DesignMode): readonly EvalCriterion[];
|
|
947
|
+
/** Get all criteria for a specific priority level */
|
|
948
|
+
declare function getCriteriaByPriority(priority: Priority, designMode?: DesignMode): EvalCriterion[];
|
|
949
|
+
/** Get a specific criterion by ID */
|
|
950
|
+
declare function getCriterionById(id: string, designMode?: DesignMode): EvalCriterion | undefined;
|
|
951
|
+
/** Get all LLM-evaluated criteria (tier 1 + 2) */
|
|
952
|
+
declare function getLLMCriteria(designMode?: DesignMode): EvalCriterion[];
|
|
953
|
+
/**
|
|
954
|
+
* Build the coding agent's criteria summary from the single source of truth.
|
|
955
|
+
* Grouped by priority for the P0→P1→P2 hierarchy.
|
|
956
|
+
*/
|
|
957
|
+
declare function buildCodingCriteriaSummary(designMode?: DesignMode): string;
|
|
958
|
+
|
|
959
|
+
export { type AxisCheck as A, type EvalOutcome as B, type CriteriaAnswerCounts as C, DEFAULT_QUALITY_CONFIG as D, type EvalIssue as E, type HostPresentationIgnoredReason as F, LLM_EVAL_STATIC_CRITERIA as G, type HostInlineFrame as H, type Priority as I, type JudgeSalvageCounts as J, type QualityMode as K, type LLMResponse as L, buildCodingCriteriaSummary as M, criteriaFor as N, getActionableIssues as O, type ProviderRetryInfo as P, type QualityConfig as Q, type RuntimeProbeStatus as R, getCriteriaByPriority as S, getCriterionById as T, getLLMCriteria as U, type VisionProvider as V, isBlocked as W, notApplicableCoverage as X, priorityForIssue as Y, type AxisCheckInput as a, type AgentConfig as b, callLLM as c, createAgent as d, type EvalResult as e, type ProbeHostLoad as f, type ProbePendingAffordance as g, type CriteriaContext as h, type CriteriaChroma as i, type CanvasJudgeRecord as j, type CriteriaBlock as k, type CanvasPresentationOutcome as l, matches as m, type CanvasHostPresentation as n, type VisualEvalSummary as o, type VisualCoverage as p, type EvalTier as q, type CriteriaSelection as r, type VisionFinishReason as s, CRITERIA as t, type CanvasVisualSummary as u, type ContractFeedbackRecord as v, type CriterionCoverage as w, type CriterionRunStatus as x, type EvalCategory as y, type EvalCriterion as z };
|