@ggui-ai/ui-gen 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter-Cg_rkpGq.d.ts +201 -0
- package/dist/adapters/generation-dispatch.d.ts +58 -15
- package/dist/adapters/generation-dispatch.js +1418 -180
- package/dist/adapters/generation-dispatch.js.map +1 -1
- package/dist/adapters/index.js +3 -1
- package/dist/adapters/index.js.map +1 -1
- package/dist/advanced/index.d.ts +12 -8
- package/dist/advanced/index.js +1467 -239
- package/dist/advanced/index.js.map +1 -1
- package/dist/{axes-CzLEMDeB.d.ts → axes-CKVCzR_x.d.ts} +1 -1
- package/dist/bank-CT4eEkta.d.ts +118 -0
- package/dist/blueprint-validator.js +153 -59
- package/dist/blueprint-validator.js.map +1 -1
- package/dist/boilerplate.js +24 -12
- package/dist/boilerplate.js.map +1 -1
- package/dist/check/index.d.ts +12 -4
- package/dist/check/index.js +94 -2
- package/dist/check/index.js.map +1 -1
- package/dist/classifier/index.d.ts +2 -2
- package/dist/compose.d.ts +2 -2
- package/dist/{create-ui-generator-D0kbWwln.d.ts → create-ui-generator-DFnnArrv.d.ts} +43 -11
- package/dist/{dispatch-wxaIMwZt.d.ts → dispatch-CStl9eAw.d.ts} +5 -3
- package/dist/evaluation/axis-checks/index.d.ts +6 -5
- package/dist/evaluation/axis-checks/index.js +90 -2
- package/dist/evaluation/axis-checks/index.js.map +1 -1
- package/dist/evaluation/axis-checks/registry.d.ts +4 -3
- package/dist/evaluation/axis-checks/registry.js +88 -1
- package/dist/evaluation/axis-checks/registry.js.map +1 -1
- package/dist/evaluation/index.d.ts +301 -23
- package/dist/evaluation/index.js +746 -30
- package/dist/evaluation/index.js.map +1 -1
- package/dist/evaluation/types.d.ts +9 -1
- package/dist/evaluation/types.js.map +1 -1
- package/dist/fragments/index.d.ts +3 -3
- package/dist/harness/check/runtime-render/index.d.ts +30 -11
- package/dist/harness/check/runtime-render/index.js +203 -60
- package/dist/harness/check/runtime-render/index.js.map +1 -1
- package/dist/harness/check/runtime-render/render-check-worker.d.ts +8 -1
- package/dist/harness/check/runtime-render/render-check-worker.js +72 -32
- package/dist/harness/check/runtime-render/render-check-worker.js.map +1 -1
- package/dist/harness/index.d.ts +5 -4
- package/dist/harness/index.js +33 -15
- package/dist/harness/index.js.map +1 -1
- package/dist/harness/prompts.js.map +1 -1
- package/dist/harness/result-types.d.ts +4 -3
- package/dist/harness/runtime.d.ts +11 -4
- package/dist/harness/runtime.js +26 -14
- package/dist/harness/runtime.js.map +1 -1
- package/dist/harness/types-public.d.ts +13 -4
- package/dist/harness/types-public.js +33 -15
- package/dist/harness/types-public.js.map +1 -1
- package/dist/harness/validator-trace-sink.d.ts +4 -3
- package/dist/hash.d.ts +1 -1
- package/dist/index.d.ts +59 -8
- package/dist/index.js +1476 -241
- package/dist/index.js.map +1 -1
- package/dist/{llm-router-DwwUK1bz.d.ts → llm-router-nfO7wPec.d.ts} +3 -1
- package/dist/patch.d.ts +4 -3
- package/dist/policy.d.ts +8 -5
- package/dist/policy.js.map +1 -1
- package/dist/tools/get-primitives-ts.d.ts +1 -1
- package/dist/tools/get-primitives-ts.js +1 -1
- package/dist/tools/get-primitives-ts.js.map +1 -1
- package/dist/tools/render-check-worker.d.ts +7 -1
- package/dist/tools/render-check-worker.js +55 -23
- package/dist/tools/render-check-worker.js.map +1 -1
- package/dist/{types-BOvHNG7K.d.ts → types-Dh1s4K9j.d.ts} +1 -1
- package/dist/{types-public-Ow-tNtFE.d.ts → types-public-BJTLpzWn.d.ts} +208 -15
- package/dist/validation/index.d.ts +1 -1
- package/dist/validation/index.js +1 -1
- package/dist/validation/index.js.map +1 -1
- package/dist/workflows.d.ts +4 -3
- package/package.json +12 -12
- package/src/adapters/generation-dispatch.ts +173 -98
- package/src/adapters/tools.ts +2 -1
- package/src/boilerplate/free-design-prompt.ts +1 -1
- package/src/boilerplate/generate.ts +26 -1
- package/src/boilerplate/hard-sections.ts +13 -13
- package/src/boilerplate/system-prompt.ts +1 -1
- package/src/check/index.ts +1 -0
- package/src/check/run-tier0.ts +80 -1
- package/src/check/sample-props.ts +63 -0
- package/src/coding-agent/tools.ts +4 -1
- package/src/create-harness.ts +3 -3
- package/src/create-ui-generator.ts +204 -95
- package/src/evaluation/__fixtures__/png.ts +100 -0
- package/src/evaluation/axis-checks/checks/state-merge.ts +1 -1
- package/src/evaluation/axis-checks/checks/universal.ts +111 -0
- package/src/evaluation/axis-checks/dispatch.ts +4 -1
- package/src/evaluation/criteria/bank.ts +85 -0
- package/src/evaluation/criteria/context.ts +52 -0
- package/src/evaluation/criteria/resolve.ts +159 -0
- package/src/evaluation/criteria/select.ts +79 -0
- package/src/evaluation/index.ts +23 -0
- package/src/evaluation/ink-extent.ts +200 -0
- package/src/evaluation/theme-css.ts +12 -5
- package/src/evaluation/types-public.ts +200 -11
- package/src/evaluation/types.ts +9 -0
- package/src/evaluation/visual-evaluator.ts +636 -30
- package/src/harness/check/runtime-render/adapter.ts +221 -75
- package/src/harness/check/runtime-render/index.ts +75 -2
- package/src/harness/check/runtime-render/render-check-host.ts +82 -19
- package/src/harness/check/runtime-render/render-check-worker.ts +83 -29
- package/src/harness/check/runtime-render/render-check.ts +154 -117
- package/src/harness/coding/generate-task-runner.ts +60 -4
- package/src/harness/coding/init-session.ts +13 -0
- package/src/harness/coding/run-coding-turn.ts +7 -3
- package/src/harness/coding/run-eval-round.ts +257 -15
- package/src/harness/policy.ts +11 -0
- package/src/harness/primitive-slice.ts +104 -0
- package/src/harness/runtime.ts +5 -0
- package/src/harness/types-public.ts +9 -1
- package/src/harness-tool-names.ts +38 -0
- package/src/index.ts +16 -0
- package/src/policy.ts +7 -4
- package/src/run-check.ts +2 -0
- package/src/tools/get-primitives-ts.ts +1 -1
- package/src/tools/get-wire.ts +1 -1
- package/src/tools/render-check-worker.ts +74 -23
- package/src/tools/render-check.ts +0 -60
- package/src/tools.ts +11 -1
- package/src/validation/primitives.ts +1 -1
- package/dist/render-check-Blwj8gB6.d.ts +0 -105
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
import type { Classification } from "../../classifier/index.js";
|
|
14
14
|
import type { JsonValue } from "@ggui-ai/protocol";
|
|
15
|
-
import type { ContractFeedbackRecord, EvalResult } from "../../evaluation/types-public.js";
|
|
15
|
+
import type { ContractFeedbackRecord, EvalResult, RuntimeProbeMeta } from "../../evaluation/types-public.js";
|
|
16
16
|
import type { Harness, Task, TaskContext } from "../types-public.js";
|
|
17
17
|
import type { SingleComponentParams } from "../runtime.js";
|
|
18
18
|
import type { TaskRunner } from "../index.js";
|
|
@@ -77,6 +77,16 @@ export interface GenerateTelemetry {
|
|
|
77
77
|
cacheCreationTokens?: number;
|
|
78
78
|
/** Latest evalResult — undefined when eval was disabled or never ran. */
|
|
79
79
|
evalResult: EvalResult | undefined;
|
|
80
|
+
/**
|
|
81
|
+
* ggui#1380 — whether the probe-only lane's one repair turn has been
|
|
82
|
+
* spent. Set when the first probe-only round fed a recoverable render
|
|
83
|
+
* crash back; read by the next round (which then always breaks) and by
|
|
84
|
+
* the turn after the repair (which never loops on a failed self-check).
|
|
85
|
+
* Stays `false` on every other lane.
|
|
86
|
+
*/
|
|
87
|
+
probeRepairUsed: boolean;
|
|
88
|
+
/** ggui#1380 C2b — the probe that bought the repair turn (the first probe-only round's meta), recorded on round 2 as `trigger`. */
|
|
89
|
+
probeRepairTrigger: RuntimeProbeMeta | undefined;
|
|
80
90
|
/**
|
|
81
91
|
* ggui#1261 — the contract-feedback round, once one fired: kept across the
|
|
82
92
|
* rounds after it (each rebuilds `evalResult`) so the assembled result can
|
|
@@ -126,6 +136,8 @@ export function createTelemetry(): GenerateTelemetry {
|
|
|
126
136
|
totalIn: 0,
|
|
127
137
|
totalOut: 0,
|
|
128
138
|
evalResult: undefined,
|
|
139
|
+
probeRepairUsed: false,
|
|
140
|
+
probeRepairTrigger: undefined,
|
|
129
141
|
contractFeedback: undefined,
|
|
130
142
|
sameExchangeBreak: undefined,
|
|
131
143
|
compiledCode: "",
|
|
@@ -248,6 +260,8 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
|
|
|
248
260
|
originalPrompt: params.userPrompt,
|
|
249
261
|
commitMeta: session.commitMeta,
|
|
250
262
|
originalProps: session.originalProps,
|
|
263
|
+
// ggui#1285: the fixture the probe and judge render with, for the echo check at auto-commit.
|
|
264
|
+
...(params.fixtureProps !== undefined ? { fixtureProps: params.fixtureProps } : {}),
|
|
251
265
|
costTracker: session.costTracker,
|
|
252
266
|
// Thread the dispatch-resolved policy so experimental
|
|
253
267
|
// profiles take effect on both preflight and tier-0 retry
|
|
@@ -274,6 +288,22 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
|
|
|
274
288
|
iconNamesCache = turn.iconNamesCache;
|
|
275
289
|
preWarmedContext = turn.preWarmedContext;
|
|
276
290
|
|
|
291
|
+
// ggui#1380 — the probe-only lane's repair turn did not produce a
|
|
292
|
+
// self-check-passing compile (a failed self-check, or no tool call):
|
|
293
|
+
// the lane grants AT MOST ONE eval-fix turn, so the loop never feeds
|
|
294
|
+
// the self-check violations back for another. The pre-repair card is
|
|
295
|
+
// what ships (`compiledCode` / `pairedSource` are untouched since the
|
|
296
|
+
// turn that compiled it), and the round's result records the repair
|
|
297
|
+
// as attempted but not compiled.
|
|
298
|
+
if (turn.control !== "proceed" && session.probeOnlyEnabled && telemetry.probeRepairUsed) {
|
|
299
|
+
if (telemetry.evalResult !== undefined) {
|
|
300
|
+
telemetry.evalResult = { ...telemetry.evalResult, runtimeProbeRepair: { attempted: true, compiled: false } };
|
|
301
|
+
}
|
|
302
|
+
console.log(
|
|
303
|
+
`[simple] probe-only repair turn did not compile (${turn.outcome ?? "no tool call"}) — serving the pre-repair card`,
|
|
304
|
+
);
|
|
305
|
+
break;
|
|
306
|
+
}
|
|
277
307
|
if (turn.control === "break") break;
|
|
278
308
|
if (turn.control === "continue") {
|
|
279
309
|
lastResultText = turn.lastResultText;
|
|
@@ -326,11 +356,14 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
|
|
|
326
356
|
}
|
|
327
357
|
|
|
328
358
|
// ── Eval round ──
|
|
359
|
+
// Two lanes open the gate: the evaluation lane (its modules loaded, an
|
|
360
|
+
// evaluator configured) and the probe-only lane (ggui#1380: the probe
|
|
361
|
+
// wired, no evaluator — the round runs the probe once and nothing else).
|
|
329
362
|
if (
|
|
330
363
|
!evalDone &&
|
|
331
364
|
telemetry.compiledCode &&
|
|
332
|
-
session.tiersMod &&
|
|
333
|
-
|
|
365
|
+
((session.tiersMod && (session.codeEvalEnabled || session.visualEvalEnabled)) ||
|
|
366
|
+
session.probeOnlyEnabled)
|
|
334
367
|
) {
|
|
335
368
|
// Measure eval-round wall-clock here (not inside runEvalRound) so the
|
|
336
369
|
// accumulator covers axis-checks + LLM+visual parallel + merge +
|
|
@@ -348,14 +381,18 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
|
|
|
348
381
|
evaluationAgent: session.agents.evaluation,
|
|
349
382
|
visualEvalAgent: session.agents.visualEval,
|
|
350
383
|
visualEvaluation: params.visualEvaluation,
|
|
384
|
+
shellType: params.shellType,
|
|
351
385
|
visualThreshold: session.visualThreshold,
|
|
352
386
|
qualityMode: session.qualityMode,
|
|
353
387
|
maxEvalRounds: session.maxEvalRounds,
|
|
354
|
-
costTracker: session.costTracker
|
|
388
|
+
costTracker: session.costTracker,
|
|
355
389
|
llmEvalMod: session.llmEvalMod,
|
|
356
390
|
visualMod: session.visualMod,
|
|
357
391
|
preWarmPromise: session.preWarmPromise,
|
|
358
392
|
onProgress: params.onProgress,
|
|
393
|
+
probeOnly: session.probeOnlyEnabled,
|
|
394
|
+
probeRepairUsed: telemetry.probeRepairUsed,
|
|
395
|
+
...(telemetry.probeRepairTrigger !== undefined ? { probeRepairTrigger: telemetry.probeRepairTrigger } : {}),
|
|
359
396
|
},
|
|
360
397
|
{
|
|
361
398
|
compiledCode: telemetry.compiledCode,
|
|
@@ -379,6 +416,11 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
|
|
|
379
416
|
|
|
380
417
|
if (round.control === "break") break;
|
|
381
418
|
// round.control === "feedback" — set next coding turn input.
|
|
419
|
+
// On the probe-only lane this is the one repair turn (ggui#1380).
|
|
420
|
+
if (session.probeOnlyEnabled) {
|
|
421
|
+
telemetry.probeRepairUsed = true;
|
|
422
|
+
telemetry.probeRepairTrigger = round.evalResult?.runtimeProbe;
|
|
423
|
+
}
|
|
382
424
|
lastResultText = round.lastResultText;
|
|
383
425
|
isEvalFeedback = round.isEvalFeedback;
|
|
384
426
|
lastDiffFailed = round.lastDiffFailed;
|
|
@@ -388,6 +430,20 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
|
|
|
388
430
|
}
|
|
389
431
|
} while (telemetry.turnsUsed < maxTurns && !evalDone);
|
|
390
432
|
|
|
433
|
+
// ggui#1380 — the probe-only lane's one repair turn was granted by the
|
|
434
|
+
// round but the turn cap had already been reached, so it was never taken:
|
|
435
|
+
// the card ships as it was, the probe's record stays, and no repair record
|
|
436
|
+
// is stamped (nothing was attempted). Say so, so the absence reads as the
|
|
437
|
+
// cap and not as a repair that vanished.
|
|
438
|
+
if (
|
|
439
|
+
session.probeOnlyEnabled &&
|
|
440
|
+
telemetry.probeRepairUsed &&
|
|
441
|
+
telemetry.evalResult !== undefined &&
|
|
442
|
+
telemetry.evalResult.runtimeProbeRepair === undefined
|
|
443
|
+
) {
|
|
444
|
+
console.log(`[simple] probe-only repair turn not taken — turn cap (${maxTurns}) reached before it`);
|
|
445
|
+
}
|
|
446
|
+
|
|
391
447
|
telemetry.codingMs = Date.now() - telemetry.codingStartedAtMs;
|
|
392
448
|
|
|
393
449
|
// ── Fallback: pull compiled code from commitMeta if loop exited dry ──
|
|
@@ -49,6 +49,16 @@ export interface CodingSession {
|
|
|
49
49
|
// Eval gate config (computed once from params/qualityConfig).
|
|
50
50
|
readonly codeEvalEnabled: boolean;
|
|
51
51
|
readonly visualEvalEnabled: boolean;
|
|
52
|
+
/**
|
|
53
|
+
* ggui#1380 — the probe-only lane: the harness wires the runtime-render
|
|
54
|
+
* check and NO evaluator is configured (neither code nor visual). A
|
|
55
|
+
* serving deployment runs the probe once after the coding turns, observes,
|
|
56
|
+
* and buys one repair turn on a recoverable render crash. `evaluation:
|
|
57
|
+
* { enabled: false }` beside a wired probe is this lane too: a config
|
|
58
|
+
* that enables nothing is no evaluator. False whenever an evaluator runs —
|
|
59
|
+
* that lane's exit probe is a different round — and false without a probe.
|
|
60
|
+
*/
|
|
61
|
+
readonly probeOnlyEnabled: boolean;
|
|
52
62
|
readonly maxEvalRounds: number;
|
|
53
63
|
readonly visualThreshold: number;
|
|
54
64
|
readonly qualityMode: "fast" | "auto-improve" | "high-quality";
|
|
@@ -194,6 +204,8 @@ export async function initSession(input: {
|
|
|
194
204
|
const qualityMode = qualityConfig?.quality ?? "fast";
|
|
195
205
|
const codeEvalEnabled = !!params.evaluation?.enabled;
|
|
196
206
|
const visualEvalEnabled = !!(params.visualEvaluation?.enabled || qualityConfig?.visualEval);
|
|
207
|
+
const probeOnlyEnabled =
|
|
208
|
+
harness.check.runtimeRender !== undefined && !codeEvalEnabled && !visualEvalEnabled;
|
|
197
209
|
const maxEvalRounds =
|
|
198
210
|
qualityMode === "high-quality"
|
|
199
211
|
? (params.evaluation?.maxRounds ?? 3)
|
|
@@ -230,6 +242,7 @@ export async function initSession(input: {
|
|
|
230
242
|
systemPrompt,
|
|
231
243
|
codeEvalEnabled,
|
|
232
244
|
visualEvalEnabled,
|
|
245
|
+
probeOnlyEnabled,
|
|
233
246
|
maxEvalRounds,
|
|
234
247
|
visualThreshold,
|
|
235
248
|
qualityMode,
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
// Behavior MUST match the inlined original byte-for-byte — this is a
|
|
15
15
|
// mechanical extraction, not a redesign.
|
|
16
16
|
|
|
17
|
-
import type { DataContract } from "@ggui-ai/protocol";
|
|
17
|
+
import type { DataContract, JsonObject } from "@ggui-ai/protocol";
|
|
18
18
|
import type { CommitMetadata } from "../../coding-agent/types.js";
|
|
19
19
|
import type { AgentWorkspace } from "../../coding-agent/workspace.js";
|
|
20
20
|
import { executeTool } from "../../coding-agent/tools.js";
|
|
@@ -30,9 +30,10 @@ import {
|
|
|
30
30
|
APPLY_CHANGES_TOOL_SCOPED,
|
|
31
31
|
GET_COMPONENTS_INFO_TOOL,
|
|
32
32
|
GET_ICONS_TOOL,
|
|
33
|
+
ICON_LIST_NO_MATCH_GUIDANCE,
|
|
33
34
|
REWRITE_TOOL,
|
|
34
|
-
WRITE_PLAN_TOOL,
|
|
35
35
|
type LLMToolDef,
|
|
36
|
+
WRITE_PLAN_TOOL,
|
|
36
37
|
} from "../../tools.js";
|
|
37
38
|
import { LUCIDE_ICON_NAMES } from "@ggui-ai/design";
|
|
38
39
|
import { formatWithHashlines } from "../hashline.js";
|
|
@@ -165,6 +166,8 @@ export interface CodingTurnContext {
|
|
|
165
166
|
readonly originalPrompt: string;
|
|
166
167
|
readonly commitMeta: Map<string, CommitMetadata>;
|
|
167
168
|
readonly originalProps: string | undefined;
|
|
169
|
+
/** The props the probe and judge render with (ggui#1285) — handed to auto-commit's axis checks; absent = the echo check stands down. */
|
|
170
|
+
readonly fixtureProps?: JsonObject;
|
|
168
171
|
readonly costTracker: CostTracker | null;
|
|
169
172
|
/**
|
|
170
173
|
* Runtime-resolved context policy. Defaults to `harness.policy.context`
|
|
@@ -614,7 +617,7 @@ ${closingInstruction}`;
|
|
|
614
617
|
iconNamesCache = LUCIDE_ICON_NAMES.join(", ");
|
|
615
618
|
}
|
|
616
619
|
console.log(`[simple] turn ${turnsUsed}: get_available_icons`);
|
|
617
|
-
let iconResult = `Available Lucide icon names (use with <Icon name="...">):\n${iconNamesCache}`;
|
|
620
|
+
let iconResult = `Available Lucide icon names (use with <Icon name="...">):\n${iconNamesCache}\n\n${ICON_LIST_NO_MATCH_GUIDANCE}`;
|
|
618
621
|
// ggui#404 same-call circuit breaker. The icon list is constant by
|
|
619
622
|
// construction, so every repeat is an identical exchange — record on
|
|
620
623
|
// the BASE text (pre-nudge) so the chain keys stay stable across the
|
|
@@ -690,6 +693,7 @@ ${closingInstruction}`;
|
|
|
690
693
|
classification: harness.classification,
|
|
691
694
|
originalPrompt: ctx.originalPrompt,
|
|
692
695
|
...(harness.canvas !== undefined ? { canvas: harness.canvas } : {}),
|
|
696
|
+
...(ctx.fixtureProps !== undefined ? { fixtureProps: ctx.fixtureProps } : {}),
|
|
693
697
|
},
|
|
694
698
|
);
|
|
695
699
|
const toolMs = Date.now() - toolStart;
|
|
@@ -10,24 +10,41 @@
|
|
|
10
10
|
// (capped at MAX_FEEDBACK_ISSUES per turn).
|
|
11
11
|
//
|
|
12
12
|
// Does NOT own: the do/while wrapper, the eligibility gate at the call
|
|
13
|
-
// site (`!evalDone && compiledCode && tiersMod && (codeEval || visualEval)
|
|
14
|
-
//
|
|
15
|
-
//
|
|
13
|
+
// site (`!evalDone && compiledCode && ((tiersMod && (codeEval || visualEval))
|
|
14
|
+
// || probeOnlyEnabled)` — the evaluation lane, or the probe-only lane a
|
|
15
|
+
// serving deployment runs when it wires the runtime probe without an
|
|
16
|
+
// evaluator, ggui#1380), or the threading of mutable state — caller still
|
|
17
|
+
// owns those for now and the closure migration (step 3) will move them too.
|
|
16
18
|
//
|
|
17
19
|
// Behavior MUST match the inlined original byte-for-byte — this is a
|
|
18
20
|
// mechanical extraction, not a redesign.
|
|
19
21
|
|
|
20
22
|
import type { DataContract, JsonObject } from "@ggui-ai/protocol";
|
|
23
|
+
import { criteriaContextFor } from "../../evaluation/criteria/context.js";
|
|
21
24
|
import { listContractGadgets } from "@ggui-ai/protocol";
|
|
22
25
|
import type { Classification } from "../../classifier/index.js";
|
|
23
26
|
import type { AgentWorkspace } from "../../coding-agent/workspace.js";
|
|
24
27
|
import type { CostTracker } from "../../evaluation/cost-tracker.js";
|
|
25
|
-
import type {
|
|
28
|
+
import type {
|
|
29
|
+
ContractFeedbackRecord,
|
|
30
|
+
EvalIssue,
|
|
31
|
+
EvalResult,
|
|
32
|
+
RuntimeProbeMeta,
|
|
33
|
+
RuntimeProbeMetaDetail,
|
|
34
|
+
RuntimeProbeRepair,
|
|
35
|
+
VisualEvalSummary,
|
|
36
|
+
VisualCoverage,
|
|
37
|
+
} from "../../evaluation/types-public.js";
|
|
26
38
|
import { notApplicableCoverage } from "../../evaluation/types-public.js";
|
|
27
39
|
import { mapProviderForEvaluator, visionJudgeProvider } from "../enforced-coding.js";
|
|
28
40
|
import { runCheck } from "../index.js";
|
|
29
|
-
import {
|
|
30
|
-
|
|
41
|
+
import {
|
|
42
|
+
RENDER_CHECK_KINDS,
|
|
43
|
+
isRecoverableRenderCrash,
|
|
44
|
+
parseRuntimeSubcategory,
|
|
45
|
+
type RenderCheckKind,
|
|
46
|
+
} from "../check/runtime-render/index.js";
|
|
47
|
+
import type { Harness, RuntimeRenderOutcome } from "../types-public.js";
|
|
31
48
|
import type { AgentSpec, SingleComponentParams } from "../runtime.js";
|
|
32
49
|
import type {
|
|
33
50
|
CanvasFitReading,
|
|
@@ -58,15 +75,48 @@ export interface EvalRoundContext {
|
|
|
58
75
|
readonly evaluationAgent: AgentSpec;
|
|
59
76
|
readonly visualEvalAgent: AgentSpec;
|
|
60
77
|
readonly visualEvaluation: SingleComponentParams["visualEvaluation"];
|
|
78
|
+
/** ggui#1436 — the shell as the caller named it, for the criteria selector's context; absent reads `unknown`. */
|
|
79
|
+
readonly shellType?: SingleComponentParams["shellType"];
|
|
61
80
|
readonly visualThreshold: number;
|
|
62
81
|
readonly qualityMode: "fast" | "auto-improve" | "high-quality";
|
|
63
82
|
readonly maxEvalRounds: number;
|
|
64
|
-
/**
|
|
65
|
-
|
|
83
|
+
/**
|
|
84
|
+
* The evaluation lane's cost tracker. `null` on the probe-only lane, which
|
|
85
|
+
* runs no evaluator and records no cost; an evaluation round without one
|
|
86
|
+
* lands on the round's infra-skipped stamp (ggui#1110) with the reason
|
|
87
|
+
* {@link COST_TRACKER_ABSENT_REASON} — never a dereference.
|
|
88
|
+
*/
|
|
89
|
+
readonly costTracker: CostTracker | null;
|
|
66
90
|
readonly llmEvalMod: LlmEvalMod | null;
|
|
67
91
|
readonly visualMod: VisualEvalMod | null;
|
|
68
92
|
readonly preWarmPromise: Promise<PreWarmedEvalContext | null> | undefined;
|
|
69
93
|
readonly onProgress?: (event: unknown) => void;
|
|
94
|
+
/**
|
|
95
|
+
* ggui#1380 — the probe-only lane: a serving deployment that wires the
|
|
96
|
+
* runtime probe and configures NO evaluator. The round runs the probe
|
|
97
|
+
* once and nothing else (no axis checks — the deterministic checks already
|
|
98
|
+
* ran in the coding turns' self-check — no evaluator, no visual leg).
|
|
99
|
+
* REQUIRED and explicit: it is never inferred from null modules, because the
|
|
100
|
+
* evaluation lane with null modules is a different round (the evaluation
|
|
101
|
+
* lane's own exit probe) and must stay one.
|
|
102
|
+
*/
|
|
103
|
+
readonly probeOnly: boolean;
|
|
104
|
+
/**
|
|
105
|
+
* ggui#1380 — whether this generation already spent its one repair turn.
|
|
106
|
+
* `false` on the first probe-only round: a recoverable render crash there
|
|
107
|
+
* buys the turn. `true` on the second: the round breaks whatever the probe
|
|
108
|
+
* reads and records the repair with the probe that bought it as `trigger`.
|
|
109
|
+
* Meaningful on the probe-only lane only.
|
|
110
|
+
*/
|
|
111
|
+
readonly probeRepairUsed: boolean;
|
|
112
|
+
/**
|
|
113
|
+
* ggui#1380 C2b — on the second probe-only round, the probe that bought the
|
|
114
|
+
* repair turn (the first round's meta), recorded as `runtimeProbeRepair.trigger`
|
|
115
|
+
* so the crash that happened is never lost behind the re-probe. The runner
|
|
116
|
+
* sets it together with `probeRepairUsed`; a re-probe round without it is a
|
|
117
|
+
* harness defect.
|
|
118
|
+
*/
|
|
119
|
+
readonly probeRepairTrigger?: RuntimeProbeMeta;
|
|
70
120
|
}
|
|
71
121
|
|
|
72
122
|
/**
|
|
@@ -177,6 +227,29 @@ function formatFitFeedback(issue: EvalIssue): string {
|
|
|
177
227
|
/** Why an eval result carries no criteria: the round itself threw (ggui#1110). */
|
|
178
228
|
const EVAL_ROUND_THREW_REASON = "eval round threw";
|
|
179
229
|
|
|
230
|
+
/**
|
|
231
|
+
* ggui#1380 — the coverage reasons a probe-only round stamps: no evaluator
|
|
232
|
+
* and no visual leg are configured, BY DESIGN, so every criterion is
|
|
233
|
+
* `not-applicable` (never `skipped`, which would read as a silent absence).
|
|
234
|
+
*/
|
|
235
|
+
const PROBE_ONLY_CRITERIA_REASON = "probe-only round: no evaluator configured";
|
|
236
|
+
const PROBE_ONLY_VISUAL_REASON = "probe-only round: no visual evaluator configured";
|
|
237
|
+
/** The `pass` entry a probe-only round records — the probe was the only gate. */
|
|
238
|
+
const PROBE_ONLY_PASS = "probe-only";
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* D3 (ggui#1380) — the evaluation lane's one named guard: a round that
|
|
242
|
+
* reaches the evaluator without a cost tracker cannot record what it spends,
|
|
243
|
+
* so it stops HERE, on the ggui#1110 infra-skipped stamp with this reason,
|
|
244
|
+
* instead of dereferencing `null` somewhere past the evaluator call.
|
|
245
|
+
*/
|
|
246
|
+
export const COST_TRACKER_ABSENT_REASON = "cost tracker absent on an evaluation round";
|
|
247
|
+
|
|
248
|
+
function requireCostTracker(costTracker: CostTracker | null): CostTracker {
|
|
249
|
+
if (costTracker === null) throw new Error(COST_TRACKER_ABSENT_REASON);
|
|
250
|
+
return costTracker;
|
|
251
|
+
}
|
|
252
|
+
|
|
180
253
|
/**
|
|
181
254
|
* The in-loop visual leg, made total (ggui#1248). The judge runs on a VISION
|
|
182
255
|
* provider: the agent's provider narrowed by `visionJudgeProvider`, never
|
|
@@ -306,6 +379,62 @@ interface ProbeAtExitResult {
|
|
|
306
379
|
readonly meta: RuntimeProbeMeta;
|
|
307
380
|
}
|
|
308
381
|
|
|
382
|
+
/**
|
|
383
|
+
* The distinct check kinds among a probe's `result: 'fail'` issues, in the
|
|
384
|
+
* order {@link RENDER_CHECK_KINDS} declares them (a render crash reads
|
|
385
|
+
* first). Parsed from the `runtime:<check>[:<subject>]` subcategory the
|
|
386
|
+
* adapter writes; a warn, an issue with no subcategory, or a subcategory that
|
|
387
|
+
* names no check contributes nothing.
|
|
388
|
+
*/
|
|
389
|
+
function failedCheckKinds(probeIssues: readonly EvalIssue[]): readonly RenderCheckKind[] {
|
|
390
|
+
const failed = new Set<RenderCheckKind>();
|
|
391
|
+
for (const issue of probeIssues) {
|
|
392
|
+
if (issue.result !== "fail" || issue.subcategory === undefined) continue;
|
|
393
|
+
const kind = parseRuntimeSubcategory(issue.subcategory);
|
|
394
|
+
if (kind !== undefined) failed.add(kind);
|
|
395
|
+
}
|
|
396
|
+
return RENDER_CHECK_KINDS.filter((kind) => failed.has(kind));
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* ONE builder for the meta every exit probe stamps (ggui#1380): the
|
|
401
|
+
* probe-only round and the evaluation lane's exit probe both go through
|
|
402
|
+
* `runProbeAtExit`, and this is where its outcome becomes a verdict. The
|
|
403
|
+
* verdict is computed from the PROBE'S OWN issues: `fail` when any check
|
|
404
|
+
* failed (`failChecks` = the distinct failing kinds), `pass` when none did;
|
|
405
|
+
* a probe that did not run to a verdict (`timed-out`, `infra-skipped`,
|
|
406
|
+
* `not-applicable`) carries no verdict key at all.
|
|
407
|
+
*
|
|
408
|
+
* The verdict is not the repair trigger. `fail` names ANY failing check;
|
|
409
|
+
* the one repair turn fires on a `render-no-throw` fail that
|
|
410
|
+
* `isRecoverableRenderCrash` recognises, and on nothing else — a `fail`
|
|
411
|
+
* verdict whose checks are `prop-sensitivity`, `action-wiring` or any other
|
|
412
|
+
* kind is recorded on the meta and never fed back.
|
|
413
|
+
*/
|
|
414
|
+
function probeMetaOf(outcome: RuntimeRenderOutcome, probeIssues: readonly EvalIssue[]): RuntimeProbeMeta {
|
|
415
|
+
const detail: RuntimeProbeMetaDetail = {
|
|
416
|
+
...(outcome.reason !== undefined ? { reason: outcome.reason } : {}),
|
|
417
|
+
...(outcome.renderMs !== undefined ? { renderMs: outcome.renderMs } : {}),
|
|
418
|
+
...(outcome.hostLoad !== undefined ? { hostLoad: outcome.hostLoad } : {}),
|
|
419
|
+
...(outcome.queuedMs !== undefined ? { queuedMs: outcome.queuedMs } : {}),
|
|
420
|
+
};
|
|
421
|
+
const elapsed = outcome.elapsedMs !== undefined ? { elapsedMs: outcome.elapsedMs } : {};
|
|
422
|
+
if (outcome.status !== "ran") {
|
|
423
|
+
return { status: outcome.status, ...elapsed, ...detail };
|
|
424
|
+
}
|
|
425
|
+
const [first, ...rest] = failedCheckKinds(probeIssues);
|
|
426
|
+
if (first === undefined) {
|
|
427
|
+
return { status: "ran", verdict: "pass", ...elapsed, ...detail };
|
|
428
|
+
}
|
|
429
|
+
return { status: "ran", verdict: "fail", failChecks: [first, ...rest], ...elapsed, ...detail };
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
/** The probe's status and, when it ran, its verdict — for the round's log line. */
|
|
433
|
+
function describeProbeMeta(meta: RuntimeProbeMeta): string {
|
|
434
|
+
if (meta.verdict === undefined) return meta.status;
|
|
435
|
+
return meta.verdict === "pass" ? "ran, verdict pass" : `ran, verdict fail (${meta.failChecks.join(", ")})`;
|
|
436
|
+
}
|
|
437
|
+
|
|
309
438
|
/**
|
|
310
439
|
* Run the runtime-render probe ONCE at an exit-decision point. Off the
|
|
311
440
|
* per-turn hot path: `runCheck` is invoked with `skipRuntimeRender: true`
|
|
@@ -356,12 +485,7 @@ async function runProbeAtExit(input: {
|
|
|
356
485
|
meta: { status: "infra-skipped", reason: message },
|
|
357
486
|
};
|
|
358
487
|
}
|
|
359
|
-
const meta
|
|
360
|
-
status: outcome.status,
|
|
361
|
-
...(outcome.reason !== undefined ? { reason: outcome.reason } : {}),
|
|
362
|
-
...(outcome.elapsedMs !== undefined ? { elapsedMs: outcome.elapsedMs } : {}),
|
|
363
|
-
...(outcome.hostLoad !== undefined ? { hostLoad: outcome.hostLoad } : {}),
|
|
364
|
-
};
|
|
488
|
+
const meta = probeMetaOf(outcome, outcome.issues);
|
|
365
489
|
if (outcome.status !== "ran") {
|
|
366
490
|
return { fired: false, recoverableFail: false, probeIssues: [], meta };
|
|
367
491
|
}
|
|
@@ -450,11 +574,13 @@ export async function runEvalRound(
|
|
|
450
574
|
visualThreshold,
|
|
451
575
|
qualityMode,
|
|
452
576
|
maxEvalRounds,
|
|
453
|
-
costTracker,
|
|
454
577
|
llmEvalMod,
|
|
455
578
|
visualMod,
|
|
456
579
|
preWarmPromise,
|
|
457
580
|
onProgress,
|
|
581
|
+
probeOnly,
|
|
582
|
+
probeRepairUsed,
|
|
583
|
+
probeRepairTrigger,
|
|
458
584
|
} = ctx;
|
|
459
585
|
const { compiledCode, prevModeSubcats, prevFailFingerprints } = input;
|
|
460
586
|
let { evalRoundsUsed, preWarmedContext } = input;
|
|
@@ -468,6 +594,102 @@ export async function runEvalRound(
|
|
|
468
594
|
let evalLlmMs = 0;
|
|
469
595
|
|
|
470
596
|
try {
|
|
597
|
+
// ── The probe-only round (ggui#1380) ──
|
|
598
|
+
// A serving deployment that wires the runtime probe and configures no
|
|
599
|
+
// evaluator: the probe runs ONCE, before anything else, and no axis
|
|
600
|
+
// check runs (the served card is the coding turns' — its deterministic
|
|
601
|
+
// checks ran in their self-check). A recoverable render crash on the
|
|
602
|
+
// first round buys exactly one repair turn; the second round breaks
|
|
603
|
+
// whatever it reads and records the repair's after. Timed-out,
|
|
604
|
+
// infra-skipped and not-applicable never block and never repair.
|
|
605
|
+
if (probeOnly) {
|
|
606
|
+
const exitProbe = await runProbeAtExit({
|
|
607
|
+
harness,
|
|
608
|
+
sourceCode: currentSource,
|
|
609
|
+
compiledCode,
|
|
610
|
+
contract,
|
|
611
|
+
fixtureProps,
|
|
612
|
+
});
|
|
613
|
+
const probeOnlyResult: EvalResult = {
|
|
614
|
+
issues: [...exitProbe.probeIssues],
|
|
615
|
+
pass: [PROBE_ONLY_PASS],
|
|
616
|
+
runtimeProbe: exitProbe.meta,
|
|
617
|
+
criteriaCoverage: notApplicableCoverage(PROBE_ONLY_CRITERIA_REASON),
|
|
618
|
+
visualCoverage: { status: "not-applicable", reason: PROBE_ONLY_VISUAL_REASON },
|
|
619
|
+
};
|
|
620
|
+
const recoverableFail = exitProbe.fired && exitProbe.recoverableFail;
|
|
621
|
+
if (probeRepairUsed) {
|
|
622
|
+
// This round IS the re-probe (a repair that did not compile never
|
|
623
|
+
// reaches a round — the runner stamps `compiled: false` itself). The
|
|
624
|
+
// top level is the re-probe; the record names the probe that bought
|
|
625
|
+
// the turn as `trigger`, so each probe is recorded exactly once.
|
|
626
|
+
if (probeRepairTrigger === undefined) {
|
|
627
|
+
throw new Error("harness defect: a probe-only re-probe round without the probe that bought the repair turn");
|
|
628
|
+
}
|
|
629
|
+
const repair: RuntimeProbeRepair = { attempted: true, compiled: true, trigger: probeRepairTrigger };
|
|
630
|
+
evalResult = { ...probeOnlyResult, runtimeProbeRepair: repair };
|
|
631
|
+
console.log(
|
|
632
|
+
`[simple] eval round ${evalRoundsUsed}: probe-only re-probe after the repair turn — ` +
|
|
633
|
+
`${describeProbeMeta(exitProbe.meta)}${recoverableFail ? ", recoverable crash still present" : ""} — serving`,
|
|
634
|
+
);
|
|
635
|
+
return {
|
|
636
|
+
control: "break",
|
|
637
|
+
evalDone: true,
|
|
638
|
+
evalResult,
|
|
639
|
+
evalRoundsUsed,
|
|
640
|
+
prevModeSubcats,
|
|
641
|
+
prevFailFingerprints,
|
|
642
|
+
preWarmedContext,
|
|
643
|
+
evalTokens,
|
|
644
|
+
evalLlmMs,
|
|
645
|
+
lastResultText: "",
|
|
646
|
+
isEvalFeedback: false,
|
|
647
|
+
lastDiffFailed: false,
|
|
648
|
+
};
|
|
649
|
+
}
|
|
650
|
+
evalResult = probeOnlyResult;
|
|
651
|
+
if (recoverableFail) {
|
|
652
|
+
const runtimeFails = exitProbe.probeIssues.filter(
|
|
653
|
+
(i) => i.result === "fail" && i.subcategory?.startsWith("runtime:"),
|
|
654
|
+
);
|
|
655
|
+
const lines = runtimeFails.slice(0, MAX_FEEDBACK_ISSUES).map(formatRuntimeProbeFeedback);
|
|
656
|
+
console.log(
|
|
657
|
+
`[simple] eval round ${evalRoundsUsed}: probe-only round — runtime probe fail (recoverable) — granting the one repair turn`,
|
|
658
|
+
);
|
|
659
|
+
return {
|
|
660
|
+
control: "feedback",
|
|
661
|
+
evalDone: false,
|
|
662
|
+
evalResult,
|
|
663
|
+
evalRoundsUsed,
|
|
664
|
+
prevModeSubcats,
|
|
665
|
+
prevFailFingerprints,
|
|
666
|
+
preWarmedContext,
|
|
667
|
+
evalTokens,
|
|
668
|
+
evalLlmMs,
|
|
669
|
+
lastResultText: lines.join("\n\n"),
|
|
670
|
+
isEvalFeedback: true,
|
|
671
|
+
lastDiffFailed: false,
|
|
672
|
+
};
|
|
673
|
+
}
|
|
674
|
+
console.log(
|
|
675
|
+
`[simple] eval round ${evalRoundsUsed}: probe-only round — probe ${describeProbeMeta(exitProbe.meta)} — serving`,
|
|
676
|
+
);
|
|
677
|
+
return {
|
|
678
|
+
control: "break",
|
|
679
|
+
evalDone: true,
|
|
680
|
+
evalResult,
|
|
681
|
+
evalRoundsUsed,
|
|
682
|
+
prevModeSubcats,
|
|
683
|
+
prevFailFingerprints,
|
|
684
|
+
preWarmedContext,
|
|
685
|
+
evalTokens,
|
|
686
|
+
evalLlmMs,
|
|
687
|
+
lastResultText: "",
|
|
688
|
+
isEvalFeedback: false,
|
|
689
|
+
lastDiffFailed: false,
|
|
690
|
+
};
|
|
691
|
+
}
|
|
692
|
+
|
|
471
693
|
// Tier 0 already ran in autoCommit — go straight to mode-checks + LLM eval.
|
|
472
694
|
|
|
473
695
|
// ── Axis-keyed deterministic checks ──────────────
|
|
@@ -582,6 +804,10 @@ export async function runEvalRound(
|
|
|
582
804
|
};
|
|
583
805
|
}
|
|
584
806
|
|
|
807
|
+
// D3 (ggui#1380) — the evaluation lane records what it spends; without a
|
|
808
|
+
// tracker it stops here, on the thrown-round stamp, with the reason.
|
|
809
|
+
const costTracker = requireCostTracker(ctx.costTracker);
|
|
810
|
+
|
|
585
811
|
// Await pre-warmed context (only on first eval round, generated during coding)
|
|
586
812
|
if (evalRoundsUsed === 1 && !preWarmedContext && preWarmPromise) {
|
|
587
813
|
preWarmedContext = await preWarmPromise;
|
|
@@ -599,6 +825,22 @@ export async function runEvalRound(
|
|
|
599
825
|
profile: harness.profile,
|
|
600
826
|
// The caller's theme, when named — the round paints what the app paints.
|
|
601
827
|
...(visualEvaluation?.cssTokens !== undefined ? { cssTokens: visualEvaluation.cssTokens } : {}),
|
|
828
|
+
// ggui#1436 — the criteria bank and the card's context; the judge selects per canvas, report-only.
|
|
829
|
+
...(visualEvaluation?.criteriaBank !== undefined
|
|
830
|
+
? {
|
|
831
|
+
criteria: {
|
|
832
|
+
bank: visualEvaluation.criteriaBank,
|
|
833
|
+
context: criteriaContextFor({
|
|
834
|
+
classification,
|
|
835
|
+
contract,
|
|
836
|
+
cssTokens: visualEvaluation.cssTokens,
|
|
837
|
+
profile: harness.profile,
|
|
838
|
+
shell: ctx.shellType ?? "unknown",
|
|
839
|
+
...(visualEvaluation.criteriaKind !== undefined ? { kind: visualEvaluation.criteriaKind } : {}),
|
|
840
|
+
}),
|
|
841
|
+
},
|
|
842
|
+
}
|
|
843
|
+
: {}),
|
|
602
844
|
};
|
|
603
845
|
const visualFrame: VisualFitConfig = {
|
|
604
846
|
// The in-loop visual round renders the same sample the runtime probe
|
package/src/harness/policy.ts
CHANGED
|
@@ -252,6 +252,17 @@ export function resolveRunPolicyForProfile(
|
|
|
252
252
|
};
|
|
253
253
|
}
|
|
254
254
|
|
|
255
|
+
// ctx-slice-primitives-v2 (ggui#1324) — the #45 slice plus the input
|
|
256
|
+
// primitives the contract's slots and action payloads imply.
|
|
257
|
+
if (profile === "ctx-slice-primitives-v2") {
|
|
258
|
+
return {
|
|
259
|
+
context: {
|
|
260
|
+
...harness.policy.context,
|
|
261
|
+
primitiveDocSlice: "axis-keyed+contract",
|
|
262
|
+
},
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
|
|
255
266
|
// Archived experimental profiles:
|
|
256
267
|
//
|
|
257
268
|
// experiment-40-provider-asymmetric — provider-specific tier-0 labels
|