@ggui-ai/ui-gen 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter-Du7P_5Fx.d.ts +201 -0
- package/dist/adapters/generation-dispatch.d.ts +33 -12
- package/dist/adapters/generation-dispatch.js +549 -129
- package/dist/adapters/generation-dispatch.js.map +1 -1
- package/dist/adapters/index.js +1 -1
- package/dist/adapters/index.js.map +1 -1
- package/dist/advanced/index.d.ts +5 -3
- package/dist/advanced/index.js +601 -144
- package/dist/advanced/index.js.map +1 -1
- package/dist/blueprint-validator.js +178 -54
- package/dist/blueprint-validator.js.map +1 -1
- package/dist/boilerplate.js +1 -0
- package/dist/boilerplate.js.map +1 -1
- package/dist/check/index.d.ts +2 -1
- package/dist/check/index.js +47 -1
- package/dist/check/index.js.map +1 -1
- package/dist/{create-ui-generator-BEkzEtNz.d.ts → create-ui-generator-GIhzx08x.d.ts} +34 -10
- package/dist/{dispatch-DJNpQ7DI.d.ts → dispatch-BnAH56p4.d.ts} +1 -1
- package/dist/evaluation/axis-checks/index.d.ts +4 -3
- package/dist/evaluation/axis-checks/index.js +19 -4
- package/dist/evaluation/axis-checks/index.js.map +1 -1
- package/dist/evaluation/axis-checks/registry.d.ts +2 -1
- package/dist/evaluation/axis-checks/registry.js +19 -4
- package/dist/evaluation/axis-checks/registry.js.map +1 -1
- package/dist/evaluation/index.d.ts +4 -3
- package/dist/evaluation/index.js +21 -6
- package/dist/evaluation/index.js.map +1 -1
- package/dist/harness/check/runtime-render/index.d.ts +27 -8
- package/dist/harness/check/runtime-render/index.js +217 -54
- package/dist/harness/check/runtime-render/index.js.map +1 -1
- package/dist/harness/check/runtime-render/render-check-worker.d.ts +8 -1
- package/dist/harness/check/runtime-render/render-check-worker.js +72 -32
- package/dist/harness/check/runtime-render/render-check-worker.js.map +1 -1
- package/dist/harness/index.d.ts +2 -1
- package/dist/harness/index.js +3 -2
- package/dist/harness/index.js.map +1 -1
- package/dist/harness/prompts.js.map +1 -1
- package/dist/harness/result-types.d.ts +2 -1
- package/dist/harness/runtime.d.ts +2 -1
- package/dist/harness/runtime.js +2 -1
- package/dist/harness/runtime.js.map +1 -1
- package/dist/harness/types-public.d.ts +18 -5
- package/dist/harness/types-public.js +1 -0
- package/dist/harness/types-public.js.map +1 -1
- package/dist/harness/validator-trace-sink.d.ts +2 -1
- package/dist/index.d.ts +38 -3
- package/dist/index.js +592 -144
- package/dist/index.js.map +1 -1
- package/dist/patch.d.ts +2 -1
- package/dist/policy.d.ts +7 -4
- package/dist/policy.js.map +1 -1
- package/dist/tools/get-primitives-ts.d.ts +1 -1
- package/dist/tools/get-primitives-ts.js +1 -1
- package/dist/tools/get-primitives-ts.js.map +1 -1
- package/dist/tools/render-check-worker.d.ts +7 -1
- package/dist/tools/render-check-worker.js +55 -23
- package/dist/tools/render-check-worker.js.map +1 -1
- package/dist/{types-public-mM_-1Yk3.d.ts → types-public-Bk6d1WLN.d.ts} +98 -9
- package/dist/validation/index.d.ts +1 -1
- package/dist/validation/index.js +1 -1
- package/dist/validation/index.js.map +1 -1
- package/dist/workflows.d.ts +2 -1
- package/package.json +12 -12
- package/src/adapters/generation-dispatch.ts +137 -96
- package/src/advanced/generator.ts +16 -1
- package/src/blueprint-validator.ts +21 -6
- package/src/boilerplate/hard-sections.ts +1 -0
- package/src/check/run-tier0.ts +80 -1
- package/src/coding-agent/tools.ts +6 -1
- package/src/create-ui-generator.ts +185 -95
- package/src/evaluation/axis-checks/checks/state-payload.ts +29 -4
- package/src/evaluation/types-public.ts +94 -8
- package/src/harness/check/runtime-render/adapter.ts +223 -58
- package/src/harness/check/runtime-render/index.ts +75 -2
- package/src/harness/check/runtime-render/render-check-host.ts +111 -26
- package/src/harness/check/runtime-render/render-check-worker.ts +83 -29
- package/src/harness/check/runtime-render/render-check.ts +173 -117
- package/src/harness/coding/generate-task-runner.ts +57 -4
- package/src/harness/coding/init-session.ts +13 -0
- package/src/harness/coding/run-eval-round.ts +238 -13
- package/src/harness/llm-router.ts +3 -3
- package/src/harness/policy.ts +11 -0
- package/src/harness/primitive-slice.ts +104 -0
- package/src/harness/types-public.ts +17 -5
- package/src/index.ts +4 -0
- package/src/policy.ts +7 -4
- package/src/tools/get-primitives-ts.ts +1 -1
- package/src/tools/render-check-worker.ts +74 -23
- package/src/validation/primitives.ts +1 -1
- package/dist/render-check-C0CqVceF.d.ts +0 -87
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
import type { Classification } from "../../classifier/index.js";
|
|
14
14
|
import type { JsonValue } from "@ggui-ai/protocol";
|
|
15
|
-
import type { ContractFeedbackRecord, EvalResult } from "../../evaluation/types-public.js";
|
|
15
|
+
import type { ContractFeedbackRecord, EvalResult, RuntimeProbeMeta } from "../../evaluation/types-public.js";
|
|
16
16
|
import type { Harness, Task, TaskContext } from "../types-public.js";
|
|
17
17
|
import type { SingleComponentParams } from "../runtime.js";
|
|
18
18
|
import type { TaskRunner } from "../index.js";
|
|
@@ -77,6 +77,16 @@ export interface GenerateTelemetry {
|
|
|
77
77
|
cacheCreationTokens?: number;
|
|
78
78
|
/** Latest evalResult — undefined when eval was disabled or never ran. */
|
|
79
79
|
evalResult: EvalResult | undefined;
|
|
80
|
+
/**
|
|
81
|
+
* ggui#1380 — whether the probe-only lane's one repair turn has been
|
|
82
|
+
* spent. Set when the first probe-only round fed a recoverable render
|
|
83
|
+
* crash back; read by the next round (which then always breaks) and by
|
|
84
|
+
* the turn after the repair (which never loops on a failed self-check).
|
|
85
|
+
* Stays `false` on every other lane.
|
|
86
|
+
*/
|
|
87
|
+
probeRepairUsed: boolean;
|
|
88
|
+
/** ggui#1380 C2b — the probe that bought the repair turn (the first probe-only round's meta), recorded on round 2 as `trigger`. */
|
|
89
|
+
probeRepairTrigger: RuntimeProbeMeta | undefined;
|
|
80
90
|
/**
|
|
81
91
|
* ggui#1261 — the contract-feedback round, once one fired: kept across the
|
|
82
92
|
* rounds after it (each rebuilds `evalResult`) so the assembled result can
|
|
@@ -126,6 +136,8 @@ export function createTelemetry(): GenerateTelemetry {
|
|
|
126
136
|
totalIn: 0,
|
|
127
137
|
totalOut: 0,
|
|
128
138
|
evalResult: undefined,
|
|
139
|
+
probeRepairUsed: false,
|
|
140
|
+
probeRepairTrigger: undefined,
|
|
129
141
|
contractFeedback: undefined,
|
|
130
142
|
sameExchangeBreak: undefined,
|
|
131
143
|
compiledCode: "",
|
|
@@ -274,6 +286,22 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
|
|
|
274
286
|
iconNamesCache = turn.iconNamesCache;
|
|
275
287
|
preWarmedContext = turn.preWarmedContext;
|
|
276
288
|
|
|
289
|
+
// ggui#1380 — the probe-only lane's repair turn did not produce a
|
|
290
|
+
// self-check-passing compile (a failed self-check, or no tool call):
|
|
291
|
+
// the lane grants AT MOST ONE eval-fix turn, so the loop never feeds
|
|
292
|
+
// the self-check violations back for another. The pre-repair card is
|
|
293
|
+
// what ships (`compiledCode` / `pairedSource` are untouched since the
|
|
294
|
+
// turn that compiled it), and the round's result records the repair
|
|
295
|
+
// as attempted but not compiled.
|
|
296
|
+
if (turn.control !== "proceed" && session.probeOnlyEnabled && telemetry.probeRepairUsed) {
|
|
297
|
+
if (telemetry.evalResult !== undefined) {
|
|
298
|
+
telemetry.evalResult = { ...telemetry.evalResult, runtimeProbeRepair: { attempted: true, compiled: false } };
|
|
299
|
+
}
|
|
300
|
+
console.log(
|
|
301
|
+
`[simple] probe-only repair turn did not compile (${turn.outcome ?? "no tool call"}) — serving the pre-repair card`,
|
|
302
|
+
);
|
|
303
|
+
break;
|
|
304
|
+
}
|
|
277
305
|
if (turn.control === "break") break;
|
|
278
306
|
if (turn.control === "continue") {
|
|
279
307
|
lastResultText = turn.lastResultText;
|
|
@@ -326,11 +354,14 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
|
|
|
326
354
|
}
|
|
327
355
|
|
|
328
356
|
// ── Eval round ──
|
|
357
|
+
// Two lanes open the gate: the evaluation lane (its modules loaded, an
|
|
358
|
+
// evaluator configured) and the probe-only lane (ggui#1380: the probe
|
|
359
|
+
// wired, no evaluator — the round runs the probe once and nothing else).
|
|
329
360
|
if (
|
|
330
361
|
!evalDone &&
|
|
331
362
|
telemetry.compiledCode &&
|
|
332
|
-
session.tiersMod &&
|
|
333
|
-
|
|
363
|
+
((session.tiersMod && (session.codeEvalEnabled || session.visualEvalEnabled)) ||
|
|
364
|
+
session.probeOnlyEnabled)
|
|
334
365
|
) {
|
|
335
366
|
// Measure eval-round wall-clock here (not inside runEvalRound) so the
|
|
336
367
|
// accumulator covers axis-checks + LLM+visual parallel + merge +
|
|
@@ -351,11 +382,14 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
|
|
|
351
382
|
visualThreshold: session.visualThreshold,
|
|
352
383
|
qualityMode: session.qualityMode,
|
|
353
384
|
maxEvalRounds: session.maxEvalRounds,
|
|
354
|
-
costTracker: session.costTracker
|
|
385
|
+
costTracker: session.costTracker,
|
|
355
386
|
llmEvalMod: session.llmEvalMod,
|
|
356
387
|
visualMod: session.visualMod,
|
|
357
388
|
preWarmPromise: session.preWarmPromise,
|
|
358
389
|
onProgress: params.onProgress,
|
|
390
|
+
probeOnly: session.probeOnlyEnabled,
|
|
391
|
+
probeRepairUsed: telemetry.probeRepairUsed,
|
|
392
|
+
...(telemetry.probeRepairTrigger !== undefined ? { probeRepairTrigger: telemetry.probeRepairTrigger } : {}),
|
|
359
393
|
},
|
|
360
394
|
{
|
|
361
395
|
compiledCode: telemetry.compiledCode,
|
|
@@ -379,6 +413,11 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
|
|
|
379
413
|
|
|
380
414
|
if (round.control === "break") break;
|
|
381
415
|
// round.control === "feedback" — set next coding turn input.
|
|
416
|
+
// On the probe-only lane this is the one repair turn (ggui#1380).
|
|
417
|
+
if (session.probeOnlyEnabled) {
|
|
418
|
+
telemetry.probeRepairUsed = true;
|
|
419
|
+
telemetry.probeRepairTrigger = round.evalResult?.runtimeProbe;
|
|
420
|
+
}
|
|
382
421
|
lastResultText = round.lastResultText;
|
|
383
422
|
isEvalFeedback = round.isEvalFeedback;
|
|
384
423
|
lastDiffFailed = round.lastDiffFailed;
|
|
@@ -388,6 +427,20 @@ export function createGenerateTaskRunner(input: CreateGenerateRunnerInput): Task
|
|
|
388
427
|
}
|
|
389
428
|
} while (telemetry.turnsUsed < maxTurns && !evalDone);
|
|
390
429
|
|
|
430
|
+
// ggui#1380 — the probe-only lane's one repair turn was granted by the
|
|
431
|
+
// round but the turn cap had already been reached, so it was never taken:
|
|
432
|
+
// the card ships as it was, the probe's record stays, and no repair record
|
|
433
|
+
// is stamped (nothing was attempted). Say so, so the absence reads as the
|
|
434
|
+
// cap and not as a repair that vanished.
|
|
435
|
+
if (
|
|
436
|
+
session.probeOnlyEnabled &&
|
|
437
|
+
telemetry.probeRepairUsed &&
|
|
438
|
+
telemetry.evalResult !== undefined &&
|
|
439
|
+
telemetry.evalResult.runtimeProbeRepair === undefined
|
|
440
|
+
) {
|
|
441
|
+
console.log(`[simple] probe-only repair turn not taken — turn cap (${maxTurns}) reached before it`);
|
|
442
|
+
}
|
|
443
|
+
|
|
391
444
|
telemetry.codingMs = Date.now() - telemetry.codingStartedAtMs;
|
|
392
445
|
|
|
393
446
|
// ── Fallback: pull compiled code from commitMeta if loop exited dry ──
|
|
@@ -49,6 +49,16 @@ export interface CodingSession {
|
|
|
49
49
|
// Eval gate config (computed once from params/qualityConfig).
|
|
50
50
|
readonly codeEvalEnabled: boolean;
|
|
51
51
|
readonly visualEvalEnabled: boolean;
|
|
52
|
+
/**
|
|
53
|
+
* ggui#1380 — the probe-only lane: the harness wires the runtime-render
|
|
54
|
+
* check and NO evaluator is configured (neither code nor visual). A
|
|
55
|
+
* serving deployment runs the probe once after the coding turns, observes,
|
|
56
|
+
* and buys one repair turn on a recoverable render crash. `evaluation:
|
|
57
|
+
* { enabled: false }` beside a wired probe is this lane too: a config
|
|
58
|
+
* that enables nothing is no evaluator. False whenever an evaluator runs —
|
|
59
|
+
* that lane's exit probe is a different round — and false without a probe.
|
|
60
|
+
*/
|
|
61
|
+
readonly probeOnlyEnabled: boolean;
|
|
52
62
|
readonly maxEvalRounds: number;
|
|
53
63
|
readonly visualThreshold: number;
|
|
54
64
|
readonly qualityMode: "fast" | "auto-improve" | "high-quality";
|
|
@@ -194,6 +204,8 @@ export async function initSession(input: {
|
|
|
194
204
|
const qualityMode = qualityConfig?.quality ?? "fast";
|
|
195
205
|
const codeEvalEnabled = !!params.evaluation?.enabled;
|
|
196
206
|
const visualEvalEnabled = !!(params.visualEvaluation?.enabled || qualityConfig?.visualEval);
|
|
207
|
+
const probeOnlyEnabled =
|
|
208
|
+
harness.check.runtimeRender !== undefined && !codeEvalEnabled && !visualEvalEnabled;
|
|
197
209
|
const maxEvalRounds =
|
|
198
210
|
qualityMode === "high-quality"
|
|
199
211
|
? (params.evaluation?.maxRounds ?? 3)
|
|
@@ -230,6 +242,7 @@ export async function initSession(input: {
|
|
|
230
242
|
systemPrompt,
|
|
231
243
|
codeEvalEnabled,
|
|
232
244
|
visualEvalEnabled,
|
|
245
|
+
probeOnlyEnabled,
|
|
233
246
|
maxEvalRounds,
|
|
234
247
|
visualThreshold,
|
|
235
248
|
qualityMode,
|
|
@@ -10,9 +10,11 @@
|
|
|
10
10
|
// (capped at MAX_FEEDBACK_ISSUES per turn).
|
|
11
11
|
//
|
|
12
12
|
// Does NOT own: the do/while wrapper, the eligibility gate at the call
|
|
13
|
-
// site (`!evalDone && compiledCode && tiersMod && (codeEval || visualEval)
|
|
14
|
-
//
|
|
15
|
-
//
|
|
13
|
+
// site (`!evalDone && compiledCode && ((tiersMod && (codeEval || visualEval))
|
|
14
|
+
// || probeOnlyEnabled)` — the evaluation lane, or the probe-only lane a
|
|
15
|
+
// serving deployment runs when it wires the runtime probe without an
|
|
16
|
+
// evaluator, ggui#1380), or the threading of mutable state — caller still
|
|
17
|
+
// owns those for now and the closure migration (step 3) will move them too.
|
|
16
18
|
//
|
|
17
19
|
// Behavior MUST match the inlined original byte-for-byte — this is a
|
|
18
20
|
// mechanical extraction, not a redesign.
|
|
@@ -22,12 +24,26 @@ import { listContractGadgets } from "@ggui-ai/protocol";
|
|
|
22
24
|
import type { Classification } from "../../classifier/index.js";
|
|
23
25
|
import type { AgentWorkspace } from "../../coding-agent/workspace.js";
|
|
24
26
|
import type { CostTracker } from "../../evaluation/cost-tracker.js";
|
|
25
|
-
import type {
|
|
27
|
+
import type {
|
|
28
|
+
ContractFeedbackRecord,
|
|
29
|
+
EvalIssue,
|
|
30
|
+
EvalResult,
|
|
31
|
+
RuntimeProbeMeta,
|
|
32
|
+
RuntimeProbeMetaDetail,
|
|
33
|
+
RuntimeProbeRepair,
|
|
34
|
+
VisualEvalSummary,
|
|
35
|
+
VisualCoverage,
|
|
36
|
+
} from "../../evaluation/types-public.js";
|
|
26
37
|
import { notApplicableCoverage } from "../../evaluation/types-public.js";
|
|
27
38
|
import { mapProviderForEvaluator, visionJudgeProvider } from "../enforced-coding.js";
|
|
28
39
|
import { runCheck } from "../index.js";
|
|
29
|
-
import {
|
|
30
|
-
|
|
40
|
+
import {
|
|
41
|
+
RENDER_CHECK_KINDS,
|
|
42
|
+
isRecoverableRenderCrash,
|
|
43
|
+
parseRuntimeSubcategory,
|
|
44
|
+
type RenderCheckKind,
|
|
45
|
+
} from "../check/runtime-render/index.js";
|
|
46
|
+
import type { Harness, RuntimeRenderOutcome } from "../types-public.js";
|
|
31
47
|
import type { AgentSpec, SingleComponentParams } from "../runtime.js";
|
|
32
48
|
import type {
|
|
33
49
|
CanvasFitReading,
|
|
@@ -61,12 +77,43 @@ export interface EvalRoundContext {
|
|
|
61
77
|
readonly visualThreshold: number;
|
|
62
78
|
readonly qualityMode: "fast" | "auto-improve" | "high-quality";
|
|
63
79
|
readonly maxEvalRounds: number;
|
|
64
|
-
/**
|
|
65
|
-
|
|
80
|
+
/**
|
|
81
|
+
* The evaluation lane's cost tracker. `null` on the probe-only lane, which
|
|
82
|
+
* runs no evaluator and records no cost; an evaluation round without one
|
|
83
|
+
* lands on the round's infra-skipped stamp (ggui#1110) with the reason
|
|
84
|
+
* {@link COST_TRACKER_ABSENT_REASON} — never a dereference.
|
|
85
|
+
*/
|
|
86
|
+
readonly costTracker: CostTracker | null;
|
|
66
87
|
readonly llmEvalMod: LlmEvalMod | null;
|
|
67
88
|
readonly visualMod: VisualEvalMod | null;
|
|
68
89
|
readonly preWarmPromise: Promise<PreWarmedEvalContext | null> | undefined;
|
|
69
90
|
readonly onProgress?: (event: unknown) => void;
|
|
91
|
+
/**
|
|
92
|
+
* ggui#1380 — the probe-only lane: a serving deployment that wires the
|
|
93
|
+
* runtime probe and configures NO evaluator. The round runs the probe
|
|
94
|
+
* once and nothing else (no axis checks — the deterministic checks already
|
|
95
|
+
* ran in the coding turns' self-check — no evaluator, no visual leg).
|
|
96
|
+
* REQUIRED and explicit: it is never inferred from null modules, because the
|
|
97
|
+
* evaluation lane with null modules is a different round (the evaluation
|
|
98
|
+
* lane's own exit probe) and must stay one.
|
|
99
|
+
*/
|
|
100
|
+
readonly probeOnly: boolean;
|
|
101
|
+
/**
|
|
102
|
+
* ggui#1380 — whether this generation already spent its one repair turn.
|
|
103
|
+
* `false` on the first probe-only round: a recoverable render crash there
|
|
104
|
+
* buys the turn. `true` on the second: the round breaks whatever the probe
|
|
105
|
+
* reads and records the repair with the probe that bought it as `trigger`.
|
|
106
|
+
* Meaningful on the probe-only lane only.
|
|
107
|
+
*/
|
|
108
|
+
readonly probeRepairUsed: boolean;
|
|
109
|
+
/**
|
|
110
|
+
* ggui#1380 C2b — on the second probe-only round, the probe that bought the
|
|
111
|
+
* repair turn (the first round's meta), recorded as `runtimeProbeRepair.trigger`
|
|
112
|
+
* so the crash that happened is never lost behind the re-probe. The runner
|
|
113
|
+
* sets it together with `probeRepairUsed`; a re-probe round without it is a
|
|
114
|
+
* harness defect.
|
|
115
|
+
*/
|
|
116
|
+
readonly probeRepairTrigger?: RuntimeProbeMeta;
|
|
70
117
|
}
|
|
71
118
|
|
|
72
119
|
/**
|
|
@@ -177,6 +224,29 @@ function formatFitFeedback(issue: EvalIssue): string {
|
|
|
177
224
|
/** Why an eval result carries no criteria: the round itself threw (ggui#1110). */
|
|
178
225
|
const EVAL_ROUND_THREW_REASON = "eval round threw";
|
|
179
226
|
|
|
227
|
+
/**
|
|
228
|
+
* ggui#1380 — the coverage reasons a probe-only round stamps: no evaluator
|
|
229
|
+
* and no visual leg are configured, BY DESIGN, so every criterion is
|
|
230
|
+
* `not-applicable` (never `skipped`, which would read as a silent absence).
|
|
231
|
+
*/
|
|
232
|
+
const PROBE_ONLY_CRITERIA_REASON = "probe-only round: no evaluator configured";
|
|
233
|
+
const PROBE_ONLY_VISUAL_REASON = "probe-only round: no visual evaluator configured";
|
|
234
|
+
/** The `pass` entry a probe-only round records — the probe was the only gate. */
|
|
235
|
+
const PROBE_ONLY_PASS = "probe-only";
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* D3 (ggui#1380) — the evaluation lane's one named guard: a round that
|
|
239
|
+
* reaches the evaluator without a cost tracker cannot record what it spends,
|
|
240
|
+
* so it stops HERE, on the ggui#1110 infra-skipped stamp with this reason,
|
|
241
|
+
* instead of dereferencing `null` somewhere past the evaluator call.
|
|
242
|
+
*/
|
|
243
|
+
export const COST_TRACKER_ABSENT_REASON = "cost tracker absent on an evaluation round";
|
|
244
|
+
|
|
245
|
+
function requireCostTracker(costTracker: CostTracker | null): CostTracker {
|
|
246
|
+
if (costTracker === null) throw new Error(COST_TRACKER_ABSENT_REASON);
|
|
247
|
+
return costTracker;
|
|
248
|
+
}
|
|
249
|
+
|
|
180
250
|
/**
|
|
181
251
|
* The in-loop visual leg, made total (ggui#1248). The judge runs on a VISION
|
|
182
252
|
* provider: the agent's provider narrowed by `visionJudgeProvider`, never
|
|
@@ -306,6 +376,62 @@ interface ProbeAtExitResult {
|
|
|
306
376
|
readonly meta: RuntimeProbeMeta;
|
|
307
377
|
}
|
|
308
378
|
|
|
379
|
+
/**
|
|
380
|
+
* The distinct check kinds among a probe's `result: 'fail'` issues, in the
|
|
381
|
+
* order {@link RENDER_CHECK_KINDS} declares them (a render crash reads
|
|
382
|
+
* first). Parsed from the `runtime:<check>[:<subject>]` subcategory the
|
|
383
|
+
* adapter writes; a warn, an issue with no subcategory, or a subcategory that
|
|
384
|
+
* names no check contributes nothing.
|
|
385
|
+
*/
|
|
386
|
+
function failedCheckKinds(probeIssues: readonly EvalIssue[]): readonly RenderCheckKind[] {
|
|
387
|
+
const failed = new Set<RenderCheckKind>();
|
|
388
|
+
for (const issue of probeIssues) {
|
|
389
|
+
if (issue.result !== "fail" || issue.subcategory === undefined) continue;
|
|
390
|
+
const kind = parseRuntimeSubcategory(issue.subcategory);
|
|
391
|
+
if (kind !== undefined) failed.add(kind);
|
|
392
|
+
}
|
|
393
|
+
return RENDER_CHECK_KINDS.filter((kind) => failed.has(kind));
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/**
|
|
397
|
+
* ONE builder for the meta every exit probe stamps (ggui#1380): the
|
|
398
|
+
* probe-only round and the evaluation lane's exit probe both go through
|
|
399
|
+
* `runProbeAtExit`, and this is where its outcome becomes a verdict. The
|
|
400
|
+
* verdict is computed from the PROBE'S OWN issues: `fail` when any check
|
|
401
|
+
* failed (`failChecks` = the distinct failing kinds), `pass` when none did;
|
|
402
|
+
* a probe that did not run to a verdict (`timed-out`, `infra-skipped`,
|
|
403
|
+
* `not-applicable`) carries no verdict key at all.
|
|
404
|
+
*
|
|
405
|
+
* The verdict is not the repair trigger. `fail` names ANY failing check;
|
|
406
|
+
* the one repair turn fires on a `render-no-throw` fail that
|
|
407
|
+
* `isRecoverableRenderCrash` recognises, and on nothing else — a `fail`
|
|
408
|
+
* verdict whose checks are `prop-sensitivity`, `action-wiring` or any other
|
|
409
|
+
* kind is recorded on the meta and never fed back.
|
|
410
|
+
*/
|
|
411
|
+
function probeMetaOf(outcome: RuntimeRenderOutcome, probeIssues: readonly EvalIssue[]): RuntimeProbeMeta {
|
|
412
|
+
const detail: RuntimeProbeMetaDetail = {
|
|
413
|
+
...(outcome.reason !== undefined ? { reason: outcome.reason } : {}),
|
|
414
|
+
...(outcome.renderMs !== undefined ? { renderMs: outcome.renderMs } : {}),
|
|
415
|
+
...(outcome.hostLoad !== undefined ? { hostLoad: outcome.hostLoad } : {}),
|
|
416
|
+
...(outcome.queuedMs !== undefined ? { queuedMs: outcome.queuedMs } : {}),
|
|
417
|
+
};
|
|
418
|
+
const elapsed = outcome.elapsedMs !== undefined ? { elapsedMs: outcome.elapsedMs } : {};
|
|
419
|
+
if (outcome.status !== "ran") {
|
|
420
|
+
return { status: outcome.status, ...elapsed, ...detail };
|
|
421
|
+
}
|
|
422
|
+
const [first, ...rest] = failedCheckKinds(probeIssues);
|
|
423
|
+
if (first === undefined) {
|
|
424
|
+
return { status: "ran", verdict: "pass", ...elapsed, ...detail };
|
|
425
|
+
}
|
|
426
|
+
return { status: "ran", verdict: "fail", failChecks: [first, ...rest], ...elapsed, ...detail };
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/** The probe's status and, when it ran, its verdict — for the round's log line. */
|
|
430
|
+
function describeProbeMeta(meta: RuntimeProbeMeta): string {
|
|
431
|
+
if (meta.verdict === undefined) return meta.status;
|
|
432
|
+
return meta.verdict === "pass" ? "ran, verdict pass" : `ran, verdict fail (${meta.failChecks.join(", ")})`;
|
|
433
|
+
}
|
|
434
|
+
|
|
309
435
|
/**
|
|
310
436
|
* Run the runtime-render probe ONCE at an exit-decision point. Off the
|
|
311
437
|
* per-turn hot path: `runCheck` is invoked with `skipRuntimeRender: true`
|
|
@@ -356,10 +482,7 @@ async function runProbeAtExit(input: {
|
|
|
356
482
|
meta: { status: "infra-skipped", reason: message },
|
|
357
483
|
};
|
|
358
484
|
}
|
|
359
|
-
const meta
|
|
360
|
-
status: outcome.status,
|
|
361
|
-
...(outcome.reason !== undefined ? { reason: outcome.reason } : {}),
|
|
362
|
-
};
|
|
485
|
+
const meta = probeMetaOf(outcome, outcome.issues);
|
|
363
486
|
if (outcome.status !== "ran") {
|
|
364
487
|
return { fired: false, recoverableFail: false, probeIssues: [], meta };
|
|
365
488
|
}
|
|
@@ -448,11 +571,13 @@ export async function runEvalRound(
|
|
|
448
571
|
visualThreshold,
|
|
449
572
|
qualityMode,
|
|
450
573
|
maxEvalRounds,
|
|
451
|
-
costTracker,
|
|
452
574
|
llmEvalMod,
|
|
453
575
|
visualMod,
|
|
454
576
|
preWarmPromise,
|
|
455
577
|
onProgress,
|
|
578
|
+
probeOnly,
|
|
579
|
+
probeRepairUsed,
|
|
580
|
+
probeRepairTrigger,
|
|
456
581
|
} = ctx;
|
|
457
582
|
const { compiledCode, prevModeSubcats, prevFailFingerprints } = input;
|
|
458
583
|
let { evalRoundsUsed, preWarmedContext } = input;
|
|
@@ -466,6 +591,102 @@ export async function runEvalRound(
|
|
|
466
591
|
let evalLlmMs = 0;
|
|
467
592
|
|
|
468
593
|
try {
|
|
594
|
+
// ── The probe-only round (ggui#1380) ──
|
|
595
|
+
// A serving deployment that wires the runtime probe and configures no
|
|
596
|
+
// evaluator: the probe runs ONCE, before anything else, and no axis
|
|
597
|
+
// check runs (the served card is the coding turns' — its deterministic
|
|
598
|
+
// checks ran in their self-check). A recoverable render crash on the
|
|
599
|
+
// first round buys exactly one repair turn; the second round breaks
|
|
600
|
+
// whatever it reads and records the repair's after. Timed-out,
|
|
601
|
+
// infra-skipped and not-applicable never block and never repair.
|
|
602
|
+
if (probeOnly) {
|
|
603
|
+
const exitProbe = await runProbeAtExit({
|
|
604
|
+
harness,
|
|
605
|
+
sourceCode: currentSource,
|
|
606
|
+
compiledCode,
|
|
607
|
+
contract,
|
|
608
|
+
fixtureProps,
|
|
609
|
+
});
|
|
610
|
+
const probeOnlyResult: EvalResult = {
|
|
611
|
+
issues: [...exitProbe.probeIssues],
|
|
612
|
+
pass: [PROBE_ONLY_PASS],
|
|
613
|
+
runtimeProbe: exitProbe.meta,
|
|
614
|
+
criteriaCoverage: notApplicableCoverage(PROBE_ONLY_CRITERIA_REASON),
|
|
615
|
+
visualCoverage: { status: "not-applicable", reason: PROBE_ONLY_VISUAL_REASON },
|
|
616
|
+
};
|
|
617
|
+
const recoverableFail = exitProbe.fired && exitProbe.recoverableFail;
|
|
618
|
+
if (probeRepairUsed) {
|
|
619
|
+
// This round IS the re-probe (a repair that did not compile never
|
|
620
|
+
// reaches a round — the runner stamps `compiled: false` itself). The
|
|
621
|
+
// top level is the re-probe; the record names the probe that bought
|
|
622
|
+
// the turn as `trigger`, so each probe is recorded exactly once.
|
|
623
|
+
if (probeRepairTrigger === undefined) {
|
|
624
|
+
throw new Error("harness defect: a probe-only re-probe round without the probe that bought the repair turn");
|
|
625
|
+
}
|
|
626
|
+
const repair: RuntimeProbeRepair = { attempted: true, compiled: true, trigger: probeRepairTrigger };
|
|
627
|
+
evalResult = { ...probeOnlyResult, runtimeProbeRepair: repair };
|
|
628
|
+
console.log(
|
|
629
|
+
`[simple] eval round ${evalRoundsUsed}: probe-only re-probe after the repair turn — ` +
|
|
630
|
+
`${describeProbeMeta(exitProbe.meta)}${recoverableFail ? ", recoverable crash still present" : ""} — serving`,
|
|
631
|
+
);
|
|
632
|
+
return {
|
|
633
|
+
control: "break",
|
|
634
|
+
evalDone: true,
|
|
635
|
+
evalResult,
|
|
636
|
+
evalRoundsUsed,
|
|
637
|
+
prevModeSubcats,
|
|
638
|
+
prevFailFingerprints,
|
|
639
|
+
preWarmedContext,
|
|
640
|
+
evalTokens,
|
|
641
|
+
evalLlmMs,
|
|
642
|
+
lastResultText: "",
|
|
643
|
+
isEvalFeedback: false,
|
|
644
|
+
lastDiffFailed: false,
|
|
645
|
+
};
|
|
646
|
+
}
|
|
647
|
+
evalResult = probeOnlyResult;
|
|
648
|
+
if (recoverableFail) {
|
|
649
|
+
const runtimeFails = exitProbe.probeIssues.filter(
|
|
650
|
+
(i) => i.result === "fail" && i.subcategory?.startsWith("runtime:"),
|
|
651
|
+
);
|
|
652
|
+
const lines = runtimeFails.slice(0, MAX_FEEDBACK_ISSUES).map(formatRuntimeProbeFeedback);
|
|
653
|
+
console.log(
|
|
654
|
+
`[simple] eval round ${evalRoundsUsed}: probe-only round — runtime probe fail (recoverable) — granting the one repair turn`,
|
|
655
|
+
);
|
|
656
|
+
return {
|
|
657
|
+
control: "feedback",
|
|
658
|
+
evalDone: false,
|
|
659
|
+
evalResult,
|
|
660
|
+
evalRoundsUsed,
|
|
661
|
+
prevModeSubcats,
|
|
662
|
+
prevFailFingerprints,
|
|
663
|
+
preWarmedContext,
|
|
664
|
+
evalTokens,
|
|
665
|
+
evalLlmMs,
|
|
666
|
+
lastResultText: lines.join("\n\n"),
|
|
667
|
+
isEvalFeedback: true,
|
|
668
|
+
lastDiffFailed: false,
|
|
669
|
+
};
|
|
670
|
+
}
|
|
671
|
+
console.log(
|
|
672
|
+
`[simple] eval round ${evalRoundsUsed}: probe-only round — probe ${describeProbeMeta(exitProbe.meta)} — serving`,
|
|
673
|
+
);
|
|
674
|
+
return {
|
|
675
|
+
control: "break",
|
|
676
|
+
evalDone: true,
|
|
677
|
+
evalResult,
|
|
678
|
+
evalRoundsUsed,
|
|
679
|
+
prevModeSubcats,
|
|
680
|
+
prevFailFingerprints,
|
|
681
|
+
preWarmedContext,
|
|
682
|
+
evalTokens,
|
|
683
|
+
evalLlmMs,
|
|
684
|
+
lastResultText: "",
|
|
685
|
+
isEvalFeedback: false,
|
|
686
|
+
lastDiffFailed: false,
|
|
687
|
+
};
|
|
688
|
+
}
|
|
689
|
+
|
|
469
690
|
// Tier 0 already ran in autoCommit — go straight to mode-checks + LLM eval.
|
|
470
691
|
|
|
471
692
|
// ── Axis-keyed deterministic checks ──────────────
|
|
@@ -580,6 +801,10 @@ export async function runEvalRound(
|
|
|
580
801
|
};
|
|
581
802
|
}
|
|
582
803
|
|
|
804
|
+
// D3 (ggui#1380) — the evaluation lane records what it spends; without a
|
|
805
|
+
// tracker it stops here, on the thrown-round stamp, with the reason.
|
|
806
|
+
const costTracker = requireCostTracker(ctx.costTracker);
|
|
807
|
+
|
|
583
808
|
// Await pre-warmed context (only on first eval round, generated during coding)
|
|
584
809
|
if (evalRoundsUsed === 1 && !preWarmedContext && preWarmPromise) {
|
|
585
810
|
preWarmedContext = await preWarmPromise;
|
|
@@ -875,8 +875,8 @@ export class AnthropicAgent extends LLMAgent {
|
|
|
875
875
|
: toolChoice;
|
|
876
876
|
if (appliedToolChoice !== toolChoice) {
|
|
877
877
|
console.warn(
|
|
878
|
-
`[anthropic] callTools: ${normalizeAnthropicModelId(resolvedModel)}
|
|
879
|
-
`caller's 'required' downgraded to 'auto'`,
|
|
878
|
+
`[anthropic] callTools: ${normalizeAnthropicModelId(resolvedModel)} would reject a forced tool_choice (the API answers 400); ` +
|
|
879
|
+
`caller's 'required' downgraded to 'auto' before sending`,
|
|
880
880
|
);
|
|
881
881
|
}
|
|
882
882
|
const response = await this.apiCall(() =>
|
|
@@ -2128,7 +2128,7 @@ export class OpenRouterAgent extends LLMAgent {
|
|
|
2128
2128
|
requested === 'required' && anthropicRejectsForcedToolChoice(resolved) ? 'auto' : requested;
|
|
2129
2129
|
if (appliedToolChoice !== requested) {
|
|
2130
2130
|
console.warn(
|
|
2131
|
-
`[openrouter] callTools: ${resolved}
|
|
2131
|
+
`[openrouter] callTools: ${resolved} would reject a forced tool_choice (the API answers 400); caller's 'required' downgraded to 'auto' before sending`,
|
|
2132
2132
|
);
|
|
2133
2133
|
}
|
|
2134
2134
|
|
package/src/harness/policy.ts
CHANGED
|
@@ -252,6 +252,17 @@ export function resolveRunPolicyForProfile(
|
|
|
252
252
|
};
|
|
253
253
|
}
|
|
254
254
|
|
|
255
|
+
// ctx-slice-primitives-v2 (ggui#1324) — the #45 slice plus the input
|
|
256
|
+
// primitives the contract's slots and action payloads imply.
|
|
257
|
+
if (profile === "ctx-slice-primitives-v2") {
|
|
258
|
+
return {
|
|
259
|
+
context: {
|
|
260
|
+
...harness.policy.context,
|
|
261
|
+
primitiveDocSlice: "axis-keyed+contract",
|
|
262
|
+
},
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
|
|
255
266
|
// Archived experimental profiles:
|
|
256
267
|
//
|
|
257
268
|
// experiment-40-provider-asymmetric — provider-specific tier-0 labels
|
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
// prompt-build time and modifies the LLM's *initial* conditioning
|
|
24
24
|
// uniformly — it cannot contaminate a retry trajectory mid-run.
|
|
25
25
|
|
|
26
|
+
import type { DataContract, JsonSchema } from "@ggui-ai/protocol";
|
|
26
27
|
import type { Classification } from "../classifier/index.js";
|
|
27
28
|
|
|
28
29
|
/** Primitives every fixture needs — layout + typography + core interaction.
|
|
@@ -162,6 +163,62 @@ export function computePrimitiveAllowlist(
|
|
|
162
163
|
return [...set].sort();
|
|
163
164
|
}
|
|
164
165
|
|
|
166
|
+
/**
|
|
167
|
+
* The input primitives a contract's own fields imply (ggui#1324).
|
|
168
|
+
*
|
|
169
|
+
* The axis-keyed allowlist is derived from the classification alone, so a card
|
|
170
|
+
* whose axes do not say "form" still loses the inputs its contract needs: the
|
|
171
|
+
* #45 kanban canary edits a task's title and priority through an action
|
|
172
|
+
* payload, and the axis slice dropped `Input` / `TextArea` / `Select`.
|
|
173
|
+
*
|
|
174
|
+
* Walks every context slot's schema and every action payload's schema down to
|
|
175
|
+
* its leaves (object `properties`, array `items`) and maps each leaf type:
|
|
176
|
+
* an `enum` → `Select`, `RadioGroup`; `string` → `Input`, `TextArea`,
|
|
177
|
+
* `FormField`; `number` / `integer` → `Input`, `Slider`; `boolean` →
|
|
178
|
+
* `Checkbox`, `Toggle`. A declared stream adds `Spinner` and `Badge`. Props are
|
|
179
|
+
* display data and add nothing (the classification covers display). Returns a
|
|
180
|
+
* sorted array.
|
|
181
|
+
*/
|
|
182
|
+
export function computeContractPrimitives(
|
|
183
|
+
contract: DataContract | undefined,
|
|
184
|
+
): readonly string[] {
|
|
185
|
+
const out = new Set<string>();
|
|
186
|
+
if (!contract) return [];
|
|
187
|
+
for (const slot of Object.values(contract.contextSpec ?? {})) addLeaves(slot.schema, out);
|
|
188
|
+
for (const action of Object.values(contract.actionSpec ?? {})) {
|
|
189
|
+
if (action.schema) addLeaves(action.schema, out);
|
|
190
|
+
}
|
|
191
|
+
if (Object.keys(contract.streamSpec ?? {}).length > 0) {
|
|
192
|
+
out.add("Spinner");
|
|
193
|
+
out.add("Badge");
|
|
194
|
+
}
|
|
195
|
+
return [...out].sort();
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function addLeaves(schema: JsonSchema, out: Set<string>): void {
|
|
199
|
+
for (const child of Object.values(schema.properties ?? {})) addLeaves(child, out);
|
|
200
|
+
if (schema.items) addLeaves(schema.items, out);
|
|
201
|
+
if (schema.enum && schema.enum.length > 0) {
|
|
202
|
+
out.add("Select");
|
|
203
|
+
out.add("RadioGroup");
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
const types = Array.isArray(schema.type) ? schema.type : schema.type ? [schema.type] : [];
|
|
207
|
+
for (const t of types) {
|
|
208
|
+
if (t === "string") {
|
|
209
|
+
out.add("Input");
|
|
210
|
+
out.add("TextArea");
|
|
211
|
+
out.add("FormField");
|
|
212
|
+
} else if (t === "number" || t === "integer") {
|
|
213
|
+
out.add("Input");
|
|
214
|
+
out.add("Slider");
|
|
215
|
+
} else if (t === "boolean") {
|
|
216
|
+
out.add("Checkbox");
|
|
217
|
+
out.add("Toggle");
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
165
222
|
/**
|
|
166
223
|
* Slice `PRIMITIVES_DOCUMENTATION` to include only the allowlisted primitive
|
|
167
224
|
* sections. The doc structure is:
|
|
@@ -222,3 +279,50 @@ export function slicePrimitiveDocumentation(
|
|
|
222
279
|
|
|
223
280
|
return preamble + kept.join("");
|
|
224
281
|
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* Slice the TypeScript-interface primitives reference
|
|
285
|
+
* (`PRIMITIVES_DOCUMENTATION_TS`) by the same allowlist the markdown
|
|
286
|
+
* slicer takes (ggui#1324, speed/002).
|
|
287
|
+
*
|
|
288
|
+
* The TS doc is a sequence of SECTIONS, each opened by a header line
|
|
289
|
+
* `// <Name> — <description>` and carrying that name's `interface`, its
|
|
290
|
+
* `// Example:` and the usage snippet that follows; category banners
|
|
291
|
+
* (`// ═══…`) sit between sections. A section whose `<Name>` is a KNOWN
|
|
292
|
+
* primitive is kept iff `<Name>` is in the allowlist. Every other line is
|
|
293
|
+
* kept: the preamble, the banners, the `CRITICAL` guidance sections, the
|
|
294
|
+
* support-type sections (`SelectOption`, `TableColumn`, …) and any section
|
|
295
|
+
* the slicer does not know — the same guidance rule as
|
|
296
|
+
* {@link slicePrimitiveDocumentation}. Allowing every known name returns the
|
|
297
|
+
* doc byte-identical.
|
|
298
|
+
*
|
|
299
|
+
* With no section header found the doc is returned whole (defensive, as the
|
|
300
|
+
* markdown slicer does when it finds no `### ` sections).
|
|
301
|
+
*/
|
|
302
|
+
export function slicePrimitiveDocumentationTs(
|
|
303
|
+
fullDoc: string,
|
|
304
|
+
allowlist: readonly string[],
|
|
305
|
+
): string {
|
|
306
|
+
const allow = new Set(allowlist);
|
|
307
|
+
const lines = fullDoc.split("\n");
|
|
308
|
+
let sawHeader = false;
|
|
309
|
+
let keep = true;
|
|
310
|
+
const kept: string[] = [];
|
|
311
|
+
for (const line of lines) {
|
|
312
|
+
const header = TS_SECTION_HEADER.exec(line);
|
|
313
|
+
if (header !== null) {
|
|
314
|
+
sawHeader = true;
|
|
315
|
+
const name = header[1]!;
|
|
316
|
+
keep = !KNOWN_PRIMITIVE_SECTIONS.has(name) || allow.has(name);
|
|
317
|
+
} else if (TS_SECTION_BANNER.test(line)) {
|
|
318
|
+
keep = true;
|
|
319
|
+
}
|
|
320
|
+
if (keep) kept.push(line);
|
|
321
|
+
}
|
|
322
|
+
return sawHeader ? kept.join("\n") : fullDoc;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/** `// Button — A clickable button …`: the line that opens a TS-doc section. */
|
|
326
|
+
const TS_SECTION_HEADER = /^\/\/ ([A-Za-z]+) — /;
|
|
327
|
+
/** `// ═══…`: a category banner between TS-doc sections; always kept. */
|
|
328
|
+
const TS_SECTION_BANNER = /^\/\/ ═/;
|