@ggui-ai/ui-gen 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/generation-dispatch.d.ts +1 -1
- package/dist/adapters/generation-dispatch.js +55 -17
- package/dist/adapters/generation-dispatch.js.map +1 -1
- package/dist/adapters/index.js +1 -1
- package/dist/adapters/index.js.map +1 -1
- package/dist/advanced/index.d.ts +3 -3
- package/dist/advanced/index.js +66 -18
- package/dist/advanced/index.js.map +1 -1
- package/dist/blueprint-validator.js +37 -7
- package/dist/blueprint-validator.js.map +1 -1
- package/dist/boilerplate.js +1 -0
- package/dist/boilerplate.js.map +1 -1
- package/dist/check/index.d.ts +1 -1
- package/dist/check/index.js.map +1 -1
- package/dist/{create-ui-generator-BEkzEtNz.d.ts → create-ui-generator-D0kbWwln.d.ts} +1 -1
- package/dist/{dispatch-DJNpQ7DI.d.ts → dispatch-wxaIMwZt.d.ts} +1 -1
- package/dist/evaluation/axis-checks/index.d.ts +3 -3
- package/dist/evaluation/axis-checks/index.js +19 -4
- package/dist/evaluation/axis-checks/index.js.map +1 -1
- package/dist/evaluation/axis-checks/registry.d.ts +1 -1
- package/dist/evaluation/axis-checks/registry.js +19 -4
- package/dist/evaluation/axis-checks/registry.js.map +1 -1
- package/dist/evaluation/index.d.ts +3 -3
- package/dist/evaluation/index.js +21 -6
- package/dist/evaluation/index.js.map +1 -1
- package/dist/harness/check/runtime-render/index.d.ts +4 -4
- package/dist/harness/check/runtime-render/index.js +26 -6
- package/dist/harness/check/runtime-render/index.js.map +1 -1
- package/dist/harness/check/runtime-render/render-check-worker.js.map +1 -1
- package/dist/harness/index.d.ts +1 -1
- package/dist/harness/index.js +3 -2
- package/dist/harness/index.js.map +1 -1
- package/dist/harness/result-types.d.ts +1 -1
- package/dist/harness/runtime.d.ts +1 -1
- package/dist/harness/runtime.js +2 -1
- package/dist/harness/runtime.js.map +1 -1
- package/dist/harness/types-public.d.ts +9 -5
- package/dist/harness/types-public.js +1 -0
- package/dist/harness/types-public.js.map +1 -1
- package/dist/harness/validator-trace-sink.d.ts +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +55 -17
- package/dist/index.js.map +1 -1
- package/dist/patch.d.ts +1 -1
- package/dist/{render-check-C0CqVceF.d.ts → render-check-Blwj8gB6.d.ts} +18 -0
- package/dist/tools/get-primitives-ts.d.ts +1 -1
- package/dist/tools/get-primitives-ts.js +1 -1
- package/dist/tools/get-primitives-ts.js.map +1 -1
- package/dist/{types-public-mM_-1Yk3.d.ts → types-public-Ow-tNtFE.d.ts} +26 -3
- package/dist/validation/index.d.ts +1 -1
- package/dist/validation/index.js +1 -1
- package/dist/validation/index.js.map +1 -1
- package/dist/workflows.d.ts +1 -1
- package/package.json +12 -12
- package/src/advanced/generator.ts +16 -1
- package/src/blueprint-validator.ts +21 -6
- package/src/boilerplate/hard-sections.ts +1 -0
- package/src/coding-agent/tools.ts +6 -1
- package/src/evaluation/axis-checks/checks/state-payload.ts +29 -4
- package/src/evaluation/types-public.ts +26 -2
- package/src/harness/check/runtime-render/adapter.ts +19 -0
- package/src/harness/check/runtime-render/render-check-host.ts +31 -9
- package/src/harness/check/runtime-render/render-check.ts +19 -0
- package/src/harness/coding/run-eval-round.ts +2 -0
- package/src/harness/llm-router.ts +3 -3
- package/src/harness/types-public.ts +9 -5
- package/src/tools/get-primitives-ts.ts +1 -1
- package/src/validation/primitives.ts +1 -1
package/dist/workflows.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { C as Classification } from './axes-CzLEMDeB.js';
|
|
|
2
2
|
import { Workflow } from './harness/types-public.js';
|
|
3
3
|
import '@ggui-ai/protocol';
|
|
4
4
|
import './styling-profile-CxfvV48g.js';
|
|
5
|
-
import './types-public-
|
|
5
|
+
import './types-public-Ow-tNtFE.js';
|
|
6
6
|
import './design-mode-DLufhrNp.js';
|
|
7
7
|
import './types-BOvHNG7K.js';
|
|
8
8
|
import './llm.js';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ggui-ai/ui-gen",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.23.0",
|
|
4
4
|
"description": "UI generation harness for ggui — the LLM system prompt, multi-turn coding loop, and evaluation engine that turn a DataContract into a React component. Implements the UiGenerator contract from @ggui-ai/mcp-server-core; consumed by @ggui-ai/mcp-server.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"keywords": [
|
|
@@ -189,9 +189,9 @@
|
|
|
189
189
|
"puppeteer-core": "^24.40.0 || ^25.0.0",
|
|
190
190
|
"react": "^18.0.0 || ^19.0.0",
|
|
191
191
|
"react-dom": "^18.0.0 || ^19.0.0",
|
|
192
|
-
"@ggui-ai/mcp-server-core": "0.
|
|
193
|
-
"@ggui-ai/
|
|
194
|
-
"@ggui-ai/
|
|
192
|
+
"@ggui-ai/mcp-server-core": "0.23.0",
|
|
193
|
+
"@ggui-ai/protocol": "0.23.0",
|
|
194
|
+
"@ggui-ai/project-config": "0.23.0"
|
|
195
195
|
},
|
|
196
196
|
"peerDependenciesMeta": {
|
|
197
197
|
"@google/adk": {
|
|
@@ -233,11 +233,11 @@
|
|
|
233
233
|
"openai": "^7.9.0",
|
|
234
234
|
"typescript": "^5.0.0",
|
|
235
235
|
"zod": "^4.3.6",
|
|
236
|
-
"@ggui-ai/design": "0.
|
|
237
|
-
"@ggui-ai/
|
|
238
|
-
"@ggui-ai/
|
|
239
|
-
"@ggui-ai/
|
|
240
|
-
"@ggui-ai/
|
|
236
|
+
"@ggui-ai/design": "0.23.0",
|
|
237
|
+
"@ggui-ai/gadgets": "0.23.0",
|
|
238
|
+
"@ggui-ai/sandbox": "0.23.0",
|
|
239
|
+
"@ggui-ai/wire": "0.23.0",
|
|
240
|
+
"@ggui-ai/ui-visual-tester": "0.23.0"
|
|
241
241
|
},
|
|
242
242
|
"devDependencies": {
|
|
243
243
|
"@google/adk": "^2.0.0",
|
|
@@ -252,9 +252,9 @@
|
|
|
252
252
|
"react-dom": "^19.0.0",
|
|
253
253
|
"tsup": "^8.5.0",
|
|
254
254
|
"vitest": "^3.2.6",
|
|
255
|
-
"@ggui-ai/mcp-server-core": "0.
|
|
256
|
-
"@ggui-ai/project-config": "0.
|
|
257
|
-
"@ggui-ai/protocol": "0.
|
|
255
|
+
"@ggui-ai/mcp-server-core": "0.23.0",
|
|
256
|
+
"@ggui-ai/project-config": "0.23.0",
|
|
257
|
+
"@ggui-ai/protocol": "0.23.0"
|
|
258
258
|
},
|
|
259
259
|
"engines": {
|
|
260
260
|
"node": ">=22.0.0"
|
|
@@ -349,6 +349,18 @@ async function runFastStage(
|
|
|
349
349
|
...(input.contract ? { contract: input.contract } : {}),
|
|
350
350
|
};
|
|
351
351
|
const result = await runRenderCheck(checkInput);
|
|
352
|
+
if (result.incomplete !== undefined) {
|
|
353
|
+
// ggui#1299: the check ran out of wall-clock time, so there is no
|
|
354
|
+
// evidence either way — the same posture as an unverified check
|
|
355
|
+
// (non-failing, the unverified score). The slow stage still gates.
|
|
356
|
+
return {
|
|
357
|
+
stage: 'fast',
|
|
358
|
+
ok: true,
|
|
359
|
+
score: UNVERIFIED_FAST_SCORE,
|
|
360
|
+
diagnostics: [],
|
|
361
|
+
durationMs: Date.now() - start,
|
|
362
|
+
};
|
|
363
|
+
}
|
|
352
364
|
const diagnostics: StageDiagnostic[] = buildFastStageComplaints(
|
|
353
365
|
result.issues as readonly RenderCheckIssue[],
|
|
354
366
|
);
|
|
@@ -405,11 +417,14 @@ async function runSlowStage(
|
|
|
405
417
|
* job is just to land above or below it. Per-issue gradations matter
|
|
406
418
|
* less than getting the pass/fail boundary right.
|
|
407
419
|
*/
|
|
420
|
+
/** Fast-stage score when nothing failed but something went unverified. */
|
|
421
|
+
const UNVERIFIED_FAST_SCORE = 0.85;
|
|
422
|
+
|
|
408
423
|
function scoreFromRenderCheck(issues: readonly RenderCheckIssue[]): number {
|
|
409
424
|
const failed = issues.filter((i) => i.outcome === 'failed').length;
|
|
410
425
|
const unverified = issues.filter((i) => i.outcome === 'unverified').length;
|
|
411
426
|
if (failed === 0 && unverified === 0) return 1.0;
|
|
412
|
-
if (failed === 0) return
|
|
427
|
+
if (failed === 0) return UNVERIFIED_FAST_SCORE;
|
|
413
428
|
if (failed === 1) return 0.4;
|
|
414
429
|
if (failed === 2) return 0.3;
|
|
415
430
|
return 0.2;
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
|
|
24
24
|
import type { DataContract, JsonObject } from '@ggui-ai/protocol';
|
|
25
25
|
import { DEFAULT_RUNTIME_RENDER_CHECK } from './harness/check/runtime-render/index.js';
|
|
26
|
-
import type { EvalIssue } from './evaluation/types-public.js';
|
|
26
|
+
import type { EvalIssue, RuntimeProbeStatus } from './evaluation/types-public.js';
|
|
27
27
|
|
|
28
28
|
/**
|
|
29
29
|
* Caller-supplied blueprint payload. `contract` is `unknown` at the
|
|
@@ -74,6 +74,23 @@ export interface ValidateBlueprintInput {
|
|
|
74
74
|
* the absence cleanly (selfCheck skips prop-coverage warnings; runtime
|
|
75
75
|
* probe skip-warns).
|
|
76
76
|
*/
|
|
77
|
+
/**
|
|
78
|
+
* The warning code for a probe that did not produce evidence. Exhaustive over
|
|
79
|
+
* the did-not-run statuses, so a new status is a compile error here instead of
|
|
80
|
+
* silently reading as whichever arm a ternary falls through to (ggui#1299:
|
|
81
|
+
* `timed-out` would otherwise have been labelled `probe-not-applicable`).
|
|
82
|
+
*/
|
|
83
|
+
function probeDidNotRunCode(status: Exclude<RuntimeProbeStatus, 'ran'>): string {
|
|
84
|
+
switch (status) {
|
|
85
|
+
case 'infra-skipped':
|
|
86
|
+
return 'runtime:probe-infra-failure';
|
|
87
|
+
case 'not-applicable':
|
|
88
|
+
return 'runtime:probe-not-applicable';
|
|
89
|
+
case 'timed-out':
|
|
90
|
+
return 'runtime:probe-timeout';
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
77
94
|
function asContract(x: RawContract): DataContract | undefined {
|
|
78
95
|
if (typeof x !== 'object' || x === null) return undefined;
|
|
79
96
|
return x as DataContract;
|
|
@@ -247,17 +264,15 @@ async function runtimeTier(input: RuntimeTierInput): Promise<RuntimeResult> {
|
|
|
247
264
|
fixtureProps: asJsonObject(input.fixtureProps),
|
|
248
265
|
});
|
|
249
266
|
if (outcome.status !== 'ran') {
|
|
250
|
-
// Probe
|
|
267
|
+
// Probe produced no evidence — infra failure, nothing to probe, or it
|
|
268
|
+
// ran out of wall-clock time before finishing.
|
|
251
269
|
// Surface as a warning rather than a hard error so a flaky probe
|
|
252
270
|
// doesn't permanently block legitimate blueprints — Claude can
|
|
253
271
|
// retry, and host-level logs record the diagnostic for ops review.
|
|
254
272
|
warnings.push({
|
|
255
273
|
_kind: 'warning',
|
|
256
274
|
tier: 'runtime',
|
|
257
|
-
code:
|
|
258
|
-
outcome.status === 'infra-skipped'
|
|
259
|
-
? 'runtime:probe-infra-failure'
|
|
260
|
-
: 'runtime:probe-not-applicable',
|
|
275
|
+
code: probeDidNotRunCode(outcome.status),
|
|
261
276
|
message: `Runtime probe did not run (${outcome.status}): ${outcome.reason ?? 'no reason recorded'}`,
|
|
262
277
|
});
|
|
263
278
|
return { errors, warnings };
|
|
@@ -250,6 +250,7 @@ export const FRAME_SIZING = `## Sizing — the frame owns the height and the wid
|
|
|
250
250
|
The host sizes the iframe; the component never sizes itself to the viewport. NEVER write \`100vh\` / \`100dvh\` / \`100svh\` (or an \`h-screen\` / \`min-h-screen\` class) as a height on any element:
|
|
251
251
|
- An inline card takes its natural height. A content-sized frame is measured FROM the content, so a \`100vh\` root can never shrink and grows the frame on every re-measure.
|
|
252
252
|
- Under fullscreen the frame already stretches your root to its full height. Centre inside it with flex (\`display: flex; flex-direction: column; justify-content: center\`), never with a viewport height.
|
|
253
|
+
- Under fullscreen the frame also keeps a 16 px inset between your content and the panel's edge — leave it. A \`surface="hero"\` band that OPENS the card runs edge to edge on its own. Any other band that is MEANT to run edge to edge (a full-width strip of rows, a closing action bar) takes \`bleed\` (\`<Box surface="sunken" bleed>…</Box>\`); only such a band should, and \`bleed\` is ignored outside fullscreen.
|
|
253
254
|
- To fill a parent that has a height use \`height: 100%\` or \`flex: 1\`; a viewport unit is only ever a \`max-height\` cap on a scroll region.
|
|
254
255
|
|
|
255
256
|
The width works the same way, and the mistake is the mirror image: **your outermost element FILLS the frame; a width cap belongs INSIDE it, on the column that holds text.**
|
|
@@ -416,7 +416,12 @@ export async function executeTool(
|
|
|
416
416
|
changes: normalizedChanges,
|
|
417
417
|
});
|
|
418
418
|
if (!patchResult.ok) {
|
|
419
|
-
|
|
419
|
+
// ggui#1261: the engine refused the patch (overlap, line bounds, missing range) before
|
|
420
|
+
// anything applied. Say so in the log, and call it what it is — PATCH_INVALID, like every
|
|
421
|
+
// other structural refusal in this tool — so the turn is never counted as a self-check
|
|
422
|
+
// failure that did not run.
|
|
423
|
+
console.log(`[coding-agent] apply_changes: REJECTED | ${patchResult.error}`);
|
|
424
|
+
return { result: `PATCH_INVALID: ${patchResult.error}`, error: true };
|
|
420
425
|
}
|
|
421
426
|
const candidate = patchResult.sourceAfter ?? currentFile;
|
|
422
427
|
|
|
@@ -79,16 +79,41 @@ function runStateCoversPayload(input: AxisCheckInput): EvalIssue[] {
|
|
|
79
79
|
return issues;
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
+
/**
|
|
83
|
+
* Whether a `useState` initializer reads `props.<name>`: directly, or through a local bound from
|
|
84
|
+
* it (`const initial = normalize(props.initialProfile)` then `useState(initial.name)`, or a
|
|
85
|
+
* destructure `const { name } = props.initialProfile ?? {}` then `useState(name)`). The property
|
|
86
|
+
* this check guards is "form state is seeded from the prop", not a spelling: ggui#1261's Exp 011
|
|
87
|
+
* cell re-seeded correctly through a normalized local, was refused, and spent a turn restoring
|
|
88
|
+
* the literal form.
|
|
89
|
+
*/
|
|
90
|
+
function seedsFromProp(src: string, name: string): boolean {
|
|
91
|
+
const propRef = `props\\??\\.${name}\\b`;
|
|
92
|
+
const initializerReads = (ref: string): boolean =>
|
|
93
|
+
new RegExp(`useState(?:<[^>]*>)?\\s*\\([\\s\\S]{0,400}?${ref}`).test(src);
|
|
94
|
+
if (initializerReads(propRef)) return true;
|
|
95
|
+
const locals = new Set<string>();
|
|
96
|
+
for (const m of src.matchAll(new RegExp(`(?:const|let|var)\\s+(\\w+)\\s*(?::[^=]{1,80})?=\\s*[^;]{0,300}?${propRef}`, "g"))) {
|
|
97
|
+
const id = m[1];
|
|
98
|
+
if (id !== undefined) locals.add(id);
|
|
99
|
+
}
|
|
100
|
+
for (const m of src.matchAll(new RegExp(`(?:const|let|var)\\s*\\{([^}]{1,400})\\}\\s*=\\s*[^;]{0,300}?${propRef}`, "g"))) {
|
|
101
|
+
for (const part of (m[1] ?? "").split(",")) {
|
|
102
|
+
const id = part.split(":").pop()?.split("=")[0]?.trim();
|
|
103
|
+
if (id !== undefined && /^\w+$/.test(id)) locals.add(id);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
for (const id of locals) if (initializerReads(`\\b${id}\\b`)) return true;
|
|
107
|
+
return false;
|
|
108
|
+
}
|
|
109
|
+
|
|
82
110
|
function runInitialValuesSeeded(input: AxisCheckInput): EvalIssue[] {
|
|
83
111
|
if (input.compiledCode === null) return [];
|
|
84
112
|
const src = input.sourceCode;
|
|
85
113
|
const initialProps = getInitialValuePropNames(input.contract);
|
|
86
114
|
const issues: EvalIssue[] = [];
|
|
87
115
|
for (const name of initialProps) {
|
|
88
|
-
|
|
89
|
-
`useState(?:<[^>]*>)?\\s*\\([\\s\\S]{0,400}?props\\.${name}\\b`,
|
|
90
|
-
);
|
|
91
|
-
if (re.test(src)) continue;
|
|
116
|
+
if (seedsFromProp(src, name)) continue;
|
|
92
117
|
issues.push(
|
|
93
118
|
mkIssue(
|
|
94
119
|
"state.payload.initial_values_seeded",
|
|
@@ -74,8 +74,28 @@ export interface EvalIssue {
|
|
|
74
74
|
* consumers MUST NOT count it as a pass.
|
|
75
75
|
* - `not-applicable`— nothing to probe (no compiled code, no contract
|
|
76
76
|
* surface, or no runtimeRender check configured).
|
|
77
|
+
* - `timed-out` — probe started but did not finish inside its
|
|
78
|
+
* wall-clock bound (ggui#1299). Carries ZERO
|
|
79
|
+
* evidence about the component: on a contended
|
|
80
|
+
* host an ordinary component crosses the bound.
|
|
81
|
+
* Consumers MUST NOT count it as a pass, and MUST
|
|
82
|
+
* NOT count it as a crash either. `elapsedMs` and
|
|
83
|
+
* `hostLoad` say how long it ran and how busy the
|
|
84
|
+
* host was, so a reader can tell the two cases apart.
|
|
77
85
|
*/
|
|
78
|
-
export type RuntimeProbeStatus = "ran" | "infra-skipped" | "not-applicable";
|
|
86
|
+
export type RuntimeProbeStatus = "ran" | "infra-skipped" | "not-applicable" | "timed-out";
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* The host's 1-minute load average at a probe's start and end, beside the
|
|
90
|
+
* number of CPUs the process can use (ggui#1299). A load well above
|
|
91
|
+
* `cores` means the probe competed for CPU; a wall-clock bound crossed
|
|
92
|
+
* there says more about the host than about the component.
|
|
93
|
+
*/
|
|
94
|
+
export interface ProbeHostLoad {
|
|
95
|
+
readonly start: number;
|
|
96
|
+
readonly end: number;
|
|
97
|
+
readonly cores: number;
|
|
98
|
+
}
|
|
79
99
|
|
|
80
100
|
/**
|
|
81
101
|
* Probe execution meta stamped onto an `EvalResult` at the exit-decision
|
|
@@ -85,8 +105,12 @@ export type RuntimeProbeStatus = "ran" | "infra-skipped" | "not-applicable";
|
|
|
85
105
|
*/
|
|
86
106
|
export interface RuntimeProbeMeta {
|
|
87
107
|
readonly status: RuntimeProbeStatus;
|
|
88
|
-
/** Populated for `infra-skipped` / `not-applicable` — why the probe didn't
|
|
108
|
+
/** Populated for `infra-skipped` / `not-applicable` / `timed-out` — why the probe didn't finish. */
|
|
89
109
|
readonly reason?: string;
|
|
110
|
+
/** `timed-out` only: how long the probe ran before it was stopped. */
|
|
111
|
+
readonly elapsedMs?: number;
|
|
112
|
+
/** Host load around the probe, when it ran isolated (absent for an in-process probe). */
|
|
113
|
+
readonly hostLoad?: ProbeHostLoad;
|
|
90
114
|
}
|
|
91
115
|
|
|
92
116
|
/**
|
|
@@ -54,11 +54,30 @@ export const DEFAULT_RUNTIME_RENDER_CHECK: RuntimeRenderCheck = {
|
|
|
54
54
|
return { status: "infra-skipped", issues: [], reason: message };
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
const hostLoad = result.stats.hostLoad;
|
|
58
|
+
const load = hostLoad !== undefined ? { hostLoad } : {};
|
|
59
|
+
|
|
60
|
+
// ggui#1299: a check that ran out of wall-clock time is not a verdict on
|
|
61
|
+
// the component. It becomes the `timed-out` status (did-not-run, never a
|
|
62
|
+
// pass, never a crash), never an eval issue — so the coding agent is never
|
|
63
|
+
// told to fix a crash that did not happen.
|
|
64
|
+
if (result.incomplete !== undefined) {
|
|
65
|
+
const { elapsedMs, boundMs } = result.incomplete;
|
|
66
|
+
const loadNote =
|
|
67
|
+
hostLoad !== undefined
|
|
68
|
+
? `; host load ${hostLoad.start.toFixed(1)} → ${hostLoad.end.toFixed(1)} on ${hostLoad.cores} CPUs`
|
|
69
|
+
: "";
|
|
70
|
+
const reason = `render check did not finish within ${boundMs} ms (stopped at ${elapsedMs} ms)${loadNote}`;
|
|
71
|
+
console.warn(`[runtime-render] probe timed out — ${reason}`);
|
|
72
|
+
return { status: "timed-out", issues: [], reason, elapsedMs, ...load };
|
|
73
|
+
}
|
|
74
|
+
|
|
57
75
|
return {
|
|
58
76
|
status: "ran",
|
|
59
77
|
issues: result.issues
|
|
60
78
|
.map(toEvalIssue)
|
|
61
79
|
.filter((x): x is EvalIssue => x !== null),
|
|
80
|
+
...load,
|
|
62
81
|
};
|
|
63
82
|
},
|
|
64
83
|
};
|
|
@@ -11,13 +11,24 @@
|
|
|
11
11
|
* `unverified`, and nothing throws):
|
|
12
12
|
*
|
|
13
13
|
* - worker exit 0 + parseable verdict → the verdict, verbatim.
|
|
14
|
-
* - timeout → `
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
14
|
+
* - timeout → `incomplete` (ggui#1299): the check
|
|
15
|
+
* did not finish, which is NOT a verdict on the component. The bound
|
|
16
|
+
* is wall-clock, and on a contended host an ordinary component
|
|
17
|
+
* crosses it; mapping it to `failed` reported crashes that never
|
|
18
|
+
* happened. It carries its elapsed ms, and the host load (below)
|
|
19
|
+
* says how busy the host was. React's own runaway guards ("Too many
|
|
20
|
+
* re-renders", "Maximum update depth exceeded") throw, so they still
|
|
21
|
+
* reach `render-no-throw` as `failed`; what now reads `incomplete`
|
|
22
|
+
* is a check that ran out of wall-clock time, whatever the cause.
|
|
23
|
+
* - overflow → `failed` (a pathological logging or
|
|
24
|
+
* error loop; output volume does not depend on host load).
|
|
25
|
+
* - spawn-error / non-zero exit / unparseable stdout
|
|
18
26
|
* → `unverified` (the harness could not
|
|
19
27
|
* run the check; never blame the component for our plumbing).
|
|
20
28
|
*
|
|
29
|
+
* Every isolated check records the host's 1-minute load average at its
|
|
30
|
+
* start and end, and the CPUs available, on `stats.hostLoad`.
|
|
31
|
+
*
|
|
21
32
|
* Spawn resolution copies the proven `src/tools/render-check.ts`
|
|
22
33
|
* pattern: prefer the built `dist/.../render-check-worker.js`; in
|
|
23
34
|
* dev/test (vitest imports `src/`), fall back to `node --import tsx`
|
|
@@ -29,9 +40,11 @@
|
|
|
29
40
|
*/
|
|
30
41
|
import { existsSync } from 'node:fs';
|
|
31
42
|
import { createRequire } from 'node:module';
|
|
43
|
+
import { availableParallelism, loadavg } from 'node:os';
|
|
32
44
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
33
45
|
import { runSandboxed } from '@ggui-ai/sandbox';
|
|
34
46
|
import type { SandboxResult } from '@ggui-ai/sandbox';
|
|
47
|
+
import type { ProbeHostLoad } from '../../../evaluation/types-public.js';
|
|
35
48
|
import type { RenderCheckResult, RunRenderCheckInput } from './render-check.js';
|
|
36
49
|
|
|
37
50
|
/**
|
|
@@ -121,10 +134,12 @@ export function mapSandboxResultToCheckResult(
|
|
|
121
134
|
t0: number,
|
|
122
135
|
): RenderCheckResult {
|
|
123
136
|
if (result.outcome === 'timeout') {
|
|
124
|
-
return
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
137
|
+
return {
|
|
138
|
+
ok: false,
|
|
139
|
+
issues: [],
|
|
140
|
+
incomplete: { kind: 'timeout', elapsedMs: result.durationMs, boundMs: CHECK_TIMEOUT_MS },
|
|
141
|
+
stats: { actionsChecked: 0, streamsChecked: 0, renderMs: Date.now() - t0 },
|
|
142
|
+
};
|
|
128
143
|
}
|
|
129
144
|
if (result.outcome === 'overflow-stdout' || result.outcome === 'overflow-stderr') {
|
|
130
145
|
return failed(
|
|
@@ -184,6 +199,7 @@ export async function runRenderCheckViaWorker(
|
|
|
184
199
|
return unverified(err instanceof Error ? err.message : String(err), t0);
|
|
185
200
|
}
|
|
186
201
|
|
|
202
|
+
const loadAtStart = loadavg()[0] ?? 0;
|
|
187
203
|
const result = await runSandboxed({
|
|
188
204
|
command: spawn.command,
|
|
189
205
|
args: spawn.args,
|
|
@@ -197,5 +213,11 @@ export async function runRenderCheckViaWorker(
|
|
|
197
213
|
env: { NODE_ENV: process.env.NODE_ENV ?? 'production' },
|
|
198
214
|
});
|
|
199
215
|
|
|
200
|
-
|
|
216
|
+
const hostLoad: ProbeHostLoad = {
|
|
217
|
+
start: loadAtStart,
|
|
218
|
+
end: loadavg()[0] ?? 0,
|
|
219
|
+
cores: availableParallelism(),
|
|
220
|
+
};
|
|
221
|
+
const mapped = mapSandboxResultToCheckResult(result, t0);
|
|
222
|
+
return { ...mapped, stats: { ...mapped.stats, hostLoad } };
|
|
201
223
|
}
|
|
@@ -45,6 +45,7 @@ import {
|
|
|
45
45
|
type ParentLink,
|
|
46
46
|
} from "./host-boundary.js";
|
|
47
47
|
import { hostGlobals, openRecord } from "../../../internal/open-record.js";
|
|
48
|
+
import type { ProbeHostLoad } from "../../../evaluation/types-public.js";
|
|
48
49
|
import { primeInputs } from "@ggui-ai/ui-visual-tester/prime-inputs";
|
|
49
50
|
|
|
50
51
|
/**
|
|
@@ -118,13 +119,31 @@ export type InputPrimingDiagnostic =
|
|
|
118
119
|
| { readonly primed: number }
|
|
119
120
|
| { readonly error: string };
|
|
120
121
|
|
|
122
|
+
/**
|
|
123
|
+
* The check started but did not finish (ggui#1299). The only kind is a
|
|
124
|
+
* wall-clock timeout in the isolated worker. It is not a verdict on the
|
|
125
|
+
* component: `issues` is empty and `ok` is false because nothing was
|
|
126
|
+
* verified, never because something failed.
|
|
127
|
+
*/
|
|
128
|
+
export interface RenderCheckIncomplete {
|
|
129
|
+
readonly kind: "timeout";
|
|
130
|
+
/** How long the worker ran before it was stopped. */
|
|
131
|
+
readonly elapsedMs: number;
|
|
132
|
+
/** The wall-clock bound it crossed. */
|
|
133
|
+
readonly boundMs: number;
|
|
134
|
+
}
|
|
135
|
+
|
|
121
136
|
export interface RenderCheckResult {
|
|
122
137
|
readonly ok: boolean;
|
|
123
138
|
readonly issues: readonly RenderCheckIssue[];
|
|
139
|
+
/** Present only when the check did not finish; see {@link RenderCheckIncomplete}. */
|
|
140
|
+
readonly incomplete?: RenderCheckIncomplete;
|
|
124
141
|
readonly stats: {
|
|
125
142
|
readonly actionsChecked: number;
|
|
126
143
|
readonly streamsChecked: number;
|
|
127
144
|
readonly renderMs: number;
|
|
145
|
+
/** Host load around an isolated check (absent in-process). */
|
|
146
|
+
readonly hostLoad?: ProbeHostLoad;
|
|
128
147
|
};
|
|
129
148
|
}
|
|
130
149
|
|
|
@@ -359,6 +359,8 @@ async function runProbeAtExit(input: {
|
|
|
359
359
|
const meta: RuntimeProbeMeta = {
|
|
360
360
|
status: outcome.status,
|
|
361
361
|
...(outcome.reason !== undefined ? { reason: outcome.reason } : {}),
|
|
362
|
+
...(outcome.elapsedMs !== undefined ? { elapsedMs: outcome.elapsedMs } : {}),
|
|
363
|
+
...(outcome.hostLoad !== undefined ? { hostLoad: outcome.hostLoad } : {}),
|
|
362
364
|
};
|
|
363
365
|
if (outcome.status !== "ran") {
|
|
364
366
|
return { fired: false, recoverableFail: false, probeIssues: [], meta };
|
|
@@ -875,8 +875,8 @@ export class AnthropicAgent extends LLMAgent {
|
|
|
875
875
|
: toolChoice;
|
|
876
876
|
if (appliedToolChoice !== toolChoice) {
|
|
877
877
|
console.warn(
|
|
878
|
-
`[anthropic] callTools: ${normalizeAnthropicModelId(resolvedModel)}
|
|
879
|
-
`caller's 'required' downgraded to 'auto'`,
|
|
878
|
+
`[anthropic] callTools: ${normalizeAnthropicModelId(resolvedModel)} would reject a forced tool_choice (the API answers 400); ` +
|
|
879
|
+
`caller's 'required' downgraded to 'auto' before sending`,
|
|
880
880
|
);
|
|
881
881
|
}
|
|
882
882
|
const response = await this.apiCall(() =>
|
|
@@ -2128,7 +2128,7 @@ export class OpenRouterAgent extends LLMAgent {
|
|
|
2128
2128
|
requested === 'required' && anthropicRejectsForcedToolChoice(resolved) ? 'auto' : requested;
|
|
2129
2129
|
if (appliedToolChoice !== requested) {
|
|
2130
2130
|
console.warn(
|
|
2131
|
-
`[openrouter] callTools: ${resolved}
|
|
2131
|
+
`[openrouter] callTools: ${resolved} would reject a forced tool_choice (the API answers 400); caller's 'required' downgraded to 'auto' before sending`,
|
|
2132
2132
|
);
|
|
2133
2133
|
}
|
|
2134
2134
|
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
import type { GenerationProfileInput } from "../boilerplate/styling-profile.js";
|
|
22
22
|
import type { GadgetDescriptor, DataContract, JsonValue } from "@ggui-ai/protocol";
|
|
23
23
|
import type { Classification } from "../classifier/axes.js";
|
|
24
|
-
import type { EvalIssue, EvalTier, AxisCheck, RuntimeProbeStatus } from "../evaluation/types-public.js";
|
|
24
|
+
import type { EvalIssue, EvalTier, AxisCheck, ProbeHostLoad, RuntimeProbeStatus } from "../evaluation/types-public.js";
|
|
25
25
|
import type { LLMToolDef } from "../llm.js";
|
|
26
26
|
import type { CacheTier, HarnessFragment } from "../fragments/index.js";
|
|
27
27
|
import type { HarnessPolicy, ProcessMode } from "../policy.js";
|
|
@@ -140,14 +140,18 @@ export interface LLMEvaluator {
|
|
|
140
140
|
export interface RuntimeRenderOutcome {
|
|
141
141
|
/**
|
|
142
142
|
* Whether the probe actually executed. `issues` is only probe
|
|
143
|
-
* evidence when `status === "ran"` — an `infra-skipped
|
|
144
|
-
* `not-applicable` outcome carries zero evidence and
|
|
145
|
-
* scored as a pass by any consumer.
|
|
143
|
+
* evidence when `status === "ran"` — an `infra-skipped`,
|
|
144
|
+
* `not-applicable` or `timed-out` outcome carries zero evidence and
|
|
145
|
+
* MUST NOT be scored as a pass by any consumer.
|
|
146
146
|
*/
|
|
147
147
|
readonly status: RuntimeProbeStatus;
|
|
148
148
|
readonly issues: readonly EvalIssue[];
|
|
149
|
-
/** Populated for `infra-skipped` / `not-applicable` — why the probe didn't
|
|
149
|
+
/** Populated for `infra-skipped` / `not-applicable` / `timed-out` — why the probe didn't finish. */
|
|
150
150
|
readonly reason?: string;
|
|
151
|
+
/** `timed-out` only: how long the probe ran before it was stopped. */
|
|
152
|
+
readonly elapsedMs?: number;
|
|
153
|
+
/** Host load around the probe, when it ran isolated (absent for an in-process probe). */
|
|
154
|
+
readonly hostLoad?: ProbeHostLoad;
|
|
151
155
|
}
|
|
152
156
|
|
|
153
157
|
export interface RuntimeRenderCheck {
|