@ggui-ai/ui-gen 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/generation-dispatch.d.ts +10 -0
- package/dist/adapters/generation-dispatch.js +172 -1392
- package/dist/adapters/generation-dispatch.js.map +1 -1
- package/dist/advanced/index.d.ts +1 -1
- package/dist/advanced/index.js +191 -1403
- package/dist/advanced/index.js.map +1 -1
- package/dist/boilerplate.js +3 -2
- package/dist/boilerplate.js.map +1 -1
- package/dist/compose.js +5 -5
- package/dist/compose.js.map +1 -1
- package/dist/{create-ui-generator-GX7tvbyH.d.ts → create-ui-generator-B9kV4U9h.d.ts} +13 -0
- package/dist/evaluation/axis-checks/index.js +32 -1
- package/dist/evaluation/axis-checks/index.js.map +1 -1
- package/dist/evaluation/axis-checks/registry.js +32 -1
- package/dist/evaluation/axis-checks/registry.js.map +1 -1
- package/dist/evaluation/index.js +57 -12
- package/dist/evaluation/index.js.map +1 -1
- package/dist/fragments/index.js +5 -5
- package/dist/fragments/index.js.map +1 -1
- package/dist/harness/index.d.ts +4 -125
- package/dist/harness/index.js +34 -19
- package/dist/harness/index.js.map +1 -1
- package/dist/harness/runtime.d.ts +9 -0
- package/dist/harness/runtime.js +3 -2
- package/dist/harness/runtime.js.map +1 -1
- package/dist/harness/types-public.js +8 -7
- package/dist/harness/types-public.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +191 -1403
- package/dist/index.js.map +1 -1
- package/dist/llm-router-1TyI0_TV.d.ts +156 -0
- package/dist/providers/index.d.ts +35 -93
- package/dist/providers/index.js +13 -8
- package/dist/providers/index.js.map +1 -1
- package/dist/validation/index.js +2 -2
- package/dist/validation/index.js.map +1 -1
- package/package.json +18 -18
- package/src/adapters/generation-dispatch.ts +19 -2
- package/src/boilerplate/system-prompt.ts +3 -2
- package/src/create-ui-generator.ts +44 -11
- package/src/evaluation/axis-checks/checks/state-merge.ts +1 -1
- package/src/evaluation/axis-checks/checks/universal.ts +39 -0
- package/src/evaluation/llm-evaluator.ts +13 -3
- package/src/evaluation/visual-evaluator.ts +5 -1
- package/src/fragments/realtime.ts +3 -3
- package/src/fragments/state.ts +2 -2
- package/src/harness/coding/init-session.ts +4 -1
- package/src/harness/coding/run-eval-round.ts +6 -0
- package/src/harness/llm-router.ts +59 -14
- package/src/harness/runtime.ts +8 -0
- package/src/providers/bedrock.ts +84 -22
- package/src/validation/ui-compiler.ts +2 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ggui-ai/ui-gen",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "UI generation harness for ggui — the LLM system prompt, multi-turn coding loop, and evaluation engine that turn a DataContract into a React component. Implements the UiGenerator contract from @ggui-ai/mcp-server-core; consumed by @ggui-ai/mcp-server.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"keywords": [
|
|
@@ -185,12 +185,12 @@
|
|
|
185
185
|
"peerDependencies": {
|
|
186
186
|
"@google/adk": "^0.5.0",
|
|
187
187
|
"@openai/agents": "^0.7.1",
|
|
188
|
-
"puppeteer-core": "^24.40.0",
|
|
188
|
+
"puppeteer-core": "^24.40.0 || ^25.0.0",
|
|
189
189
|
"react": "^18.0.0 || ^19.0.0",
|
|
190
190
|
"react-dom": "^18.0.0 || ^19.0.0",
|
|
191
|
-
"@ggui-ai/mcp-server-core": "0.
|
|
192
|
-
"@ggui-ai/protocol": "0.
|
|
193
|
-
"@ggui-ai/project-config": "0.
|
|
191
|
+
"@ggui-ai/mcp-server-core": "0.9.0",
|
|
192
|
+
"@ggui-ai/protocol": "0.9.0",
|
|
193
|
+
"@ggui-ai/project-config": "0.9.0"
|
|
194
194
|
},
|
|
195
195
|
"peerDependenciesMeta": {
|
|
196
196
|
"@google/adk": {
|
|
@@ -210,9 +210,9 @@
|
|
|
210
210
|
}
|
|
211
211
|
},
|
|
212
212
|
"dependencies": {
|
|
213
|
-
"@anthropic-ai/bedrock-sdk": "^0.
|
|
214
|
-
"@anthropic-ai/claude-agent-sdk": "0.
|
|
215
|
-
"@anthropic-ai/sdk": "^0.
|
|
213
|
+
"@anthropic-ai/bedrock-sdk": "^0.32.1",
|
|
214
|
+
"@anthropic-ai/claude-agent-sdk": "0.3.229",
|
|
215
|
+
"@anthropic-ai/sdk": "^0.116.0",
|
|
216
216
|
"@google/genai": "^2.8.0",
|
|
217
217
|
"@testing-library/dom": "^10.4.0",
|
|
218
218
|
"@testing-library/react": "^16.0.0",
|
|
@@ -229,28 +229,28 @@
|
|
|
229
229
|
"openai": "^6.29.0",
|
|
230
230
|
"typescript": "^5.0.0",
|
|
231
231
|
"zod": "^4.3.6",
|
|
232
|
-
"@ggui-ai/
|
|
233
|
-
"@ggui-ai/
|
|
234
|
-
"@ggui-ai/
|
|
235
|
-
"@ggui-ai/
|
|
236
|
-
"@ggui-ai/
|
|
232
|
+
"@ggui-ai/design": "0.9.0",
|
|
233
|
+
"@ggui-ai/gadgets": "0.9.0",
|
|
234
|
+
"@ggui-ai/ui-visual-tester": "0.9.0",
|
|
235
|
+
"@ggui-ai/wire": "0.9.0",
|
|
236
|
+
"@ggui-ai/sandbox": "0.9.0"
|
|
237
237
|
},
|
|
238
238
|
"devDependencies": {
|
|
239
239
|
"@google/adk": "^0.5.0",
|
|
240
240
|
"@openai/agents": "^0.7.1",
|
|
241
|
-
"@sparticuz/chromium": "^
|
|
241
|
+
"@sparticuz/chromium": "^149.0.0",
|
|
242
242
|
"@types/eslint": "^9.6.1",
|
|
243
243
|
"@types/node": "^22.0.0",
|
|
244
244
|
"@types/react": "^19.0.0",
|
|
245
245
|
"@types/react-dom": "^19.0.0",
|
|
246
|
-
"puppeteer-core": "^
|
|
246
|
+
"puppeteer-core": "^25.6.0",
|
|
247
247
|
"react": "^19.0.0",
|
|
248
248
|
"react-dom": "^19.0.0",
|
|
249
249
|
"tsup": "^8.5.0",
|
|
250
250
|
"vitest": "^3.2.6",
|
|
251
|
-
"@ggui-ai/
|
|
252
|
-
"@ggui-ai/
|
|
253
|
-
"@ggui-ai/protocol": "0.
|
|
251
|
+
"@ggui-ai/project-config": "0.9.0",
|
|
252
|
+
"@ggui-ai/mcp-server-core": "0.9.0",
|
|
253
|
+
"@ggui-ai/protocol": "0.9.0"
|
|
254
254
|
},
|
|
255
255
|
"engines": {
|
|
256
256
|
"node": ">=20.0.0"
|
|
@@ -53,6 +53,7 @@ import type {
|
|
|
53
53
|
} from "../harness/result-types.js";
|
|
54
54
|
import type { AdapterResult, ProviderName, ToolDefinition } from "./types.js";
|
|
55
55
|
import type { QualityConfig } from "../evaluation/types-public.js";
|
|
56
|
+
import type { AgentConfig } from "../harness/llm-router.js";
|
|
56
57
|
import {
|
|
57
58
|
type GadgetDescriptor,
|
|
58
59
|
type JsonObject,
|
|
@@ -147,6 +148,14 @@ export interface GenerationDispatchParams {
|
|
|
147
148
|
* STDLIB-only / bench / direct callers.
|
|
148
149
|
*/
|
|
149
150
|
gadgetTypes?: Readonly<Record<string, string>>;
|
|
151
|
+
/**
|
|
152
|
+
* Explicit per-call credentials/routing (#484) — see
|
|
153
|
+
* `AgentConfig.routeOverride`. Threaded onto the coding + evaluation
|
|
154
|
+
* agent specs so `resolveSessionAgents` → `createAgent` never falls
|
|
155
|
+
* back to reading `process.env`. Absent (default) preserves
|
|
156
|
+
* existing behavior for callers that don't set it.
|
|
157
|
+
*/
|
|
158
|
+
routeOverride?: AgentConfig["routeOverride"];
|
|
150
159
|
}
|
|
151
160
|
|
|
152
161
|
/**
|
|
@@ -186,9 +195,17 @@ export async function dispatchGeneration(
|
|
|
186
195
|
|
|
187
196
|
const routerProvider = mapProviderForLLMRouter(params.provider);
|
|
188
197
|
const resolvedModel = params.models?.coding ?? params.models?.default ?? params.model;
|
|
189
|
-
const codingAgent = {
|
|
198
|
+
const codingAgent = {
|
|
199
|
+
provider: routerProvider,
|
|
200
|
+
model: resolvedModel,
|
|
201
|
+
routeOverride: params.routeOverride,
|
|
202
|
+
};
|
|
190
203
|
const evaluationAgent = params.models?.evaluation
|
|
191
|
-
? {
|
|
204
|
+
? {
|
|
205
|
+
provider: routerProvider,
|
|
206
|
+
model: params.models.evaluation,
|
|
207
|
+
routeOverride: params.routeOverride,
|
|
208
|
+
}
|
|
192
209
|
: undefined;
|
|
193
210
|
// ModelRoles doesn't currently expose a visualEval slot — falls through
|
|
194
211
|
// to the documented evaluation → coding chain inside resolveSessionAgents.
|
|
@@ -347,13 +347,14 @@ A \`DataContract\` declares everything a GguiSession exchanges with the outside
|
|
|
347
347
|
|
|
348
348
|
## Defensive coding for absent / late-arriving data
|
|
349
349
|
|
|
350
|
-
Props arrive
|
|
350
|
+
Props arrive at mount and may be partial on first render; later prop changes reach the SAME mounted component via \`ggui_amend\` (in-place repaint) on the live channel. Stream channels start empty and fill over time. Context slots start at their declared default (often \`null\`). **Never assume a field exists before you read it.**
|
|
351
351
|
|
|
352
352
|
- **Array iteration**: always default to \`[]\` before \`.map\`/\`.filter\`/\`.length\`. Use \`(props.items ?? []).map(...)\` not \`props.items.map(...)\`. Same for stream.all, stream.latest, etc.
|
|
353
353
|
- **Object access**: optional-chain through nested fields. \`props.user?.name ?? 'Anonymous'\` not \`props.user.name\`.
|
|
354
354
|
- **Number ops**: default before arithmetic. \`(props.count ?? 0) + 1\` not \`props.count + 1\`.
|
|
355
355
|
- **Stream latest**: \`useStream\` returns \`{latest: T | null, all: T[], isComplete: boolean}\`. The default \`all\` is \`[]\` so it's safe to map; \`latest\` is null until the first frame arrives — guard before reading \`.foo\`.
|
|
356
356
|
- **Stream reconciliation**: when a stream event carries an \`action\` discriminant (e.g. \`create | move | edit | delete\`), the channel is a CRUD feed — your handler MUST branch on EVERY value: append on \`create\`, drop on \`delete\`, replace-by-id on \`move\` / \`edit\`. Merging only the "edit" case silently loses created and deleted items. Reconcile into the SAME state that seeds from \`props\` (e.g. \`useState(() => props.tasks ?? [])\`) so the seed data and the live feed render as one list — and handle an event whose id is not yet present (a \`create\` for an unknown item) by inserting it, not ignoring it.
|
|
357
|
+
- **Props re-sync on repaint**: props the agent owns are the source of truth for display — prefer rendering DIRECTLY from \`props\` (controlled, no local copy). When local optimistic state genuinely needs a prop seed (a click bumps a count before the agent confirms), it MUST re-sync on prop change — \`useEffect(() => setCount(props.count ?? 0), [props.count])\` — or a \`ggui_amend\` repaint applies the new props while the display keeps the stale local value. BAD: \`const [count, setCount] = useState(props.count ?? 0)\` alone. GOOD: the same seed + the one-line re-sync effect.
|
|
357
358
|
- **Loading state**: while data is still absent, render \`<Skeleton>\` placeholders — never a blank screen. \`<Skeleton variant="text" />\` for a text line, \`variant="circle"\` for an avatar slot, default \`rect\` for a block.
|
|
358
359
|
- **Empty state**: when a list or results array is empty, render \`<EmptyState title="…" description="…" />\` — a region that renders nothing when empty looks broken to the user.
|
|
359
360
|
|
|
@@ -481,7 +482,7 @@ Every value change is mirrored to the host LLM's context automatically (debounce
|
|
|
481
482
|
|
|
482
483
|
**When NOT to use it.** Local UI state the contract did NOT declare — \`isDropdownOpen\`, hover flags, animation phase, ephemeral toggles. For those, use a plain \`useState\` directly. The runtime ignores undeclared state.
|
|
483
484
|
|
|
484
|
-
**\`contextSpec\` direction is one-way: client → agent.** The agent uses \`propsSpec\` (via \`ggui_update\`) and \`streamSpec\` (via the live channel) to push state TO the client. Don't try to write to the agent via \`contextSpec\` — there is no return path.
|
|
485
|
+
**\`contextSpec\` direction is one-way: client → agent.** The agent uses \`propsSpec\` (via \`ggui_amend\` for in-place repaints of this mount, or \`ggui_update\` when it mints a new history card) and \`streamSpec\` (via the live channel) to push state TO the client. Don't try to write to the agent via \`contextSpec\` — there is no return path.
|
|
485
486
|
|
|
486
487
|
**Schema mismatches drop silently.** If you set a value that doesn't match the slot's schema (e.g. a string into a \`{type: 'number'}\` slot), the runtime logs a dev \`console.warn\` and skips the post. Make sure your setter calls produce values that match the declared shape.
|
|
487
488
|
|
|
@@ -138,6 +138,19 @@ export interface CreateUiGeneratorOptions {
|
|
|
138
138
|
* `appGadgets` themselves; the legacy handler path stays unchanged.
|
|
139
139
|
*/
|
|
140
140
|
readonly gadgetCatalog?: GadgetCatalogAdapter;
|
|
141
|
+
/**
|
|
142
|
+
* When `true`, `generate()` never mutates `process.env` — the
|
|
143
|
+
* resolved provider route is threaded structurally into the
|
|
144
|
+
* dispatch call instead of being written to and restored from the
|
|
145
|
+
* process environment. Use this when running concurrent
|
|
146
|
+
* generations in one process (the default backup/mutate/restore
|
|
147
|
+
* dance is NOT concurrency-safe: two simultaneous calls race on
|
|
148
|
+
* the shared `process.env`).
|
|
149
|
+
*
|
|
150
|
+
* Default `false` — preserves the original env-mutation behavior
|
|
151
|
+
* for callers that don't need concurrent generations in-process.
|
|
152
|
+
*/
|
|
153
|
+
readonly disableEnvMutation?: boolean;
|
|
141
154
|
}
|
|
142
155
|
|
|
143
156
|
export function createUiGenerator(
|
|
@@ -149,6 +162,7 @@ export function createUiGenerator(
|
|
|
149
162
|
maxEvalRounds,
|
|
150
163
|
qualityConfig,
|
|
151
164
|
gadgetCatalog,
|
|
165
|
+
disableEnvMutation = false,
|
|
152
166
|
} = options;
|
|
153
167
|
|
|
154
168
|
const identity = resolveIdentity(options);
|
|
@@ -193,19 +207,37 @@ export function createUiGenerator(
|
|
|
193
207
|
}
|
|
194
208
|
|
|
195
209
|
const envBackup: Record<string, string | undefined> = {};
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
210
|
+
if (!disableEnvMutation) {
|
|
211
|
+
for (const k of Object.keys(route.env)) envBackup[k] = process.env[k];
|
|
212
|
+
const baseEnvForApply: Record<string, string> = {};
|
|
213
|
+
for (const [k, v] of Object.entries(process.env)) {
|
|
214
|
+
if (v !== undefined) baseEnvForApply[k] = v;
|
|
215
|
+
}
|
|
216
|
+
const routedEnv = applyRouteToEnv(baseEnvForApply, route);
|
|
217
|
+
for (const [k, v] of Object.entries(routedEnv)) process.env[k] = v;
|
|
218
|
+
// Also delete any keys the route asked to clear (route.env values
|
|
219
|
+
// marked undefined).
|
|
220
|
+
for (const [k, v] of Object.entries(route.env)) {
|
|
221
|
+
if (v === undefined) delete process.env[k];
|
|
222
|
+
}
|
|
207
223
|
}
|
|
208
224
|
|
|
225
|
+
// #484 — when `disableEnvMutation` is set, skip the process-env
|
|
226
|
+
// mutation above entirely and thread the resolved route
|
|
227
|
+
// structurally into the dispatch call instead. `route.env`
|
|
228
|
+
// already carries exactly the key(s) that would have gone into
|
|
229
|
+
// `process.env`, so no extra resolution work is needed here.
|
|
230
|
+
const routeOverride = disableEnvMutation
|
|
231
|
+
? {
|
|
232
|
+
apiKey:
|
|
233
|
+
route.env['ANTHROPIC_API_KEY'] ??
|
|
234
|
+
route.env['OPENAI_API_KEY'] ??
|
|
235
|
+
route.env['GEMINI_API_KEY'] ??
|
|
236
|
+
route.env['OPENROUTER_API_KEY'],
|
|
237
|
+
useBedrock: route.env['CLAUDE_CODE_USE_BEDROCK'] === '1',
|
|
238
|
+
}
|
|
239
|
+
: undefined;
|
|
240
|
+
|
|
209
241
|
try {
|
|
210
242
|
const tools = createGeneratorTools({
|
|
211
243
|
...(input.contract ? { contract: input.contract } : {}),
|
|
@@ -269,6 +301,7 @@ export function createUiGenerator(
|
|
|
269
301
|
? { gadgetTypes: input.gadgetTypes }
|
|
270
302
|
: {}),
|
|
271
303
|
enableRuntimeRender,
|
|
304
|
+
...(routeOverride !== undefined ? { routeOverride } : {}),
|
|
272
305
|
});
|
|
273
306
|
|
|
274
307
|
const metadata: GenerationMetadata = {
|
|
@@ -29,7 +29,7 @@ function runStateSeededFromProps(input: AxisCheckInput): EvalIssue[] {
|
|
|
29
29
|
mkIssue(
|
|
30
30
|
"state.merge.seeded_from_props",
|
|
31
31
|
`Entity collection "${e.name}" is not seeded from props — no useState initializer reads props.${e.name}.`,
|
|
32
|
-
`Add \`const [${e.name}, set${cap(e.name)}] = useState(props.${e.name});\` so stream/action updates
|
|
32
|
+
`Add \`const [${e.name}, set${cap(e.name)}] = useState(props.${e.name});\` paired with \`useEffect(() => set${cap(e.name)}(props.${e.name}), [props.${e.name}]);\` so stream/action updates merge into live state AND agent repaints (ggui_amend) re-seed it (#480).`,
|
|
33
33
|
),
|
|
34
34
|
);
|
|
35
35
|
}
|
|
@@ -86,6 +86,39 @@ function runNoPhantomUseState(input: AxisCheckInput): EvalIssue[] {
|
|
|
86
86
|
return issues;
|
|
87
87
|
}
|
|
88
88
|
|
|
89
|
+
function runPropSeedNoResync(input: AxisCheckInput): EvalIssue[] {
|
|
90
|
+
if (input.compiledCode === null) return [];
|
|
91
|
+
const src = input.sourceCode;
|
|
92
|
+
const issues: EvalIssue[] = [];
|
|
93
|
+
const re =
|
|
94
|
+
/const\s*\[\s*(\w+)\s*,\s*(\w+)\s*\]\s*=\s*useState(?:<[^>]*>)?\s*\(\s*props\??\.(\w+)/g;
|
|
95
|
+
for (const m of src.matchAll(re)) {
|
|
96
|
+
const [full, stateVar, setter, propName] = m;
|
|
97
|
+
const idx = m.index ?? 0;
|
|
98
|
+
const after = src.slice(idx + full.length);
|
|
99
|
+
// Never-called setter is `no_prop_mirror`'s case — this check is
|
|
100
|
+
// its #480 complement: the setter IS used (real optimistic state)
|
|
101
|
+
// but nothing re-seeds when the agent repaints the prop.
|
|
102
|
+
if (!new RegExp(`\\b${setter}\\s*\\(`).test(after)) continue;
|
|
103
|
+
// A re-seed effect names the prop in a dependency array. Bracket
|
|
104
|
+
// shape is the pragmatic signal (same looseness as the sibling
|
|
105
|
+
// checks): any `[... props.X ...]` array in the source counts.
|
|
106
|
+
const resync = new RegExp(
|
|
107
|
+
`\\[[^\\[\\]]*props\\??\\.${propName}\\b[^\\[\\]]*\\]`,
|
|
108
|
+
);
|
|
109
|
+
if (resync.test(src)) continue;
|
|
110
|
+
issues.push(
|
|
111
|
+
mkIssue(
|
|
112
|
+
"universal.prop_seed_no_resync",
|
|
113
|
+
`useState(props.${propName}) seeds "${stateVar}" and mutates it, but no effect depends on [props.${propName}] — a ggui_amend repaint pushes new props that never reach this display (#480).`,
|
|
114
|
+
`Add \`useEffect(() => ${setter}(props.${propName}), [props.${propName}]);\` so agent repaints re-seed the state, or derive the display directly from props.${propName}.`,
|
|
115
|
+
"warn",
|
|
116
|
+
),
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
return issues;
|
|
120
|
+
}
|
|
121
|
+
|
|
89
122
|
export const UNIVERSAL_CHECKS: readonly AxisCheck[] = [
|
|
90
123
|
{
|
|
91
124
|
id: "universal.prop_coverage",
|
|
@@ -105,4 +138,10 @@ export const UNIVERSAL_CHECKS: readonly AxisCheck[] = [
|
|
|
105
138
|
values: ALL_RENDER_VALUES,
|
|
106
139
|
run: runNoPhantomUseState,
|
|
107
140
|
},
|
|
141
|
+
{
|
|
142
|
+
id: "universal.prop_seed_no_resync",
|
|
143
|
+
axis: "render",
|
|
144
|
+
values: ALL_RENDER_VALUES,
|
|
145
|
+
run: runPropSeedNoResync,
|
|
146
|
+
},
|
|
108
147
|
];
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
|
|
17
17
|
import type { EvalIssue, EvalResult, EvalCategory } from './types-public.js';
|
|
18
18
|
import { createAgent } from '../harness/llm-router';
|
|
19
|
-
import type { LLMToolDef, LLMAgent } from '../harness/llm-router';
|
|
19
|
+
import type { AgentConfig, LLMToolDef, LLMAgent } from '../harness/llm-router';
|
|
20
20
|
import type { JsonObject } from '@ggui-ai/protocol';
|
|
21
21
|
import { getCriterionById } from './types-public.js';
|
|
22
22
|
|
|
@@ -64,6 +64,13 @@ export interface LLMEvalContext {
|
|
|
64
64
|
export interface LLMEvalConfig {
|
|
65
65
|
provider: 'claude' | 'openai' | 'google' | 'openrouter';
|
|
66
66
|
model?: string;
|
|
67
|
+
/**
|
|
68
|
+
* Explicit per-call credentials/routing (#484) — see
|
|
69
|
+
* `AgentConfig.routeOverride`. Threaded onto the agent this
|
|
70
|
+
* evaluator constructs so it never falls back to reading
|
|
71
|
+
* `process.env`. Absent (default) preserves existing behavior.
|
|
72
|
+
*/
|
|
73
|
+
routeOverride?: AgentConfig['routeOverride'];
|
|
67
74
|
}
|
|
68
75
|
|
|
69
76
|
/** A request-specific evaluation criterion generated by the LLM. */
|
|
@@ -745,8 +752,11 @@ export async function runLLMEvaluation(
|
|
|
745
752
|
.map((line, i) => `${String(i + 1).padStart(4)} | ${line}`)
|
|
746
753
|
.join('\n');
|
|
747
754
|
|
|
748
|
-
// Create a single agent — all calls share the same client + cache
|
|
749
|
-
|
|
755
|
+
// Create a single agent — all calls share the same client + cache.
|
|
756
|
+
// Pass the full config (not just the bare provider string) so a
|
|
757
|
+
// caller-supplied `routeOverride` (#484) reaches the constructed
|
|
758
|
+
// `LLMAgent` instance and it never falls back to `process.env`.
|
|
759
|
+
const agent = createAgent({ provider: routerProvider, model, routeOverride: config.routeOverride });
|
|
750
760
|
|
|
751
761
|
// Dynamic criteria removed — they generated overly strict edge-case checks
|
|
752
762
|
// that wasted coding turns on nitpicks. Will be replaced by a holistic
|
|
@@ -185,7 +185,11 @@ async function captureScreenshot(
|
|
|
185
185
|
const browser = await puppeteer.default.launch(launchOptions);
|
|
186
186
|
try {
|
|
187
187
|
const page = await browser.newPage();
|
|
188
|
-
|
|
188
|
+
// puppeteer ≥24.43 narrowed setContent's waitUntil to
|
|
189
|
+
// 'load' | 'domcontentloaded'; waitForNetworkIdle (0 connections
|
|
190
|
+
// for 500ms) reproduces the old 'networkidle0' semantics.
|
|
191
|
+
await page.setContent(html, { waitUntil: 'load', timeout: 15000 });
|
|
192
|
+
await page.waitForNetworkIdle({ idleTime: 500, timeout: 15000 }).catch(() => {});
|
|
189
193
|
await page.waitForSelector('#root > *', { timeout: 10000 }).catch(() => {});
|
|
190
194
|
// Wait a bit for CSS/fonts to settle
|
|
191
195
|
await new Promise(r => setTimeout(r, 1000));
|
|
@@ -16,14 +16,14 @@ export const realtimeFragments: Record<string, HarnessFragment> = {
|
|
|
16
16
|
value: "merge",
|
|
17
17
|
cacheTier: "axisDelta",
|
|
18
18
|
promptText:
|
|
19
|
-
"## Realtime: merge\nStream events carry an id; merge into LOCAL STATE. Subscribing alone is not enough — the local list must update on each event for the DOM to re-render.\n\n```tsx\nconst stream = useStream<UpdateT>('streamName');\nconst [items, setItems] = useState(props.items);\nuseEffect(() => {\n if (!stream.latest) return;\n setItems((prev) => prev.map((it) => it.id === stream.latest!.id ? { ...it, ...stream.latest } : it));\n}, [stream.latest]);\n// Render `items.map(...)` — NOT props.items, NOT stream.all.\n```\n\nNever append — stream=merge means the entity already exists locally.",
|
|
19
|
+
"## Realtime: merge\nStream events carry an id; merge into LOCAL STATE. Subscribing alone is not enough — the local list must update on each event for the DOM to re-render.\n\n```tsx\nconst stream = useStream<UpdateT>('streamName');\nconst [items, setItems] = useState(props.items);\nuseEffect(() => setItems(props.items), [props.items]); // re-seed on ggui_amend repaint — props stay the source of truth\nuseEffect(() => {\n if (!stream.latest) return;\n setItems((prev) => prev.map((it) => it.id === stream.latest!.id ? { ...it, ...stream.latest } : it));\n}, [stream.latest]);\n// Render `items.map(...)` — NOT props.items, NOT stream.all.\n```\n\nNever append — stream=merge means the entity already exists locally.",
|
|
20
20
|
},
|
|
21
21
|
append: {
|
|
22
22
|
axis: "realtime",
|
|
23
23
|
value: "append",
|
|
24
24
|
cacheTier: "axisDelta",
|
|
25
25
|
promptText:
|
|
26
|
-
"## Realtime: append\nStream events are new entities; APPEND to local state. Subscribing alone is not enough — the local list must update for the DOM to re-render.\n\n```tsx\nconst stream = useStream<EventT>('streamName');\nconst [items, setItems] = useState(props.items);\nuseEffect(() => {\n if (!stream.latest) return;\n setItems((prev) => [...prev, stream.latest!]);\n}, [stream.latest]);\n// Render `items.map(...)` — NOT props.items, NOT stream.all (which accumulates only post-mount).\n```\n\nUse head for newest-first, tail for chat-style. Cap list length if needed. Do not dedupe unless the contract guarantees at-most-once.",
|
|
26
|
+
"## Realtime: append\nStream events are new entities; APPEND to local state. Subscribing alone is not enough — the local list must update for the DOM to re-render.\n\n```tsx\nconst stream = useStream<EventT>('streamName');\nconst [items, setItems] = useState(props.items);\nuseEffect(() => setItems(props.items), [props.items]); // re-seed on ggui_amend repaint — props stay the source of truth\nuseEffect(() => {\n if (!stream.latest) return;\n setItems((prev) => [...prev, stream.latest!]);\n}, [stream.latest]);\n// Render `items.map(...)` — NOT props.items, NOT stream.all (which accumulates only post-mount).\n```\n\nUse head for newest-first, tail for chat-style. Cap list length if needed. Do not dedupe unless the contract guarantees at-most-once.",
|
|
27
27
|
},
|
|
28
28
|
status: {
|
|
29
29
|
axis: "realtime",
|
|
@@ -44,7 +44,7 @@ export const realtimeFragments: Record<string, HarnessFragment> = {
|
|
|
44
44
|
value: "mixed",
|
|
45
45
|
cacheTier: "axisDelta",
|
|
46
46
|
promptText:
|
|
47
|
-
"## Realtime: mixed\nMultiple stream channels with DIFFERENT semantics. Each `streamKinds` entry in the contract maps to one of: merge, append, status, presence. Subscribe + sync to local state PER CHANNEL — subscribing alone never re-renders the DOM.\n\nExample (chat-interface — message=append, typing=presence):\n\n```tsx\nconst messages = useStream<MessageT>('message');\nconst typing = useStream<TypingT>('typing');\nconst [messageList, setMessageList] = useState(props.messages);\nconst [typingUsers, setTypingUsers] = useState<Set<string>>(new Set());\n\nuseEffect(() => {\n if (!messages.latest) return;\n setMessageList((prev) => [...prev, messages.latest!]);\n}, [messages.latest]);\n\nuseEffect(() => {\n if (!typing.latest) return;\n setTypingUsers((prev) => {\n const next = new Set(prev);\n if (typing.latest!.active) next.add(typing.latest!.sender);\n else next.delete(typing.latest!.sender);\n return next;\n });\n}, [typing.latest]);\n```\n\nDo NOT treat all events the same. Do NOT skip the local-state sync — `useStream` alone won't trigger DOM updates of your derived view.",
|
|
47
|
+
"## Realtime: mixed\nMultiple stream channels with DIFFERENT semantics. Each `streamKinds` entry in the contract maps to one of: merge, append, status, presence. Subscribe + sync to local state PER CHANNEL — subscribing alone never re-renders the DOM.\n\nExample (chat-interface — message=append, typing=presence):\n\n```tsx\nconst messages = useStream<MessageT>('message');\nconst typing = useStream<TypingT>('typing');\nconst [messageList, setMessageList] = useState(props.messages);\nconst [typingUsers, setTypingUsers] = useState<Set<string>>(new Set());\n\nuseEffect(() => setMessageList(props.messages), [props.messages]); // re-seed on ggui_amend repaint\n\nuseEffect(() => {\n if (!messages.latest) return;\n setMessageList((prev) => [...prev, messages.latest!]);\n}, [messages.latest]);\n\nuseEffect(() => {\n if (!typing.latest) return;\n setTypingUsers((prev) => {\n const next = new Set(prev);\n if (typing.latest!.active) next.add(typing.latest!.sender);\n else next.delete(typing.latest!.sender);\n return next;\n });\n}, [typing.latest]);\n```\n\nDo NOT treat all events the same. Do NOT skip the local-state sync — `useStream` alone won't trigger DOM updates of your derived view.",
|
|
48
48
|
boilerplateMarker: [
|
|
49
49
|
"",
|
|
50
50
|
" // ── Mixed stream handlers ──",
|
package/src/fragments/state.ts
CHANGED
|
@@ -24,11 +24,11 @@ export const stateFragments: Record<string, HarnessFragment> = {
|
|
|
24
24
|
value: "merge",
|
|
25
25
|
cacheTier: "axisDelta",
|
|
26
26
|
promptText:
|
|
27
|
-
"## State: merge (live entity reconciliation)\n1. Seed `const [items, setItems] = useState(props.items ?? [])
|
|
27
|
+
"## State: merge (live entity reconciliation)\n1. Seed `const [items, setItems] = useState(props.items ?? [])` AND re-seed on prop change: `useEffect(() => setItems(props.items ?? []), [props.items])` — a `ggui_amend` repaint must move the list.\n2. On stream updates: `setItems(prev => prev.map(it => it.id === event.id ? { ...it, ...event } : it))` — merge by id, do NOT append.\n3. Memoize derived views (grouping/sorting/filtering) with useMemo.\n4. Per-item actions pass `{ id, ... }` in the payload.\n5. Never push the stream payload into an append-only list unless realtime=append.",
|
|
28
28
|
boilerplateMarker: [
|
|
29
29
|
"",
|
|
30
30
|
" // ── Live entity state (merge-by-id) ──",
|
|
31
|
-
" // useState(props.items ?? []); merge stream events by item.id; never append.",
|
|
31
|
+
" // useState(props.items ?? []); re-seed via useEffect on [props.items]; merge stream events by item.id; never append.",
|
|
32
32
|
"",
|
|
33
33
|
].join("\n"),
|
|
34
34
|
},
|
|
@@ -144,7 +144,10 @@ export async function initSession(input: {
|
|
|
144
144
|
const systemPrompt = input.systemPromptOverride ?? harness.how.systemPrompt;
|
|
145
145
|
|
|
146
146
|
// ── Create + warm-cache the coding agent ──
|
|
147
|
-
|
|
147
|
+
// Pass the full spec (not just `.provider`) so a caller-supplied
|
|
148
|
+
// `routeOverride` (#484) reaches the constructed `LLMAgent` instance
|
|
149
|
+
// and it never falls back to reading `process.env`.
|
|
150
|
+
const codingAgent = createAgent(agents.coding);
|
|
148
151
|
await codingAgent.warmCache(
|
|
149
152
|
agents.coding.model,
|
|
150
153
|
systemPrompt,
|
|
@@ -418,6 +418,12 @@ export async function runEvalRound(
|
|
|
418
418
|
{
|
|
419
419
|
provider: mapProviderForEvaluator(evaluationAgent.provider),
|
|
420
420
|
model: evaluationAgent.model,
|
|
421
|
+
// #484 — thread the coding-agent's routing override
|
|
422
|
+
// through to the evaluator so its LLM call never falls
|
|
423
|
+
// back to `process.env` either (the eval calls run
|
|
424
|
+
// inside the same concurrent generation, so the race is
|
|
425
|
+
// real here too, not just on the coding turn).
|
|
426
|
+
routeOverride: evaluationAgent.routeOverride,
|
|
421
427
|
},
|
|
422
428
|
preWarmedContext,
|
|
423
429
|
)
|
|
@@ -53,6 +53,21 @@ export interface AgentConfig {
|
|
|
53
53
|
* behavior for all existing callers).
|
|
54
54
|
*/
|
|
55
55
|
temperature?: number;
|
|
56
|
+
/**
|
|
57
|
+
* Explicit per-call credentials/routing, bypassing `process.env`
|
|
58
|
+
* entirely when supplied. Absent (default) preserves today's
|
|
59
|
+
* behavior — each agent falls back to reading its provider's env
|
|
60
|
+
* var(s) directly, so existing callers (including self-hosters who
|
|
61
|
+
* set env vars once at process start) are unaffected. Use this to
|
|
62
|
+
* run concurrent generations safely in one process: two simultaneous
|
|
63
|
+
* calls with different `routeOverride`s never race on shared
|
|
64
|
+
* `process.env` state.
|
|
65
|
+
*/
|
|
66
|
+
routeOverride?: {
|
|
67
|
+
readonly apiKey?: string;
|
|
68
|
+
/** Anthropic-only: force the Bedrock-IAM client instead of the direct API client. */
|
|
69
|
+
readonly useBedrock?: boolean;
|
|
70
|
+
};
|
|
56
71
|
}
|
|
57
72
|
|
|
58
73
|
export interface LLMResponse {
|
|
@@ -181,6 +196,17 @@ export abstract class LLMAgent {
|
|
|
181
196
|
// Call resetSession() between independent generation runs.
|
|
182
197
|
protected lastSessionId: string | undefined;
|
|
183
198
|
|
|
199
|
+
/**
|
|
200
|
+
* Explicit per-call routing override — see `AgentConfig.routeOverride`.
|
|
201
|
+
* Absent when constructed via the bare `new XAgent()` form (every
|
|
202
|
+
* existing call site), so `createClient()`/`resolveModel()` fall
|
|
203
|
+
* back to `process.env` exactly as before.
|
|
204
|
+
*/
|
|
205
|
+
protected readonly routeOverride: AgentConfig['routeOverride'];
|
|
206
|
+
|
|
207
|
+
constructor(routeOverride?: AgentConfig['routeOverride']) {
|
|
208
|
+
this.routeOverride = routeOverride;
|
|
209
|
+
}
|
|
184
210
|
|
|
185
211
|
protected abstract resolveModel(model: string): string;
|
|
186
212
|
protected abstract createClient(): Promise<unknown>;
|
|
@@ -294,7 +320,9 @@ export class AnthropicAgent extends LLMAgent {
|
|
|
294
320
|
protected resolveModel(model: string): string {
|
|
295
321
|
// Bedrock IAM path — the upstream model id must be the cross-region
|
|
296
322
|
// inference-profile form (`us.anthropic.*`), not the bare API id.
|
|
297
|
-
|
|
323
|
+
const useBedrock =
|
|
324
|
+
this.routeOverride?.useBedrock ?? process.env.CLAUDE_CODE_USE_BEDROCK === '1';
|
|
325
|
+
if (useBedrock) {
|
|
298
326
|
return getBedrockModelId(model);
|
|
299
327
|
}
|
|
300
328
|
return model.startsWith('anthropic/')
|
|
@@ -308,7 +336,9 @@ export class AnthropicAgent extends LLMAgent {
|
|
|
308
336
|
// pool-funded cloud pod. Auth is the pod's IRSA role — no API key.
|
|
309
337
|
// `AnthropicBedrock` is wire-compatible with `Anthropic` for the
|
|
310
338
|
// `.messages` API this agent uses.
|
|
311
|
-
|
|
339
|
+
const useBedrock =
|
|
340
|
+
this.routeOverride?.useBedrock ?? process.env.CLAUDE_CODE_USE_BEDROCK === '1';
|
|
341
|
+
if (useBedrock) {
|
|
312
342
|
const { AnthropicBedrock } = await import('@anthropic-ai/bedrock-sdk');
|
|
313
343
|
return new AnthropicBedrock({
|
|
314
344
|
awsRegion:
|
|
@@ -319,8 +349,10 @@ export class AnthropicAgent extends LLMAgent {
|
|
|
319
349
|
}
|
|
320
350
|
// SDK construction lives in `adapters/claude/client.ts`. The BYOK
|
|
321
351
|
// resolver writes the raw key into `process.env.ANTHROPIC_API_KEY`
|
|
322
|
-
// before this runs (see `applyRouteToEnv` in provider-router)
|
|
323
|
-
|
|
352
|
+
// before this runs (see `applyRouteToEnv` in provider-router) —
|
|
353
|
+
// `routeOverride.apiKey`, when supplied, skips that env round-trip
|
|
354
|
+
// entirely.
|
|
355
|
+
return createAnthropicClient(this.routeOverride?.apiKey ?? process.env.ANTHROPIC_API_KEY);
|
|
324
356
|
}
|
|
325
357
|
|
|
326
358
|
async callText(
|
|
@@ -670,7 +702,7 @@ export class OpenAIAgent extends LLMAgent {
|
|
|
670
702
|
|
|
671
703
|
protected async createClient(): Promise<OpenAI> {
|
|
672
704
|
const { default: OpenAISDK } = await import('openai');
|
|
673
|
-
return new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY });
|
|
705
|
+
return new OpenAISDK({ apiKey: this.routeOverride?.apiKey ?? process.env.OPENAI_API_KEY });
|
|
674
706
|
}
|
|
675
707
|
|
|
676
708
|
async callText(
|
|
@@ -928,7 +960,8 @@ export class GoogleAgent extends LLMAgent {
|
|
|
928
960
|
protected async createClient(): Promise<GoogleGenAI> {
|
|
929
961
|
const { GoogleGenAI: GoogleGenAISDK } = await import('@google/genai');
|
|
930
962
|
return new GoogleGenAISDK({
|
|
931
|
-
apiKey:
|
|
963
|
+
apiKey:
|
|
964
|
+
this.routeOverride?.apiKey ?? (process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY),
|
|
932
965
|
httpOptions: { timeout: 300_000 }, // 5 min — Pro models can be slow
|
|
933
966
|
});
|
|
934
967
|
}
|
|
@@ -1248,7 +1281,7 @@ export class OpenRouterAgent extends LLMAgent {
|
|
|
1248
1281
|
protected async createClient(): Promise<unknown> {
|
|
1249
1282
|
const { OpenRouterClient } = await import('../adapters/openrouter/client');
|
|
1250
1283
|
return new OpenRouterClient({
|
|
1251
|
-
apiKey: process.env.OPENROUTER_API_KEY!,
|
|
1284
|
+
apiKey: this.routeOverride?.apiKey ?? process.env.OPENROUTER_API_KEY!,
|
|
1252
1285
|
});
|
|
1253
1286
|
}
|
|
1254
1287
|
|
|
@@ -1432,17 +1465,29 @@ export class OpenRouterAgent extends LLMAgent {
|
|
|
1432
1465
|
/**
|
|
1433
1466
|
* Create a fresh agent instance for the given provider.
|
|
1434
1467
|
* Each call returns a new instance — no shared state between callers.
|
|
1468
|
+
*
|
|
1469
|
+
* Accepts either a bare provider string (every pre-existing call
|
|
1470
|
+
* site — the agent falls back to reading its provider's env var(s)
|
|
1471
|
+
* directly) or a full {@link AgentConfig}, whose `routeOverride`
|
|
1472
|
+
* (when present) is threaded into the constructed agent so it never
|
|
1473
|
+
* touches `process.env`.
|
|
1435
1474
|
*/
|
|
1436
|
-
export function createAgent(provider: AgentConfig['provider']): LLMAgent
|
|
1475
|
+
export function createAgent(provider: AgentConfig['provider']): LLMAgent;
|
|
1476
|
+
export function createAgent(config: AgentConfig): LLMAgent;
|
|
1477
|
+
export function createAgent(providerOrConfig: AgentConfig['provider'] | AgentConfig): LLMAgent {
|
|
1478
|
+
const provider =
|
|
1479
|
+
typeof providerOrConfig === 'string' ? providerOrConfig : providerOrConfig.provider;
|
|
1480
|
+
const routeOverride =
|
|
1481
|
+
typeof providerOrConfig === 'string' ? undefined : providerOrConfig.routeOverride;
|
|
1437
1482
|
switch (provider) {
|
|
1438
1483
|
case 'anthropic':
|
|
1439
|
-
return new AnthropicAgent();
|
|
1484
|
+
return new AnthropicAgent(routeOverride);
|
|
1440
1485
|
case 'openai':
|
|
1441
|
-
return new OpenAIAgent();
|
|
1486
|
+
return new OpenAIAgent(routeOverride);
|
|
1442
1487
|
case 'google':
|
|
1443
|
-
return new GoogleAgent();
|
|
1488
|
+
return new GoogleAgent(routeOverride);
|
|
1444
1489
|
case 'openrouter':
|
|
1445
|
-
return new OpenRouterAgent();
|
|
1490
|
+
return new OpenRouterAgent(routeOverride);
|
|
1446
1491
|
}
|
|
1447
1492
|
}
|
|
1448
1493
|
|
|
@@ -1457,7 +1502,7 @@ export async function callLLM(
|
|
|
1457
1502
|
maxTokens?: number,
|
|
1458
1503
|
): Promise<LLMResponse> {
|
|
1459
1504
|
const start = Date.now();
|
|
1460
|
-
const agent = createAgent(config
|
|
1505
|
+
const agent = createAgent(config);
|
|
1461
1506
|
const result = await agent.callText(
|
|
1462
1507
|
config.model,
|
|
1463
1508
|
systemPrompt,
|
|
@@ -1479,7 +1524,7 @@ export async function callLLMWithTools(
|
|
|
1479
1524
|
maxTurns: number = 10,
|
|
1480
1525
|
): Promise<LLMWithToolsResponse> {
|
|
1481
1526
|
const start = Date.now();
|
|
1482
|
-
const agent = createAgent(config
|
|
1527
|
+
const agent = createAgent(config);
|
|
1483
1528
|
const result = await agent.callWithTools(
|
|
1484
1529
|
config.model,
|
|
1485
1530
|
systemPrompt,
|
package/src/harness/runtime.ts
CHANGED
|
@@ -31,6 +31,7 @@ import { WIRE_DOCUMENTATION } from "../tools/get-wire.js";
|
|
|
31
31
|
import { renderPitfallsBlock } from "./pitfalls.js";
|
|
32
32
|
import type { GadgetDescriptor, DataContract, JsonObject } from "@ggui-ai/protocol";
|
|
33
33
|
import { buildSystemPrompt as buildSystemPromptSkeleton } from "../boilerplate.js";
|
|
34
|
+
import type { AgentConfig } from "./llm-router.js";
|
|
34
35
|
|
|
35
36
|
// Re-export the boilerplate generator so existing internal importers
|
|
36
37
|
// (create-harness.ts, benchmarks/preview-boilerplate.ts) keep working.
|
|
@@ -46,6 +47,13 @@ type Provider = "anthropic" | "openai" | "google" | "openrouter";
|
|
|
46
47
|
export interface AgentSpec {
|
|
47
48
|
provider: Provider;
|
|
48
49
|
model: string;
|
|
50
|
+
/**
|
|
51
|
+
* Explicit per-call credentials/routing (#484) — see
|
|
52
|
+
* `AgentConfig.routeOverride`. Threaded through unchanged so
|
|
53
|
+
* `createAgent()` can bypass `process.env` when the caller supplies
|
|
54
|
+
* it. Absent for every pre-existing caller.
|
|
55
|
+
*/
|
|
56
|
+
routeOverride?: AgentConfig["routeOverride"];
|
|
49
57
|
}
|
|
50
58
|
|
|
51
59
|
export interface SingleComponentParams {
|