@ggui-ai/ui-gen 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/generation-dispatch.d.ts +1 -1
- package/dist/adapters/generation-dispatch.js +198 -59
- package/dist/adapters/generation-dispatch.js.map +1 -1
- package/dist/adapters/index.js +35 -10
- package/dist/adapters/index.js.map +1 -1
- package/dist/advanced/index.d.ts +2 -2
- package/dist/advanced/index.js +381 -58
- package/dist/advanced/index.js.map +1 -1
- package/dist/boilerplate.js.map +1 -1
- package/dist/check/index.d.ts +1 -1
- package/dist/check/index.js.map +1 -1
- package/dist/{create-ui-generator-D7kOXNjJ.d.ts → create-ui-generator-BEkzEtNz.d.ts} +1 -1
- package/dist/{dispatch-DGvhmARp.d.ts → dispatch-DJNpQ7DI.d.ts} +1 -1
- package/dist/evaluation/axis-checks/index.d.ts +3 -3
- package/dist/evaluation/axis-checks/index.js.map +1 -1
- package/dist/evaluation/axis-checks/registry.d.ts +1 -1
- package/dist/evaluation/index.d.ts +3 -3
- package/dist/evaluation/index.js +37 -20
- package/dist/evaluation/index.js.map +1 -1
- package/dist/harness/check/runtime-render/index.d.ts +1 -1
- package/dist/harness/index.d.ts +1 -1
- package/dist/harness/index.js +34 -9
- package/dist/harness/index.js.map +1 -1
- package/dist/harness/prompts.js +1 -1
- package/dist/harness/prompts.js.map +1 -1
- package/dist/harness/result-types.d.ts +15 -2
- package/dist/harness/result-types.js.map +1 -1
- package/dist/harness/runtime.d.ts +1 -1
- package/dist/harness/runtime.js.map +1 -1
- package/dist/harness/types-public.d.ts +1 -1
- package/dist/harness/types-public.js.map +1 -1
- package/dist/harness/validator-trace-sink.d.ts +1 -1
- package/dist/index.d.ts +22 -6
- package/dist/index.js +381 -58
- package/dist/index.js.map +1 -1
- package/dist/patch.d.ts +1 -1
- package/dist/{types-public-CzYqZclY.d.ts → types-public-mM_-1Yk3.d.ts} +16 -1
- package/dist/workflows.d.ts +1 -1
- package/package.json +12 -12
- package/src/adapters/claude/claude-code-login.ts +62 -7
- package/src/adapters/claude/sdk.ts +6 -5
- package/src/boilerplate/templates/layouts/chat-mobile.tsx.tmpl +7 -3
- package/src/boilerplate/templates/layouts/chat-universal.tsx.tmpl +7 -3
- package/src/contract-context.ts +1 -1
- package/src/create-ui-generator.ts +30 -0
- package/src/evaluation/index.ts +1 -0
- package/src/evaluation/llm-evaluator.ts +40 -13
- package/src/evaluation/types-public.ts +16 -0
- package/src/evaluation/visual-evaluator.ts +9 -6
- package/src/generator-build.ts +158 -0
- package/src/harness/claude-code-login-query.ts +5 -4
- package/src/harness/coding/assemble-result.ts +8 -1
- package/src/harness/coding/generate-task-runner.ts +75 -36
- package/src/harness/coding/run-coding-turn.ts +9 -0
- package/src/harness/coding/run-eval-round.ts +160 -5
- package/src/harness/result-types.ts +14 -0
- package/src/index.ts +2 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { EvaluationConfig } from '../evaluation/types.js';
|
|
2
2
|
import { ModelRoles, GenerationResult } from '../harness/result-types.js';
|
|
3
3
|
import { ProviderName, ToolDefinition } from './types.js';
|
|
4
|
-
import { Q as QualityConfig } from '../types-public-
|
|
4
|
+
import { Q as QualityConfig } from '../types-public-mM_-1Yk3.js';
|
|
5
5
|
import { A as AgentConfig } from '../llm-router-DwwUK1bz.js';
|
|
6
6
|
import { DataContract, JsonObject, GadgetDescriptor } from '@ggui-ai/protocol';
|
|
7
7
|
import { C as CanvasClass, D as DesignMode } from '../design-mode-DLufhrNp.js';
|
|
@@ -4,7 +4,7 @@ import { z } from 'zod';
|
|
|
4
4
|
import { getCssTokens, fillFitRule, FORBIDDEN_IMPORT_SPECIFIERS } from '@ggui-ai/design/rendering';
|
|
5
5
|
import * as esbuild from 'esbuild';
|
|
6
6
|
import { build } from 'esbuild';
|
|
7
|
-
import fs, { existsSync,
|
|
7
|
+
import fs, { existsSync, mkdtempSync, writeFileSync, rmSync, readFileSync, readdirSync, statSync } from 'fs';
|
|
8
8
|
import { contentHash } from '@ggui-ai/protocol/content-hash';
|
|
9
9
|
import { createRequire } from 'module';
|
|
10
10
|
import { fileURLToPath, pathToFileURL } from 'url';
|
|
@@ -649,26 +649,51 @@ var init_client = __esm({
|
|
|
649
649
|
});
|
|
650
650
|
|
|
651
651
|
// src/adapters/claude/claude-code-login.ts
|
|
652
|
-
function
|
|
652
|
+
function loginChildEnv(env) {
|
|
653
653
|
const out = {};
|
|
654
|
-
const drop = new Set(PROVIDER_KEY_ENV_NAMES);
|
|
655
654
|
for (const [k, v] of Object.entries(env)) {
|
|
656
|
-
if (v !== void 0 &&
|
|
655
|
+
if (v !== void 0 && (LOGIN_CHILD_ENV.has(k) || k.startsWith("LC_"))) out[k] = v;
|
|
657
656
|
}
|
|
658
657
|
return out;
|
|
659
658
|
}
|
|
660
659
|
function claudeCodeLoginQueryOptions() {
|
|
661
660
|
return { tools: [], settingSources: [] };
|
|
662
661
|
}
|
|
663
|
-
var
|
|
662
|
+
var LOGIN_CHILD_ENV_NAMES, LOGIN_CHILD_ENV;
|
|
664
663
|
var init_claude_code_login = __esm({
|
|
665
664
|
"src/adapters/claude/claude-code-login.ts"() {
|
|
666
|
-
|
|
667
|
-
"
|
|
668
|
-
"
|
|
669
|
-
"
|
|
670
|
-
"
|
|
665
|
+
LOGIN_CHILD_ENV_NAMES = [
|
|
666
|
+
"PATH",
|
|
667
|
+
"HOME",
|
|
668
|
+
"USER",
|
|
669
|
+
"LOGNAME",
|
|
670
|
+
"SHELL",
|
|
671
|
+
"TMPDIR",
|
|
672
|
+
"TMP",
|
|
673
|
+
"TEMP",
|
|
674
|
+
"TERM",
|
|
675
|
+
"TZ",
|
|
676
|
+
"LANG",
|
|
677
|
+
"CLAUDE_CONFIG_DIR",
|
|
678
|
+
"CLAUDE_CODE_OAUTH_TOKEN",
|
|
679
|
+
"XDG_CONFIG_HOME",
|
|
680
|
+
"XDG_DATA_HOME",
|
|
681
|
+
"XDG_CACHE_HOME",
|
|
682
|
+
"XDG_STATE_HOME",
|
|
683
|
+
"XDG_RUNTIME_DIR",
|
|
684
|
+
"HTTPS_PROXY",
|
|
685
|
+
"https_proxy",
|
|
686
|
+
"HTTP_PROXY",
|
|
687
|
+
"http_proxy",
|
|
688
|
+
"NO_PROXY",
|
|
689
|
+
"no_proxy",
|
|
690
|
+
"ALL_PROXY",
|
|
691
|
+
"all_proxy",
|
|
692
|
+
"NODE_EXTRA_CA_CERTS",
|
|
693
|
+
"SSL_CERT_FILE",
|
|
694
|
+
"SSL_CERT_DIR"
|
|
671
695
|
];
|
|
696
|
+
LOGIN_CHILD_ENV = new Set(LOGIN_CHILD_ENV_NAMES);
|
|
672
697
|
}
|
|
673
698
|
});
|
|
674
699
|
|
|
@@ -815,7 +840,7 @@ function buildLoginQueryOptions(params) {
|
|
|
815
840
|
systemPrompt: params.systemPrompt,
|
|
816
841
|
maxTurns: 1,
|
|
817
842
|
thinking: { type: "disabled" },
|
|
818
|
-
env:
|
|
843
|
+
env: loginChildEnv(params.env ?? process.env),
|
|
819
844
|
...claudeCodeLoginQueryOptions(),
|
|
820
845
|
...params.tools !== void 0 ? {
|
|
821
846
|
mcpServers: { [MCP_SERVER_NAME]: params.tools.server },
|
|
@@ -5891,6 +5916,14 @@ function getDefaultEvalModel(provider) {
|
|
|
5891
5916
|
return "claude-haiku-4-5-20251001";
|
|
5892
5917
|
}
|
|
5893
5918
|
}
|
|
5919
|
+
function callTokens(response) {
|
|
5920
|
+
return {
|
|
5921
|
+
in: response.inputTokens,
|
|
5922
|
+
out: response.outputTokens,
|
|
5923
|
+
...response.cacheReadTokens !== void 0 ? { cacheRead: response.cacheReadTokens } : {},
|
|
5924
|
+
...response.cacheCreationTokens !== void 0 ? { cacheCreation: response.cacheCreationTokens } : {}
|
|
5925
|
+
};
|
|
5926
|
+
}
|
|
5894
5927
|
async function evalCriterion(criterion, tier, motherPrompt, numberedSource, agent, model, designMode) {
|
|
5895
5928
|
const start = Date.now();
|
|
5896
5929
|
const tool = tier === 1 ? makeTier1Tool(criterion) : makeTier2Tool(criterion);
|
|
@@ -5914,7 +5947,7 @@ async function evalCriterion(criterion, tier, motherPrompt, numberedSource, agen
|
|
|
5914
5947
|
pass: true,
|
|
5915
5948
|
coverage: "skipped",
|
|
5916
5949
|
skipReason: "no tool call returned",
|
|
5917
|
-
tokens:
|
|
5950
|
+
tokens: callTokens(response),
|
|
5918
5951
|
durationMs
|
|
5919
5952
|
};
|
|
5920
5953
|
}
|
|
@@ -5967,7 +6000,7 @@ async function evalCriterion(criterion, tier, motherPrompt, numberedSource, agen
|
|
|
5967
6000
|
issues,
|
|
5968
6001
|
pass,
|
|
5969
6002
|
coverage: "ran",
|
|
5970
|
-
tokens:
|
|
6003
|
+
tokens: callTokens(response),
|
|
5971
6004
|
durationMs
|
|
5972
6005
|
};
|
|
5973
6006
|
} catch (e) {
|
|
@@ -6050,7 +6083,8 @@ async function runLLMEvaluation(context, config, preWarmedContext) {
|
|
|
6050
6083
|
const passCategories = [];
|
|
6051
6084
|
let totalIn = 0;
|
|
6052
6085
|
let totalOut = 0;
|
|
6053
|
-
let
|
|
6086
|
+
let cacheRead;
|
|
6087
|
+
let cacheCreation;
|
|
6054
6088
|
for (const r of results) {
|
|
6055
6089
|
allIssues.push(...r.issues);
|
|
6056
6090
|
if (r.pass) {
|
|
@@ -6058,10 +6092,8 @@ async function runLLMEvaluation(context, config, preWarmedContext) {
|
|
|
6058
6092
|
}
|
|
6059
6093
|
totalIn += r.tokens.in;
|
|
6060
6094
|
totalOut += r.tokens.out;
|
|
6061
|
-
|
|
6062
|
-
|
|
6063
|
-
if (firstResult && firstResult.tokens.in > 0) {
|
|
6064
|
-
cachedTokensEstimate = firstResult.tokens.in * (totalCriteriaCount - 1);
|
|
6095
|
+
if (r.tokens.cacheRead !== void 0) cacheRead = (cacheRead ?? 0) + r.tokens.cacheRead;
|
|
6096
|
+
if (r.tokens.cacheCreation !== void 0) cacheCreation = (cacheCreation ?? 0) + r.tokens.cacheCreation;
|
|
6065
6097
|
}
|
|
6066
6098
|
const wallMs = Date.now() - evalStart;
|
|
6067
6099
|
const criteriaCoverage = results.map((r) => ({
|
|
@@ -6078,14 +6110,16 @@ async function runLLMEvaluation(context, config, preWarmedContext) {
|
|
|
6078
6110
|
console.log(`[eval] ${r.criterion}: ${status}${issueLabel} ${(r.durationMs / 1e3).toFixed(1)}s`);
|
|
6079
6111
|
}
|
|
6080
6112
|
console.log(
|
|
6081
|
-
`[eval] total: ${(wallMs / 1e3).toFixed(1)}s wall (${totalCriteriaCount} parallel) | in=${totalIn} out=${totalOut}
|
|
6113
|
+
`[eval] total: ${(wallMs / 1e3).toFixed(1)}s wall (${totalCriteriaCount} parallel) | in=${totalIn} out=${totalOut} cacheRead=${cacheRead ?? "unreported"} cacheWrite=${cacheCreation ?? "unreported"} | issues=${allIssues.length} pass=${passCategories.length}` + (skippedCount > 0 ? ` skipped=${skippedCount}` : "")
|
|
6082
6114
|
);
|
|
6083
6115
|
return {
|
|
6084
6116
|
issues: allIssues,
|
|
6085
6117
|
pass: passCategories,
|
|
6086
6118
|
criteriaCoverage,
|
|
6087
6119
|
inputTokens: totalIn,
|
|
6088
|
-
outputTokens: totalOut
|
|
6120
|
+
outputTokens: totalOut,
|
|
6121
|
+
...cacheRead !== void 0 ? { cacheReadTokens: cacheRead } : {},
|
|
6122
|
+
...cacheCreation !== void 0 ? { cacheCreationTokens: cacheCreation } : {}
|
|
6089
6123
|
};
|
|
6090
6124
|
}
|
|
6091
6125
|
var FREE_STYLING_RULES, CONSTRAINED_STYLING_RULES, CONSTRAINED_VISUAL_REFERENCE, FREE_VISUAL_REFERENCE, CONSTRAINED_RAW_CLICK_TARGET_GAP, FREE_RAW_CLICK_TARGET_GAP, TIER_1_CRITERIA, TIER_2_CRITERIA, TIER_1_CATEGORIES, TIER_2_CATEGORIES, VALID_CATEGORIES;
|
|
@@ -6339,8 +6373,7 @@ function resolveDesignPackageDir() {
|
|
|
6339
6373
|
}
|
|
6340
6374
|
}
|
|
6341
6375
|
async function bundleForRendering(compiledCode, sampleProps, designSrc = resolve(resolveDesignPackageDir(), "src")) {
|
|
6342
|
-
const tmpDir = resolve(tmpdir(), "ggui-visual-eval-"
|
|
6343
|
-
mkdirSync(tmpDir, { recursive: true });
|
|
6376
|
+
const tmpDir = mkdtempSync(resolve(tmpdir(), "ggui-visual-eval-"));
|
|
6344
6377
|
const componentFile = resolve(tmpDir, "component.tsx");
|
|
6345
6378
|
const entryFile = resolve(tmpDir, "entry.tsx");
|
|
6346
6379
|
const designPkgDir = resolve(designSrc, "..");
|
|
@@ -6415,14 +6448,7 @@ try {
|
|
|
6415
6448
|
});
|
|
6416
6449
|
return result.outputFiles[0]?.text ?? "";
|
|
6417
6450
|
} finally {
|
|
6418
|
-
|
|
6419
|
-
unlinkSync(entryFile);
|
|
6420
|
-
} catch {
|
|
6421
|
-
}
|
|
6422
|
-
try {
|
|
6423
|
-
unlinkSync(componentFile);
|
|
6424
|
-
} catch {
|
|
6425
|
-
}
|
|
6451
|
+
rmSync(tmpDir, { recursive: true, force: true });
|
|
6426
6452
|
}
|
|
6427
6453
|
}
|
|
6428
6454
|
function buildRenderHTML(bundledCode, cssTokens, fit) {
|
|
@@ -14344,7 +14370,11 @@ async function assembleGenerationResult(input) {
|
|
|
14344
14370
|
generationTimeMs: totalMs,
|
|
14345
14371
|
turnsUsed: telemetry.turnsUsed,
|
|
14346
14372
|
passesUsed: 1,
|
|
14347
|
-
|
|
14373
|
+
// ggui#1261 — the contract-feedback round's record rides the result the
|
|
14374
|
+
// generation ends with (whose probe stamp is the post-round verdict).
|
|
14375
|
+
evalResult: telemetry.evalResult !== void 0 && telemetry.contractFeedback !== void 0 ? { ...telemetry.evalResult, contractFeedback: telemetry.contractFeedback } : telemetry.evalResult,
|
|
14376
|
+
// ggui#404 — present only when the same-exchange guard ended the run.
|
|
14377
|
+
...telemetry.sameExchangeBreak !== void 0 ? { sameExchangeBreak: telemetry.sameExchangeBreak } : {},
|
|
14348
14378
|
needsBackgroundImprovement,
|
|
14349
14379
|
selfCheckPassed: telemetry.selfCheckPassed,
|
|
14350
14380
|
timing,
|
|
@@ -15107,7 +15137,8 @@ ${iconNamesCache}`;
|
|
|
15107
15137
|
lastDiffFailed: false,
|
|
15108
15138
|
isEvalFeedback: false,
|
|
15109
15139
|
iconNamesCache,
|
|
15110
|
-
preWarmedContext
|
|
15140
|
+
preWarmedContext,
|
|
15141
|
+
sameExchangeBreak: { tool: call.name, repeats: iconRepeats }
|
|
15111
15142
|
};
|
|
15112
15143
|
}
|
|
15113
15144
|
return {
|
|
@@ -15307,7 +15338,8 @@ Repeated error: ${errText}
|
|
|
15307
15338
|
lastDiffFailed: false,
|
|
15308
15339
|
isEvalFeedback: false,
|
|
15309
15340
|
iconNamesCache,
|
|
15310
|
-
preWarmedContext
|
|
15341
|
+
preWarmedContext,
|
|
15342
|
+
sameExchangeBreak: { tool: call.name, repeats: sameExchangeRepeats }
|
|
15311
15343
|
};
|
|
15312
15344
|
}
|
|
15313
15345
|
const turnDiffFailed = !!result.error && result.result.includes("DIFF");
|
|
@@ -15496,6 +15528,20 @@ function formatRuntimeProbeFeedback(issue) {
|
|
|
15496
15528
|
return `[runtime] ${issue.category}${subcat}: ${issue.description}
|
|
15497
15529
|
Fix: ${issue.fix ?? ""}`;
|
|
15498
15530
|
}
|
|
15531
|
+
var FEEDBACK_PROBE_CHECKS = ["prop-sensitivity"];
|
|
15532
|
+
var CONTRACT_FEEDBACK_BONUS = 1;
|
|
15533
|
+
function isFeedbackProbeFail(issue) {
|
|
15534
|
+
const sub = issue.subcategory;
|
|
15535
|
+
if (issue.result !== "fail" || typeof sub !== "string") return false;
|
|
15536
|
+
return FEEDBACK_PROBE_CHECKS.some((check) => sub === `runtime:${check}` || sub.startsWith(`runtime:${check}:`));
|
|
15537
|
+
}
|
|
15538
|
+
function contractFeedbackRecord(fails, sourceBefore) {
|
|
15539
|
+
return { firedOn: fails.flatMap((i) => i.subcategory !== void 0 ? [i.subcategory] : []), sourceBefore };
|
|
15540
|
+
}
|
|
15541
|
+
function contractFeedbackFails(exitProbe, alreadyFedBack) {
|
|
15542
|
+
if (!exitProbe.fired) return [];
|
|
15543
|
+
return exitProbe.probeIssues.filter((i) => isFeedbackProbeFail(i) && !alreadyFedBack.has(fingerprintFail(i)));
|
|
15544
|
+
}
|
|
15499
15545
|
async function runEvalRound(ctx, input) {
|
|
15500
15546
|
const {
|
|
15501
15547
|
workspace,
|
|
@@ -15685,7 +15731,9 @@ async function runEvalRound(ctx, input) {
|
|
|
15685
15731
|
costTracker.record(evaluationAgent.model, llmResult.inputTokens, llmResult.outputTokens);
|
|
15686
15732
|
evalTokens = {
|
|
15687
15733
|
input: llmResult.inputTokens,
|
|
15688
|
-
output: llmResult.outputTokens
|
|
15734
|
+
output: llmResult.outputTokens,
|
|
15735
|
+
...llmResult.cacheReadTokens !== void 0 ? { cacheRead: llmResult.cacheReadTokens } : {},
|
|
15736
|
+
...llmResult.cacheCreationTokens !== void 0 ? { cacheCreation: llmResult.cacheCreationTokens } : {}
|
|
15689
15737
|
};
|
|
15690
15738
|
}
|
|
15691
15739
|
const allIssues = [
|
|
@@ -15770,6 +15818,34 @@ async function runEvalRound(ctx, input) {
|
|
|
15770
15818
|
lastDiffFailed: false
|
|
15771
15819
|
};
|
|
15772
15820
|
}
|
|
15821
|
+
const contractFails = contractFeedbackFails(exitProbe, prevFailFingerprints);
|
|
15822
|
+
if (contractFails.length > 0 && evalRoundsUsed < maxEvalRounds + CONTRACT_FEEDBACK_BONUS) {
|
|
15823
|
+
const lines = contractFails.slice(0, MAX_FEEDBACK_ISSUES).map(formatRuntimeProbeFeedback);
|
|
15824
|
+
evalResult = {
|
|
15825
|
+
...evalResult,
|
|
15826
|
+
issues: [...allIssues, ...exitProbe.probeIssues],
|
|
15827
|
+
pass: allPass,
|
|
15828
|
+
runtimeProbe: exitProbe.meta
|
|
15829
|
+
};
|
|
15830
|
+
console.log(
|
|
15831
|
+
`[simple] eval round ${evalRoundsUsed}: tier-1/2 clean BUT contract probe fail \u2014 granting +1 turn (ggui#1261; ${contractFails.map((i) => i.subcategory).join(", ")})`
|
|
15832
|
+
);
|
|
15833
|
+
return {
|
|
15834
|
+
control: "feedback",
|
|
15835
|
+
evalDone: false,
|
|
15836
|
+
evalResult,
|
|
15837
|
+
evalRoundsUsed,
|
|
15838
|
+
prevModeSubcats: updatedPrevModeSubcats,
|
|
15839
|
+
prevFailFingerprints: /* @__PURE__ */ new Set([...currFailFingerprints, ...contractFails.map(fingerprintFail)]),
|
|
15840
|
+
preWarmedContext,
|
|
15841
|
+
evalTokens,
|
|
15842
|
+
evalLlmMs,
|
|
15843
|
+
lastResultText: lines.join("\n\n"),
|
|
15844
|
+
isEvalFeedback: true,
|
|
15845
|
+
lastDiffFailed: false,
|
|
15846
|
+
contractFeedback: contractFeedbackRecord(contractFails, currentSource)
|
|
15847
|
+
};
|
|
15848
|
+
}
|
|
15773
15849
|
evalResult = {
|
|
15774
15850
|
...evalResult,
|
|
15775
15851
|
issues: [...allIssues, ...exitProbe.probeIssues],
|
|
@@ -15809,6 +15885,28 @@ async function runEvalRound(ctx, input) {
|
|
|
15809
15885
|
issues: [...evalResult.issues, ...stuckExitProbe.probeIssues],
|
|
15810
15886
|
runtimeProbe: stuckExitProbe.meta
|
|
15811
15887
|
};
|
|
15888
|
+
const stuckContractFails = contractFeedbackFails(stuckExitProbe, prevFailFingerprints);
|
|
15889
|
+
if (stuckContractFails.length > 0 && evalRoundsUsed < maxEvalRounds + CONTRACT_FEEDBACK_BONUS) {
|
|
15890
|
+
const lines = stuckContractFails.slice(0, MAX_FEEDBACK_ISSUES).map(formatRuntimeProbeFeedback);
|
|
15891
|
+
console.log(
|
|
15892
|
+
`[simple] eval round ${evalRoundsUsed}: stuck BUT contract probe fail never delivered \u2014 granting +1 turn (ggui#1261; ${stuckContractFails.map((i) => i.subcategory).join(", ")})`
|
|
15893
|
+
);
|
|
15894
|
+
return {
|
|
15895
|
+
control: "feedback",
|
|
15896
|
+
evalDone: false,
|
|
15897
|
+
evalResult,
|
|
15898
|
+
evalRoundsUsed,
|
|
15899
|
+
prevModeSubcats: updatedPrevModeSubcats,
|
|
15900
|
+
prevFailFingerprints: /* @__PURE__ */ new Set([...currFailFingerprints, ...stuckContractFails.map(fingerprintFail)]),
|
|
15901
|
+
preWarmedContext,
|
|
15902
|
+
evalTokens,
|
|
15903
|
+
evalLlmMs,
|
|
15904
|
+
lastResultText: lines.join("\n\n"),
|
|
15905
|
+
isEvalFeedback: true,
|
|
15906
|
+
lastDiffFailed: false,
|
|
15907
|
+
contractFeedback: contractFeedbackRecord(stuckContractFails, currentSource)
|
|
15908
|
+
};
|
|
15909
|
+
}
|
|
15812
15910
|
return {
|
|
15813
15911
|
control: "break",
|
|
15814
15912
|
evalDone: false,
|
|
@@ -15863,6 +15961,33 @@ async function runEvalRound(ctx, input) {
|
|
|
15863
15961
|
lastDiffFailed: false
|
|
15864
15962
|
};
|
|
15865
15963
|
}
|
|
15964
|
+
const capContractFails = allowRuntimeExtension ? [] : contractFeedbackFails(exitProbe, prevFailFingerprints);
|
|
15965
|
+
if (capContractFails.length > 0 && evalRoundsUsed < maxEvalRounds + CONTRACT_FEEDBACK_BONUS) {
|
|
15966
|
+
const contractLines = capContractFails.slice(0, MAX_FEEDBACK_ISSUES).map(formatRuntimeProbeFeedback);
|
|
15967
|
+
evalResult = {
|
|
15968
|
+
...evalResult,
|
|
15969
|
+
issues: [...evalResult.issues, ...exitProbe.probeIssues],
|
|
15970
|
+
runtimeProbe: exitProbe.meta
|
|
15971
|
+
};
|
|
15972
|
+
console.log(
|
|
15973
|
+
`[simple] eval round ${evalRoundsUsed}: contract probe fail active at cap \u2014 granting +1 retry (ggui#1261; ${capContractFails.map((i) => i.subcategory).join(", ")})`
|
|
15974
|
+
);
|
|
15975
|
+
return {
|
|
15976
|
+
control: "feedback",
|
|
15977
|
+
evalDone: false,
|
|
15978
|
+
evalResult,
|
|
15979
|
+
evalRoundsUsed,
|
|
15980
|
+
prevModeSubcats: updatedPrevModeSubcats,
|
|
15981
|
+
prevFailFingerprints: /* @__PURE__ */ new Set([...currFailFingerprints, ...capContractFails.map(fingerprintFail)]),
|
|
15982
|
+
preWarmedContext,
|
|
15983
|
+
evalTokens,
|
|
15984
|
+
evalLlmMs,
|
|
15985
|
+
lastResultText: contractLines.join("\n\n"),
|
|
15986
|
+
isEvalFeedback: true,
|
|
15987
|
+
lastDiffFailed: false,
|
|
15988
|
+
contractFeedback: contractFeedbackRecord(capContractFails, currentSource)
|
|
15989
|
+
};
|
|
15990
|
+
}
|
|
15866
15991
|
if (!allowRuntimeExtension) {
|
|
15867
15992
|
evalResult = {
|
|
15868
15993
|
...evalResult,
|
|
@@ -16005,6 +16130,8 @@ function createTelemetry() {
|
|
|
16005
16130
|
totalIn: 0,
|
|
16006
16131
|
totalOut: 0,
|
|
16007
16132
|
evalResult: void 0,
|
|
16133
|
+
contractFeedback: void 0,
|
|
16134
|
+
sameExchangeBreak: void 0,
|
|
16008
16135
|
compiledCode: "",
|
|
16009
16136
|
pairedSource: "",
|
|
16010
16137
|
selfCheckPassed: false,
|
|
@@ -16013,6 +16140,40 @@ function createTelemetry() {
|
|
|
16013
16140
|
codingMs: 0
|
|
16014
16141
|
};
|
|
16015
16142
|
}
|
|
16143
|
+
function absorbTokens(telemetry, usage) {
|
|
16144
|
+
telemetry.totalIn += usage.input;
|
|
16145
|
+
telemetry.totalOut += usage.output;
|
|
16146
|
+
if (usage.cacheRead !== void 0) {
|
|
16147
|
+
telemetry.cacheReadTokens = (telemetry.cacheReadTokens ?? 0) + usage.cacheRead;
|
|
16148
|
+
}
|
|
16149
|
+
if (usage.cacheCreation !== void 0) {
|
|
16150
|
+
telemetry.cacheCreationTokens = (telemetry.cacheCreationTokens ?? 0) + usage.cacheCreation;
|
|
16151
|
+
}
|
|
16152
|
+
}
|
|
16153
|
+
function absorbTurn(telemetry, turn) {
|
|
16154
|
+
absorbTokens(telemetry, {
|
|
16155
|
+
input: turn.tokens.input,
|
|
16156
|
+
output: turn.tokens.output,
|
|
16157
|
+
...turn.cacheReadTokens !== void 0 ? { cacheRead: turn.cacheReadTokens } : {},
|
|
16158
|
+
...turn.cacheCreationTokens !== void 0 ? { cacheCreation: turn.cacheCreationTokens } : {}
|
|
16159
|
+
});
|
|
16160
|
+
telemetry.cumulativeLlmMs += turn.llmMs;
|
|
16161
|
+
telemetry.cumulativeToolMs += turn.toolMs;
|
|
16162
|
+
if (turn.phase) {
|
|
16163
|
+
if (turn.phase === "scaffold") telemetry.counters.phases.scaffold++;
|
|
16164
|
+
else if (turn.phase === "fill") telemetry.counters.phases.fill++;
|
|
16165
|
+
else if (turn.phase === "impl") telemetry.counters.phases.impl++;
|
|
16166
|
+
else if (turn.phase === "eval-fix") telemetry.counters.phases.evalFix++;
|
|
16167
|
+
else telemetry.counters.phases.patch++;
|
|
16168
|
+
}
|
|
16169
|
+
if (turn.outcome) {
|
|
16170
|
+
if (turn.outcome === "PASS") telemetry.counters.outcomes.pass++;
|
|
16171
|
+
else if (turn.outcome === "PATCH_INVALID") telemetry.counters.outcomes.patchInvalid++;
|
|
16172
|
+
else if (turn.outcome === "DIFF_FAIL") telemetry.counters.outcomes.diffFail++;
|
|
16173
|
+
else telemetry.counters.outcomes.selfCheckFail++;
|
|
16174
|
+
}
|
|
16175
|
+
if (turn.sameExchangeBreak !== void 0) telemetry.sameExchangeBreak = turn.sameExchangeBreak;
|
|
16176
|
+
}
|
|
16016
16177
|
function createGenerateTaskRunner(input) {
|
|
16017
16178
|
const { session, params, classification, telemetry } = input;
|
|
16018
16179
|
const maxTurns = input.maxTurns ?? 30;
|
|
@@ -16065,31 +16226,9 @@ function createGenerateTaskRunner(input) {
|
|
|
16065
16226
|
preWarmPromise: session.preWarmPromise
|
|
16066
16227
|
}
|
|
16067
16228
|
);
|
|
16068
|
-
telemetry
|
|
16069
|
-
telemetry.totalOut += turn.tokens.output;
|
|
16070
|
-
if (turn.cacheReadTokens !== void 0) {
|
|
16071
|
-
telemetry.cacheReadTokens = (telemetry.cacheReadTokens ?? 0) + turn.cacheReadTokens;
|
|
16072
|
-
}
|
|
16073
|
-
if (turn.cacheCreationTokens !== void 0) {
|
|
16074
|
-
telemetry.cacheCreationTokens = (telemetry.cacheCreationTokens ?? 0) + turn.cacheCreationTokens;
|
|
16075
|
-
}
|
|
16076
|
-
telemetry.cumulativeLlmMs += turn.llmMs;
|
|
16077
|
-
telemetry.cumulativeToolMs += turn.toolMs;
|
|
16229
|
+
absorbTurn(telemetry, turn);
|
|
16078
16230
|
iconNamesCache = turn.iconNamesCache;
|
|
16079
16231
|
preWarmedContext = turn.preWarmedContext;
|
|
16080
|
-
if (turn.phase) {
|
|
16081
|
-
if (turn.phase === "scaffold") telemetry.counters.phases.scaffold++;
|
|
16082
|
-
else if (turn.phase === "fill") telemetry.counters.phases.fill++;
|
|
16083
|
-
else if (turn.phase === "impl") telemetry.counters.phases.impl++;
|
|
16084
|
-
else if (turn.phase === "eval-fix") telemetry.counters.phases.evalFix++;
|
|
16085
|
-
else telemetry.counters.phases.patch++;
|
|
16086
|
-
}
|
|
16087
|
-
if (turn.outcome) {
|
|
16088
|
-
if (turn.outcome === "PASS") telemetry.counters.outcomes.pass++;
|
|
16089
|
-
else if (turn.outcome === "PATCH_INVALID") telemetry.counters.outcomes.patchInvalid++;
|
|
16090
|
-
else if (turn.outcome === "DIFF_FAIL") telemetry.counters.outcomes.diffFail++;
|
|
16091
|
-
else telemetry.counters.outcomes.selfCheckFail++;
|
|
16092
|
-
}
|
|
16093
16232
|
if (turn.control === "break") break;
|
|
16094
16233
|
if (turn.control === "continue") {
|
|
16095
16234
|
lastResultText = turn.lastResultText;
|
|
@@ -16165,9 +16304,9 @@ function createGenerateTaskRunner(input) {
|
|
|
16165
16304
|
prevFailFingerprints = round.prevFailFingerprints;
|
|
16166
16305
|
preWarmedContext = round.preWarmedContext;
|
|
16167
16306
|
if (round.evalResult) telemetry.evalResult = round.evalResult;
|
|
16307
|
+
if (round.contractFeedback) telemetry.contractFeedback = round.contractFeedback;
|
|
16168
16308
|
evalDone = round.evalDone;
|
|
16169
|
-
telemetry
|
|
16170
|
-
telemetry.totalOut += round.evalTokens.output;
|
|
16309
|
+
absorbTokens(telemetry, round.evalTokens);
|
|
16171
16310
|
if (round.control === "break") break;
|
|
16172
16311
|
lastResultText = round.lastResultText;
|
|
16173
16312
|
isEvalFeedback = round.isEvalFeedback;
|