@ggui-ai/ui-gen 0.21.0 → 0.23.0

package/src/harness/coding/run-eval-round.ts

@@ -22,7 +22,7 @@ import { listContractGadgets } from "@ggui-ai/protocol";
 import type { Classification } from "../../classifier/index.js";
 import type { AgentWorkspace } from "../../coding-agent/workspace.js";
 import type { CostTracker } from "../../evaluation/cost-tracker.js";
-import type { EvalIssue, EvalResult, RuntimeProbeMeta, VisualEvalSummary, VisualCoverage } from "../../evaluation/types-public.js";
+import type { ContractFeedbackRecord, EvalIssue, EvalResult, RuntimeProbeMeta, VisualEvalSummary, VisualCoverage } from "../../evaluation/types-public.js";
 import { notApplicableCoverage } from "../../evaluation/types-public.js";
 import { mapProviderForEvaluator, visionJudgeProvider } from "../enforced-coding.js";
 import { runCheck } from "../index.js";
@@ -40,6 +40,7 @@ import type {
 type PreWarmedEvalContext =
   import("../../evaluation/llm-evaluator.js").PreWarmedEvalContext;
 type LlmEvalMod = typeof import("../../evaluation/llm-evaluator.js");
+type LLMEvalTokenUsage = import("../../evaluation/llm-evaluator.js").LLMEvalTokenUsage;
 type VisualEvalMod = typeof import("../../evaluation/visual-evaluator.js");
 
 /**
@@ -100,6 +101,20 @@ export interface EvalRoundInput {
  */
 export type EvalRoundControl = "break" | "feedback";
 
+/**
+ * One call's (or one round's) token accounting. `input` is the NON-cached
+ * input; the cache counts are present only when the provider reported them —
+ * absent means unreported, never zero. The runner folds every coding turn and
+ * every eval round through the same rule (`absorbTokens`), so the published
+ * `tokens.total` holds one convention across both (ggui#1281, #1186).
+ */
+export interface TokenUsage {
+  readonly input: number;
+  readonly output: number;
+  readonly cacheRead?: number;
+  readonly cacheCreation?: number;
+}
+
 export interface EvalRoundResult {
   readonly control: EvalRoundControl;
   /** True only on the low-risk bypass + clean-pass paths. */
@@ -115,7 +130,7 @@ export interface EvalRoundResult {
   readonly prevFailFingerprints: Set<string>;
   readonly preWarmedContext: PreWarmedEvalContext | null | undefined;
   /** Tokens added during this round — caller adds to its own totals. */
-  readonly evalTokens: { input: number; output: number };
+  readonly evalTokens: TokenUsage;
   /** Wall-clock of the parallel LLM + visual eval block (zero on the
    *  low-risk bypass path, which exits before `Promise.all`). Caller
    *  accumulates into its `cumulativeEvalLlmMs`. */
@@ -124,6 +139,13 @@ export interface EvalRoundResult {
   readonly lastResultText: string;
   readonly isEvalFeedback: boolean;
   readonly lastDiffFailed: boolean;
+  /**
+   * ggui#1261 — set ONLY on the round that granted a contract-feedback round:
+   * what bought it and the source it was fed back on. The caller keeps it past
+   * this round (the next round rebuilds `evalResult`) and stamps it on the
+   * result the generation ends with.
+   */
+  readonly contractFeedback?: ContractFeedbackRecord;
 }
 
 /** Cap feedback to N issues per eval-fix turn so the LLM stays surgical. */
@@ -337,6 +359,8 @@ async function runProbeAtExit(input: {
   const meta: RuntimeProbeMeta = {
     status: outcome.status,
     ...(outcome.reason !== undefined ? { reason: outcome.reason } : {}),
+    ...(outcome.elapsedMs !== undefined ? { elapsedMs: outcome.elapsedMs } : {}),
+    ...(outcome.hostLoad !== undefined ? { hostLoad: outcome.hostLoad } : {}),
   };
   if (outcome.status !== "ran") {
     return { fired: false, recoverableFail: false, probeIssues: [], meta };
@@ -368,6 +392,42 @@ function formatRuntimeProbeFeedback(issue: EvalIssue): string {
   );
 }
 
+/**
+ * Exit-probe checks whose FAIL buys ONE feedback round although it is not a
+ * crash: a contract defect the probe diagnoses deterministically, with a fix
+ * text the model can act on in a turn. Without the round the loop recorded the
+ * finding and shipped it — its own evaluator named the defect and the model
+ * never heard. Named, and grown only with evidence: `prop-sensitivity` (a
+ * declared prop ignored, a literal baked in its place — "You" for
+ * `currentUser`) is the class measured failing across models (ggui#1261).
+ */
+const FEEDBACK_PROBE_CHECKS: readonly string[] = ["prop-sensitivity"];
+
+/** Rounds past the cap the contract-feedback round may use: one, as the fit round (ggui#1195). */
+const CONTRACT_FEEDBACK_BONUS = 1;
+
+/** `runtime:<check>` or `runtime:<check>:<subject>` for a named feedback check, FAIL only. */
+function isFeedbackProbeFail(issue: EvalIssue): boolean {
+  const sub = issue.subcategory;
+  if (issue.result !== "fail" || typeof sub !== "string") return false;
+  return FEEDBACK_PROBE_CHECKS.some((check) => sub === `runtime:${check}` || sub.startsWith(`runtime:${check}:`));
+}
+
+/**
+ * The exit probe's feedback-worthy FAILs this generation has not been told
+ * about yet. A finding whose fingerprint already rode a feedback round and
+ * recurred is left out — one attempt per finding, then it ships as recorded.
+ */
+/** The round's record: the findings that bought it, and the source they were found on. */
+function contractFeedbackRecord(fails: readonly EvalIssue[], sourceBefore: string): ContractFeedbackRecord {
+  return { firedOn: fails.flatMap((i) => (i.subcategory !== undefined ? [i.subcategory] : [])), sourceBefore };
+}
+
+function contractFeedbackFails(exitProbe: ProbeAtExitResult, alreadyFedBack: ReadonlySet<string>): EvalIssue[] {
+  if (!exitProbe.fired) return [];
+  return exitProbe.probeIssues.filter((i) => isFeedbackProbeFail(i) && !alreadyFedBack.has(fingerprintFail(i)));
+}
+
 /**
  * Run one eval round. Mutates `ctx.costTracker` (records LLM eval tokens)
  * and `ctx.workspace` is read-only from this function's POV. Returns
@@ -404,7 +464,7 @@ export async function runEvalRound(
   const currentSource = workspace.read() ?? "";
 
   let evalResult: EvalResult | undefined;
-  let evalTokens = { input: 0, output: 0 };
+  let evalTokens: TokenUsage = { input: 0, output: 0 };
   let evalLlmMs = 0;
 
   try {
@@ -528,7 +588,7 @@ export async function runEvalRound(
     }
 
     // ── Tier 1+2: LLM evaluation + Visual eval in parallel ──
-    let llmResult: (EvalResult & { inputTokens: number; outputTokens: number }) | null = null;
+    let llmResult: (EvalResult & LLMEvalTokenUsage) | null = null;
     let visualIssues: EvalIssue[] | null = null;
 
     // The visual leg's page and frame — ONE value each, read by the judge and
@@ -629,6 +689,8 @@ export async function runEvalRound(
       evalTokens = {
         input: llmResult.inputTokens,
         output: llmResult.outputTokens,
+        ...(llmResult.cacheReadTokens !== undefined ? { cacheRead: llmResult.cacheReadTokens } : {}),
+        ...(llmResult.cacheCreationTokens !== undefined ? { cacheCreation: llmResult.cacheCreationTokens } : {}),
       };
     }
 
@@ -748,6 +810,40 @@ export async function runEvalRound(
           lastDiffFailed: false,
         };
       }
+      // ggui#1261 — a contract defect the probe diagnosed (a named check,
+      // FAIL) that this generation has not been told about: ONE feedback
+      // round with the probe's own diagnosis and fix. The next round re-runs
+      // the probe at its exit, so the result this generation ends with
+      // carries the post-fix stamp, never this one.
+      const contractFails = contractFeedbackFails(exitProbe, prevFailFingerprints);
+      if (contractFails.length > 0 && evalRoundsUsed < maxEvalRounds + CONTRACT_FEEDBACK_BONUS) {
+        const lines = contractFails.slice(0, MAX_FEEDBACK_ISSUES).map(formatRuntimeProbeFeedback);
+        evalResult = {
+          ...evalResult,
+          issues: [...allIssues, ...exitProbe.probeIssues],
+          pass: allPass,
+          runtimeProbe: exitProbe.meta,
+        };
+        console.log(
+          `[simple] eval round ${evalRoundsUsed}: tier-1/2 clean BUT contract probe fail — granting +1 turn ` +
+            `(ggui#1261; ${contractFails.map((i) => i.subcategory).join(", ")})`,
+        );
+        return {
+          control: "feedback",
+          evalDone: false,
+          evalResult,
+          evalRoundsUsed,
+          prevModeSubcats: updatedPrevModeSubcats,
+          prevFailFingerprints: new Set([...currFailFingerprints, ...contractFails.map(fingerprintFail)]),
+          preWarmedContext,
+          evalTokens,
+          evalLlmMs,
+          lastResultText: lines.join("\n\n"),
+          isEvalFeedback: true,
+          lastDiffFailed: false,
+          contractFeedback: contractFeedbackRecord(contractFails, currentSource),
+        };
+      }
       // Probe didn't trip a recoverable fail — fold its issues (if any)
       // into the merged result and ALWAYS stamp its execution meta, so a
       // did-not-run probe is visible downstream even on this clean exit.
@@ -811,7 +907,8 @@ export async function runEvalRound(
       // the runtimeProbe stamp lands (Exp 48 found this exit surfacing
       // as "SKIP — no runtimeProbe stamp" in the bench: honest, but a
       // probe here is cheap and the shipped code deserves the signal).
-      // No feedback is granted — stuck means the LLM stopped converging.
+      // The RECURRING fails get no more feedback — stuck means the model
+      // saw them and stopped converging on them.
       const stuckExitProbe = await runProbeAtExit({
         harness,
         sourceCode: currentSource,
@@ -824,6 +921,36 @@ export async function runEvalRound(
         issues: [...evalResult.issues, ...stuckExitProbe.probeIssues],
         runtimeProbe: stuckExitProbe.meta,
       };
+      // ggui#1261 — a contract defect the exit probe diagnosed here was never
+      // DELIVERED: the stuck set is non-runtime by construction (runtime-only
+      // sets are exempt above), and the stuck premise is "the model saw these
+      // and did not resolve them". So it gets the same one round as at the
+      // clean-PASS and cap exits — the probe's diagnosis only, never the
+      // recurring fail; one attempt per finding; bounded at the cap + 1 (Exp
+      // 011, candidate r5: the one FAIL that shipped without a round).
+      const stuckContractFails = contractFeedbackFails(stuckExitProbe, prevFailFingerprints);
+      if (stuckContractFails.length > 0 && evalRoundsUsed < maxEvalRounds + CONTRACT_FEEDBACK_BONUS) {
+        const lines = stuckContractFails.slice(0, MAX_FEEDBACK_ISSUES).map(formatRuntimeProbeFeedback);
+        console.log(
+          `[simple] eval round ${evalRoundsUsed}: stuck BUT contract probe fail never delivered — granting +1 turn ` +
+            `(ggui#1261; ${stuckContractFails.map((i) => i.subcategory).join(", ")})`,
+        );
+        return {
+          control: "feedback",
+          evalDone: false,
+          evalResult,
+          evalRoundsUsed,
+          prevModeSubcats: updatedPrevModeSubcats,
+          prevFailFingerprints: new Set([...currFailFingerprints, ...stuckContractFails.map(fingerprintFail)]),
+          preWarmedContext,
+          evalTokens,
+          evalLlmMs,
+          lastResultText: lines.join("\n\n"),
+          isEvalFeedback: true,
+          lastDiffFailed: false,
+          contractFeedback: contractFeedbackRecord(stuckContractFails, currentSource),
+        };
+      }
       return {
         control: "break",
         evalDone: false,
@@ -909,6 +1036,36 @@ export async function runEvalRound(
           lastDiffFailed: false,
         };
       }
+      // ggui#1261 — a named contract FAIL from the exit probe active at the
+      // cap: the same one round as at the clean exit, bounded at the cap + 1.
+      const capContractFails = allowRuntimeExtension ? [] : contractFeedbackFails(exitProbe, prevFailFingerprints);
+      if (capContractFails.length > 0 && evalRoundsUsed < maxEvalRounds + CONTRACT_FEEDBACK_BONUS) {
+        const contractLines = capContractFails.slice(0, MAX_FEEDBACK_ISSUES).map(formatRuntimeProbeFeedback);
+        evalResult = {
+          ...evalResult,
+          issues: [...evalResult.issues, ...exitProbe.probeIssues],
+          runtimeProbe: exitProbe.meta,
+        };
+        console.log(
+          `[simple] eval round ${evalRoundsUsed}: contract probe fail active at cap — granting +1 retry ` +
+            `(ggui#1261; ${capContractFails.map((i) => i.subcategory).join(", ")})`,
+        );
+        return {
+          control: "feedback",
+          evalDone: false,
+          evalResult,
+          evalRoundsUsed,
+          prevModeSubcats: updatedPrevModeSubcats,
+          prevFailFingerprints: new Set([...currFailFingerprints, ...capContractFails.map(fingerprintFail)]),
+          preWarmedContext,
+          evalTokens,
+          evalLlmMs,
+          lastResultText: contractLines.join("\n\n"),
+          isEvalFeedback: true,
+          lastDiffFailed: false,
+          contractFeedback: contractFeedbackRecord(capContractFails, currentSource),
+        };
+      }
       if (!allowRuntimeExtension) {
         // Fold probe issues (if any) into evalResult for telemetry and
         // ALWAYS stamp the probe's execution meta, even though we're not

package/src/harness/llm-router.ts

@@ -875,8 +875,8 @@ export class AnthropicAgent extends LLMAgent {
           : toolChoice;
       if (appliedToolChoice !== toolChoice) {
         console.warn(
-          `[anthropic] callTools: ${normalizeAnthropicModelId(resolvedModel)} rejects forced tool_choice (HTTP 400); ` +
-            `caller's 'required' downgraded to 'auto'`,
+          `[anthropic] callTools: ${normalizeAnthropicModelId(resolvedModel)} would reject a forced tool_choice (the API answers 400); ` +
+            `caller's 'required' downgraded to 'auto' before sending`,
         );
       }
       const response = await this.apiCall(() =>
@@ -2128,7 +2128,7 @@ export class OpenRouterAgent extends LLMAgent {
       requested === 'required' && anthropicRejectsForcedToolChoice(resolved) ? 'auto' : requested;
     if (appliedToolChoice !== requested) {
       console.warn(
-        `[openrouter] callTools: ${resolved} rejects forced tool_choice (HTTP 400); caller's 'required' downgraded to 'auto'`,
+        `[openrouter] callTools: ${resolved} would reject a forced tool_choice (the API answers 400); caller's 'required' downgraded to 'auto' before sending`,
       );
     }
 

package/src/harness/result-types.ts

@@ -49,11 +49,25 @@ export type ModelRoles = Partial<Record<ModelRole, string>>;
 // Generation Result
 // =============================================================================
 
+/**
+ * The same-exchange guard (ggui#404) ended this generation: the coding model
+ * made one identical tool exchange (same tool, same input, same result)
+ * `repeats` times, so the loop stopped instead of burning its remaining turns.
+ * Present ONLY when that guard ended the run — whether anything shipped is
+ * the compiled code's to say (the loop falls back to an earlier build).
+ */
+export interface SameExchangeBreak {
+  readonly tool: string;
+  readonly repeats: number;
+}
+
 export interface GenerationResult extends AdapterResult {
   /** Number of generation passes the harness performed */
   passesUsed: number;
   /** Three-tier evaluation result (tier 0 + LLM tier 1+2 + visual) */
   evalResult?: EvalResult;
+  /** Set only when the same-exchange guard ended the run (ggui#404); no key otherwise. */
+  sameExchangeBreak?: SameExchangeBreak;
   /** Whether background improvement should be spawned (auto-improve mode) */
   needsBackgroundImprovement?: boolean;
   /** Whether at least one commit passed the self-check (compile + type check + lint) */

package/src/harness/types-public.ts

@@ -21,7 +21,7 @@
 import type { GenerationProfileInput } from "../boilerplate/styling-profile.js";
 import type { GadgetDescriptor, DataContract, JsonValue } from "@ggui-ai/protocol";
 import type { Classification } from "../classifier/axes.js";
-import type { EvalIssue, EvalTier, AxisCheck, RuntimeProbeStatus } from "../evaluation/types-public.js";
+import type { EvalIssue, EvalTier, AxisCheck, ProbeHostLoad, RuntimeProbeStatus } from "../evaluation/types-public.js";
 import type { LLMToolDef } from "../llm.js";
 import type { CacheTier, HarnessFragment } from "../fragments/index.js";
 import type { HarnessPolicy, ProcessMode } from "../policy.js";
@@ -140,14 +140,18 @@ export interface LLMEvaluator {
 export interface RuntimeRenderOutcome {
   /**
    * Whether the probe actually executed. `issues` is only probe
-   * evidence when `status === "ran"` — an `infra-skipped` or
-   * `not-applicable` outcome carries zero evidence and MUST NOT be
-   * scored as a pass by any consumer.
+   * evidence when `status === "ran"` — an `infra-skipped`,
+   * `not-applicable` or `timed-out` outcome carries zero evidence and
+   * MUST NOT be scored as a pass by any consumer.
    */
   readonly status: RuntimeProbeStatus;
   readonly issues: readonly EvalIssue[];
-  /** Populated for `infra-skipped` / `not-applicable` — why the probe didn't run. */
+  /** Populated for `infra-skipped` / `not-applicable` / `timed-out` — why the probe didn't finish. */
   readonly reason?: string;
+  /** `timed-out` only: how long the probe ran before it was stopped. */
+  readonly elapsedMs?: number;
+  /** Host load around the probe, when it ran isolated (absent for an in-process probe). */
+  readonly hostLoad?: ProbeHostLoad;
 }
 
 export interface RuntimeRenderCheck {

package/src/index.ts

@@ -21,7 +21,8 @@ export {
 } from './create-ui-generator.js';
 export type { CreateUiGeneratorOptions } from './create-ui-generator.js';
 export type { ProviderRetryInfo } from './harness/llm-router.js';
-export type { GenerationResult } from './harness/result-types.js';
+export type { GenerationResult, SameExchangeBreak } from './harness/result-types.js';
+export type { UiGenBuild } from './generator-build.js';
 
 // Generator registry seam re-export. The interface + slug helpers live
 // in `@ggui-ai/mcp-server-core`; the in-memory factory lives in