@ggui-ai/ui-gen 0.25.0 → 0.27.0

package/src/evaluation/index.ts

@@ -129,6 +129,7 @@ export type {
   ScreenshotAttempt,
   CanvasVisualResult,
   CriteriaCallTokens,
+  VisualLegTokens,
   CanvasFitPolicy,
   CaptureMode,
   ScreenshotDeps,
@@ -168,6 +169,9 @@ export { designTreeSha256, judgeDesignIdentity, resetJudgeDesignIdentityCache, t
 export { bankRows, criteriaBankSchema, parseCriteriaBank, type BankRow, type CriteriaBank, type CriteriaScope } from './criteria/bank.js';
 export { CRITERIA_SELECTOR_VERSION, canonicalCriteriaContext, criteriaSetIdFor, selectCriteria, type CriteriaSelectionResult } from './criteria/select.js';
 export { chromaOfCssTokens, criteriaContextFor, type CriteriaContextInput, type CriteriaContextSources } from './criteria/context.js';
-export { INSTRUMENT_BY_ID, buildCriteriaJudgeBlock, resolveCriteriaBlock, type CriteriaMeasurements } from './criteria/resolve.js';
+export { INSTRUMENT_BESIDE_JUDGE, INSTRUMENT_BY_ID, buildCriteriaJudgeBlock, resolveCriteriaBlock, type CriteriaMeasurements } from './criteria/resolve.js';
+export { EDGE_TOLERANCE_PX, judgeEdges, type EdgeBlock, type EdgeOffset, type EdgeVerdict } from './criteria/edge.js';
+export { EDGE_BLOCKS_EXPRESSION, parseEdgeProbe, type EdgeProbe } from './criteria/edge-probe.js';
 export { judgeStoredCapture, type StoredCaptureInput, type StoredCaptureOutcome, type StoredCaptureVerdict } from './visual-evaluator.js';
+export { judgedFrameTokens, type JudgeSpend, type JudgedTokens } from './judge-spend.js';
 

package/src/evaluation/judge-spend.ts

@@ -0,0 +1,48 @@
+/**
+ * ggui#1640 — one judged frame's (or one visual evaluation's) spend, summed over EVERY leg the judge reports.
+ *
+ * The judge reports its legs apart: the scoring calls' `inputTokens` / `outputTokens`, and, since ggui#1436, the
+ * report-only criteria call's `criteriaTokens`. A caller that sums only the first leg under-counts, and a dollar
+ * ceiling built on that sum stops late. {@link judgedFrameTokens} is the one sum. The two `satisfies` tables below list
+ * every `*Tokens` member of the results it reads, so a leg added to either result type fails the build here until the
+ * sum counts it.
+ */
+import type { CriteriaCallTokens, StoredCaptureVerdict, VisualEvaluationResult } from './visual-evaluator.js';
+
+/** A spend in tokens: input and output, over every leg. */
+export interface JudgeSpend {
+  readonly inputTokens: number;
+  readonly outputTokens: number;
+}
+
+/** The members of `T` whose names end in `Tokens`. */
+type TokenMember<T> = { [K in keyof T]-?: K extends `${string}Tokens` ? K : never }[keyof T];
+
+/** Which leg each `*Tokens` member belongs to. A member missing here, or listed and gone, is a compile error. */
+const STORED_VERDICT_LEGS = {
+  inputTokens: 'scoring',
+  outputTokens: 'scoring',
+  criteriaTokens: 'criteria',
+} as const satisfies Record<TokenMember<StoredCaptureVerdict>, 'scoring' | 'criteria'>;
+const VISUAL_RESULT_LEGS = {
+  inputTokens: 'scoring',
+  outputTokens: 'scoring',
+  criteriaTokens: 'criteria',
+} as const satisfies Record<TokenMember<VisualEvaluationResult>, 'scoring' | 'criteria'>;
+void STORED_VERDICT_LEGS;
+void VISUAL_RESULT_LEGS;
+
+/** The legs {@link judgedFrameTokens} reads, as both result types carry them. */
+export interface JudgedTokens {
+  readonly inputTokens?: number;
+  readonly outputTokens?: number;
+  readonly criteriaTokens?: CriteriaCallTokens;
+}
+
+/** A judged frame's spend over every leg: the scoring calls plus the report-only criteria call, when one ran. */
+export function judgedFrameTokens(result: JudgedTokens): JudgeSpend {
+  return {
+    inputTokens: (result.inputTokens ?? 0) + (result.criteriaTokens?.inputTokens ?? 0),
+    outputTokens: (result.outputTokens ?? 0) + (result.criteriaTokens?.outputTokens ?? 0),
+  };
+}

package/src/evaluation/loop.ts

@@ -1,6 +1,7 @@
 // packages/ui-gen/src/evaluation/loop.ts
 
 import { query, type McpServerConfig } from '@anthropic-ai/claude-agent-sdk';
+import { sdkToolPins } from '../adapters/claude/sdk-tool-pins';
 import { runEvaluation } from './evaluator';
 import {
   extractCompiledCodeFromMessage,
@@ -39,8 +40,13 @@ export interface EvaluationLoopOptions {
     cwd?: string;
     /** MCP servers available to the fix agent */
     mcpServers?: Record<string, McpServerConfig>;
-    /** Tools the fix agent is allowed to call */
+    /** Tool names to approve. Absent → every tool of every server in `mcpServers` (ggui#1714). */
     allowedTools?: string[];
+    /**
+     * CLI built-in tools to offer the fix agent (e.g. `['Write']`). Default: none.
+     * An explicit opt-in: a built-in that writes or executes acts on the host (ggui#1714).
+     */
+    builtinTools?: readonly string[];
     /** LLM model for fix rounds */
     model?: string;
     /** Environment variables (includes BYOK credentials) */
@@ -90,6 +96,18 @@ export async function runEvaluationLoop(
   let round = 0;
   const maxRounds = Math.min(config.maxRounds ?? 3, MAX_EVAL_ROUNDS_HARD_LIMIT);
 
+  // ggui#1714 — a fix round runs with no CLI built-ins unless asked for. With no
+  // MCP server and no built-in either, the fix agent has nothing that can return
+  // code, and every fix round would silently hand back the original: refuse.
+  const offersATool =
+    Object.keys(generatorOptions?.mcpServers ?? {}).length > 0 || (generatorOptions?.builtinTools?.length ?? 0) > 0;
+  if (maxRounds > 1 && !offersATool) {
+    throw new Error(
+      'runEvaluationLoop: a fix round needs a tool that can return code — pass generatorOptions.mcpServers ' +
+        "(e.g. a server with compile_component) or opt into a built-in with generatorOptions.builtinTools (e.g. ['Write']).",
+    );
+  }
+
   while (round < maxRounds) {
     round++;
 
@@ -145,12 +163,15 @@ export async function runEvaluationLoop(
         maxTurns: 15,
         maxBudgetUsd: config.maxBudgetPerFix,
         env,
-        permissionMode: 'bypassPermissions',
-        allowDangerouslySkipPermissions: true,
         ...(cliPath && { pathToClaudeCodeExecutable: cliPath }),
         ...(generatorOptions?.cwd && { cwd: generatorOptions.cwd }),
         ...(generatorOptions?.mcpServers && { mcpServers: generatorOptions.mcpServers }),
-        ...(generatorOptions?.allowedTools && { allowedTools: generatorOptions.allowedTools }),
+        // ggui#1714 — no CLI built-ins unless asked for, no host settings, no permission bypass.
+        ...sdkToolPins({
+          mcpServers: generatorOptions?.mcpServers,
+          allowedTools: generatorOptions?.allowedTools,
+          builtinTools: generatorOptions?.builtinTools,
+        }),
         ...(generatorOptions?.model && { model: generatorOptions.model }),
         ...(generatorOptions?.stderr && { stderr: generatorOptions.stderr }),
       },

package/src/evaluation/message-parsing.ts

@@ -10,6 +10,7 @@
  */
 
 import type { EvaluationResult } from './types';
+import { sourceCodeFromToolUse } from '../adapters/claude/source-capture';
 
 // ── SDK message type aliases ────────────────────────────────────────
 
@@ -137,13 +138,14 @@ export function extractCompiledCodeFromMessage(message: SdkMessage): string | un
 // ── sourceCode extraction ───────────────────────────────────────────
 
 /**
- * Extract source code from an assistant's Write tool_use message.
+ * Extract source code from an assistant message's tool_use blocks.
  *
- * Looks for `tool_use` blocks with `name === 'Write'` and returns
- * the `input.content` string. If multiple Write calls exist, returns
- * the last one (the final version).
+ * Reads the bridged compile tool (`*__compile_component`, `input.code`)
+ * and the CLI's `Write` (`input.content`, offered only by opt-in), by the
+ * one rule in `adapters/claude/source-capture.ts` (ggui#1714). If several
+ * calls carry source, returns the last one (the final version).
  *
- * Used by: loop.ts, generator.ts
+ * Used by: loop.ts
  */
 export function extractSourceCodeFromMessage(message: SdkMessage): string | undefined {
   if ((message.type as string) !== 'assistant') return undefined;
@@ -155,12 +157,9 @@ export function extractSourceCodeFromMessage(message: SdkMessage): string | unde
   if (!Array.isArray(content)) return undefined;
 
   for (const block of content) {
-    if (block.type === 'tool_use' && block.name === 'Write') {
-      const input = block.input as { content?: string } | undefined;
-      if (input?.content) {
-        sourceCode = input.content;
-      }
-    }
+    if (block.type !== 'tool_use') continue;
+    const value = sourceCodeFromToolUse(block.name, block.input);
+    if (value !== undefined) sourceCode = value;
   }
 
   return sourceCode;
@@ -183,7 +182,7 @@ export function extractCompiledCode(messages: SdkMessage[]): string | undefined
 }
 
 /**
- * Scan all messages for the last sourceCode from Write tool_use.
+ * Scan all messages for the last sourceCode a tool_use carried.
  *
  * Convenience wrapper for tests and one-shot extraction.
  */

package/src/evaluation/types-public.ts

@@ -20,6 +20,7 @@
 
 export type Priority = "P0" | "P1" | "P2";
 
+import type { VisionFinishReason } from '../harness/llm-router.js';
 import type { GenerationRuntimeProbeOutcome, GenerationRuntimeProbeStatus } from "@ggui-ai/mcp-server-core";
 import type { CanvasClass, DesignMode } from "../design-mode.js";
 
@@ -61,6 +62,11 @@ export interface EvalIssue {
   description: string;
   fix: string;
   line?: number;
+  /**
+   * The visual leg's provenance (ggui#1545): `judge` (the model's finding) or `instrument` (a deterministic
+   * measurement). Absent on every other leg's issues. `visualEvaluation.actOn` acts on `judge` only.
+   */
+  origin?: "judge" | "instrument";
 }
 
 /**
@@ -100,6 +106,23 @@ export interface ProbeHostLoad {
   readonly cores: number;
 }
 
+/**
+ * ggui#1398 — whether the controls the probe clicked showed they were
+ * working once their action was pending. Report-only: it is read by the
+ * benchmark and never counted into the probe's warnings or fed back to
+ * the coding loop.
+ */
+export interface ProbePendingAffordance {
+  /** Controls the pending phase marked pending: those whose click dispatched their action (click or submit wiring) and that were still in the page. `visible + missing.length === dispatched`. */
+  readonly dispatched: number;
+  /** Of those, how many visibly changed while pending (disabled, busy, text, or replaced). */
+  readonly visible: number;
+  /** The actions whose control did not visibly change, including one whose card threw while pending. */
+  readonly missing: readonly string[];
+  /** Actions whose control was no longer in the page when its turn came (the card replaced it, or blanked after a throw): never marked, not counted as dispatched. */
+  readonly gone: readonly string[];
+}
+
 /**
  * The engine's own fields on a probe's meta, beside the outcome a
  * generation's metadata reports: why a probe did not finish, its render
@@ -116,6 +139,11 @@ export interface RuntimeProbeMetaDetail {
   readonly hostLoad?: ProbeHostLoad;
   /** Time the check waited for a probe slot before it started, ms; present only when > 0 (ggui#1380 C2b). */
   readonly queuedMs?: number;
+  /**
+   * `ran` only: the pending-affordance walk (ggui#1398). Absent when no
+   * action walk ran, so "no data" and "0 of 0" read differently.
+   */
+  readonly pendingAffordance?: ProbePendingAffordance;
 }
 
 /**
@@ -334,6 +362,58 @@ export interface CanvasJudgeRecord {
   readonly sigma: number;
   /** The judge's per-canvas critique per sample — the words beside a swing. */
   readonly notes: string[];
+  /**
+   * ggui#1127 — the cause of each scoring sample whose answer failed to parse and was SALVAGED: its four dimensions
+   * were recovered from the closed prefix, its issues kept only as far as they closed, its critique lost. One entry
+   * per salvaged sample; absent when every sample parsed. The salvaged sample's score still counts.
+   */
+  readonly salvaged?: readonly JudgeSalvageCause[];
+  /**
+   * ggui#1127 — the criteria call's answer was salvaged. A salvage keeps no criteria, so that frame's judge rows read
+   * "not answered" with this cause rather than a clean `n/a`. Absent when the call parsed or was not asked.
+   */
+  readonly criteriaSalvaged?: JudgeSalvageCause;
+  /**
+   * ggui#1687 — the model asked for, and every model the provider said answered this canvas's calls (scoring and
+   * criteria), deduplicated. `served` is empty when no provider named one: unknown, never "the requested model".
+   * When the request names an alias and the provider echoes the alias (measured on a hosted route, 125 of 125
+   * canvases), `served` confirms the alias was honoured but cannot tell two snapshots behind it apart. A judge that
+   * must is pinned to a dated snapshot id instead. With the alias, a snapshot change shows only as a step on a date in
+   * the judge's pooled within-call spread, read by week. That is a read someone runs, not a monitor.
+   */
+  readonly models?: { readonly requested: string; readonly served: readonly string[] };
+}
+
+/**
+ * ggui#1687 — what the criteria call's answer held, before resolution. A row the judge left unanswered reads `n/a`
+ * either way; this says whether the call came back empty, cut, or with entries the parser had to drop.
+ */
+export interface CriteriaAnswerCounts {
+  /** The answer carried a `criteria` array. `false`: absent or not an array, so no row was read from it. */
+  readonly array: boolean;
+  /** Entries in the array. */
+  readonly received: number;
+  /** Entries kept: an object with a string id and a verdict of pass, fail or n/a. */
+  readonly kept: number;
+  /** Entries dropped, by why. */
+  readonly dropped: { readonly notObject: number; readonly badId: number; readonly badVerdict: number };
+}
+
+/**
+ * ggui#1127 — why a judge answer that failed to parse had to be salvaged. `output-cap`: the provider reported it
+ * stopped at the output cap. `malformed`: the provider reported a normal stop, so the answer ended but did not
+ * parse. `stopped`: the provider stopped it for another reason it named (a safety stop, or one it reports as
+ * other), so it is cut, not malformed. `unknown`: the provider or route reported no stop reason. Never inferred
+ * from the answer's length.
+ */
+export type JudgeSalvageCause = "output-cap" | "malformed" | "stopped" | "unknown";
+
+/** ggui#1127 — salvaged judge answers across a canvas run, per leg, as N of M. */
+export interface JudgeSalvageCounts {
+  /** Scoring answers: `answers` = samples that parsed or were salvaged, `salvaged` = those salvaged. */
+  readonly scoring: { readonly salvaged: number; readonly answers: number };
+  /** Criteria calls: `answered` = calls that parsed or were salvaged, `salvaged` = those salvaged (criteria lost). */
+  readonly criteria: { readonly salvaged: number; readonly answered: number };
 }
 
 // ─── Visual criteria (ggui#1436) ──────────────────────────────────────────
@@ -393,6 +473,19 @@ export interface CriterionVerdict {
   source: CriteriaSource;
   verdict: CriteriaVerdict;
   evidence: string;
+  /**
+   * ggui#1663 — a measured reading beside a judge row, REPORT-ONLY: the row's `verdict` stays the judge's, so this
+   * changes no verdict, score or bar. Present only on rows an instrument answers beside the judge (`space.edge`).
+   */
+  instrument?: { verdict: CriteriaVerdict; evidence: string };
+}
+/** ggui#1687 — the criteria call as it came back: its answer's counts, ids it answered outside the bank, its stop. */
+export interface CriteriaCallRecord {
+  readonly answer: CriteriaAnswerCounts;
+  /** Kept entries whose id names no row of the bank. */
+  readonly unknownIds: number;
+  /** The provider's normalized stop reason (`length` = the output cap); absent when it reported none. */
+  readonly finishReason?: VisionFinishReason;
 }
 export interface CriteriaBlock {
   /** sha256(bank version | selector version | canonical context), 16 hex — the block's own digest. */
@@ -402,6 +495,8 @@ export interface CriteriaBlock {
   context: CriteriaContext;
   selection: CriteriaSelection[];
   verdicts: CriterionVerdict[];
+  /** ggui#1687 — the call that answered this block, when one did. Absent for a block resolved with no call. */
+  call?: CriteriaCallRecord;
 }
 /** The roll-up across canvases; a `must` at `n/a` is listed, never counted as a pass. */
 export interface CriteriaRollup {

package/src/evaluation/types.ts

@@ -1,3 +1,4 @@
+import type { CriteriaAnswerCounts } from './types-public.js';
 // packages/ui-gen/src/evaluation/types.ts
 
 /**
@@ -28,6 +29,13 @@ export interface EvaluationIssue {
   severity: 'critical' | 'major' | 'minor';
   /** Suggested fix for the agent to apply */
   fix: string;
+  /**
+   * Who produced it, on the visual leg (ggui#1545): `judge` for the model's own findings, `instrument` for a
+   * deterministic measurement (fit, overflow, blank). Absent on issues from other evaluators. The loop's
+   * `visualEvaluation.actOn` acts on `judge` findings only, so a new instrument check never joins the treatment
+   * by accident: it has to say it is a judge finding, and absence reads as "not acted on".
+   */
+  origin?: 'judge' | 'instrument';
 }
 
 /**
@@ -57,6 +65,8 @@ export interface EvaluationResult {
   outputTokens?: number;
   /** ggui#1436 — the judge's criteria answers, when the prompt asked for them; never part of the score. */
   criteriaAnswers?: CriteriaAnswer[];
+  /** ggui#1687 — what the answer's criteria array held and what the parser dropped; set whenever the answer parsed. */
+  criteriaCounts?: CriteriaAnswerCounts;
 }
 
 /**