@ggui-ai/ui-gen 0.9.0 → 0.10.0

package/dist/evaluation/index.js

@@ -2,6 +2,7 @@ import { query, createSdkMcpServer, tool } from '@anthropic-ai/claude-agent-sdk'
 import Anthropic from '@anthropic-ai/sdk';
 import { z } from 'zod';
 import { listContractGadgets, HOOK_NAME_RE, STDLIB_GADGETS_PACKAGE } from '@ggui-ai/protocol';
+import 'esbuild';
 
 var __defProp = Object.defineProperty;
 var __getOwnPropNames = Object.getOwnPropertyNames;
@@ -23,14 +24,16 @@ var OpenRouterError;
 var init_types = __esm({
   "src/adapters/openrouter/types.ts"() {
     OpenRouterError = class extends Error {
-      constructor(message, status, code) {
+      constructor(message, status, code, headers) {
         super(message);
         this.status = status;
         this.code = code;
+        this.headers = headers;
         this.name = "OpenRouterError";
       }
       status;
       code;
+      headers;
       get isTransient() {
         return this.status === 408 || this.status === 429 || this.status === 502 || this.status === 503;
       }
@@ -75,7 +78,7 @@ var init_client = __esm({
         if (!response.ok) {
           const errorBody = await response.json().catch(() => ({ error: { message: response.statusText } }));
           const message = errorBody.error?.message ?? response.statusText;
-          throw new OpenRouterError(message, response.status);
+          throw new OpenRouterError(message, response.status, void 0, response.headers);
         }
         return response.json();
       }
@@ -93,7 +96,7 @@ var init_client = __esm({
         if (!response.ok) {
           const errorBody = await response.json().catch(() => ({ error: { message: response.statusText } }));
           const message = errorBody.error?.message ?? response.statusText;
-          throw new OpenRouterError(message, response.status);
+          throw new OpenRouterError(message, response.status, void 0, response.headers);
         }
         if (!response.body) {
           throw new OpenRouterError("No response body for streaming request", 500);
@@ -455,6 +458,34 @@ function errorSummary(e) {
   }
   return `${e.constructor.name}${status}${code}: ${e.message.slice(0, 120)}${body}`;
 }
+function isRateLimited(e) {
+  if (!(e instanceof Error) || !("status" in e)) return false;
+  return e.status === 429;
+}
+function extractStatus(e) {
+  if (!(e instanceof Error) || !("status" in e)) return void 0;
+  const status = e.status;
+  return typeof status === "number" ? status : void 0;
+}
+function extractRetryAfterSec(e) {
+  if (!(e instanceof Error) || !("headers" in e)) return void 0;
+  const headers = e.headers;
+  if (headers === null || typeof headers !== "object" || !("get" in headers) || typeof headers.get !== "function") {
+    return void 0;
+  }
+  const raw = headers.get("retry-after");
+  if (raw === null) return void 0;
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : void 0;
+}
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+var MAX_RETRY_ATTEMPTS = 2;
+var RETRY_AFTER_CAP_SEC = 15;
+var DEFAULT_BACKOFF_BASE_MS = 1e3;
+var DEFAULT_BACKOFF_MAX_MS = 8e3;
+var MAX_TOTAL_RETRY_DELAY_MS = 2e4;
 var LLMAgent = class {
   client = null;
   // ── Session state for server-side chaining ──────────────
@@ -469,8 +500,11 @@ var LLMAgent = class {
    * back to `process.env` exactly as before.
    */
   routeOverride;
-  constructor(routeOverride) {
+  /** See `AgentConfig.onRetry` (#489). Absent is a no-op. */
+  onRetry;
+  constructor(routeOverride, onRetry) {
     this.routeOverride = routeOverride;
+    this.onRetry = onRetry;
   }
   async getClient() {
     if (!this.client) {
@@ -505,17 +539,83 @@ var LLMAgent = class {
   async sendToolResult(_results) {
   }
   /**
-   * Execute an API call. No retry — if it fails, it fails.
-   * Logs the error with provider context and re-throws.
+   * Execute an API call, retrying once or twice on HTTP 429
+   * (rate-limited) before giving up (#489).
+   *
+   * Policy: on a 429, honor the provider's `Retry-After` header when
+   * present (capped at `RETRY_AFTER_CAP_SEC` — a provider asking for a
+   * longer wait is treated as "don't retry", not "wait longer": making
+   * the caller sit through 15+ more seconds on a single failed call is
+   * a bad experience regardless of what's driving the request);
+   * otherwise fall back to exponential backoff with jitter, capped at
+   * `DEFAULT_BACKOFF_MAX_MS` per attempt. Stops after
+   * `MAX_RETRY_ATTEMPTS` retries OR once the cumulative delay would
+   * exceed `MAX_TOTAL_RETRY_DELAY_MS`, whichever comes first. Any
+   * non-429 error, or a 429 with no attempts left, is logged and
+   * re-thrown immediately — unchanged from the pre-#489 behavior.
+   *
+   * **Budget scope (final-review correction):** `MAX_TOTAL_RETRY_DELAY_MS`
+   * bounds a single `apiCall()` invocation, not a whole generation. A
+   * generation issues many sequential `apiCall()` calls (coding turns,
+   * tool-result round-trips, eval rounds); under sustained throttling
+   * each one can independently add up to `MAX_TOTAL_RETRY_DELAY_MS`, so
+   * a generation with k rate-limited calls can add up to roughly `k *
+   * MAX_TOTAL_RETRY_DELAY_MS` to how long it holds its caller's
+   * resources (e.g. the caller's concurrency slot) — not a flat 20s
+   * ceiling. No shared per-generation budget is threaded here by
+   * design (accepted ruling, not an oversight): the caller's own
+   * queue/admission backstop is what bounds how long other
+   * queued work waits, independent of any one generation's retry
+   * total.
    */
   async apiCall(fn) {
     const start = Date.now();
-    try {
-      return await fn();
-    } catch (e) {
-      const ms = Date.now() - start;
-      console.error(`[${this.provider}] API error after ${ms}ms: ${errorSummary(e)}`);
-      throw e;
+    let totalDelayMs = 0;
+    for (let attempt = 0; ; attempt++) {
+      try {
+        return await fn();
+      } catch (e) {
+        const ms = Date.now() - start;
+        if (attempt >= MAX_RETRY_ATTEMPTS || !isRateLimited(e)) {
+          console.error(`[${this.provider}] API error after ${ms}ms: ${errorSummary(e)}`);
+          throw e;
+        }
+        const retryAfterSec = extractRetryAfterSec(e);
+        if (retryAfterSec !== void 0 && retryAfterSec > RETRY_AFTER_CAP_SEC) {
+          console.error(
+            `[${this.provider}] API error after ${ms}ms (retry-after ${retryAfterSec}s exceeds ${RETRY_AFTER_CAP_SEC}s cap, not retrying): ${errorSummary(e)}`
+          );
+          throw e;
+        }
+        const delayMs = retryAfterSec !== void 0 ? retryAfterSec * 1e3 : Math.min(DEFAULT_BACKOFF_BASE_MS * 2 ** attempt, DEFAULT_BACKOFF_MAX_MS) + Math.floor(Math.random() * 250);
+        if (totalDelayMs + delayMs > MAX_TOTAL_RETRY_DELAY_MS) {
+          console.error(
+            `[${this.provider}] API error after ${ms}ms (retry budget exhausted, not retrying): ${errorSummary(e)}`
+          );
+          throw e;
+        }
+        totalDelayMs += delayMs;
+        const info = {
+          provider: this.provider,
+          attempt: attempt + 1,
+          maxAttempts: MAX_RETRY_ATTEMPTS,
+          status: extractStatus(e),
+          retryAfterSec,
+          delayMs,
+          message: errorSummary(e)
+        };
+        console.warn(
+          `[${this.provider}] rate-limited (attempt ${info.attempt}/${info.maxAttempts}), retrying in ${delayMs}ms`
+        );
+        try {
+          this.onRetry?.(info);
+        } catch (observerError) {
+          console.warn(
+            `[${this.provider}] onRetry observer threw (ignored, retry proceeds): ${errorSummary(observerError)}`
+          );
+        }
+        await sleep(delayMs);
+      }
     }
   }
 };
@@ -598,6 +698,63 @@ var AnthropicAgent = class extends LLMAgent {
       throw e;
     }
   }
+  async callVision(model, systemPrompt, userPrompt, image, maxTokens) {
+    const client = await this.getClient();
+    const traceId = newLlmTraceId();
+    const startedAt = Date.now();
+    const resolvedModel = this.resolveModel(model);
+    const imageBlock = {
+      type: "image",
+      source: {
+        type: "base64",
+        media_type: image.mediaType,
+        data: image.base64
+      }
+    };
+    try {
+      const response = await this.apiCall(
+        () => client.messages.create({
+          model: resolvedModel,
+          max_tokens: maxTokens ?? 1500,
+          system: systemPrompt,
+          messages: [
+            {
+              role: "user",
+              content: [imageBlock, { type: "text", text: userPrompt }]
+            }
+          ]
+        })
+      );
+      const text = response.content.filter((block) => block.type === "text").map((block) => block.text).join("");
+      const endedAt = Date.now();
+      emitLlmTraceEvent({
+        id: traceId,
+        at: startedAt,
+        endedAt,
+        durationMs: endedAt - startedAt,
+        provider: "anthropic",
+        model: resolvedModel,
+        kind: "callVision",
+        systemPrompt,
+        userPrompt,
+        result: {
+          inputTokens: response.usage.input_tokens,
+          outputTokens: response.usage.output_tokens,
+          text
+        }
+      });
+      return {
+        text,
+        inputTokens: response.usage.input_tokens,
+        outputTokens: response.usage.output_tokens
+      };
+    } catch (e) {
+      emitLlmTraceEvent({
+        error: { message: e instanceof Error ? e.message : String(e) }
+      });
+      throw e;
+    }
+  }
   async callTools(model, systemPrompt, userPrompt, tools, toolChoice = "required", _scopedTools) {
     const client = await this.getClient();
     const traceId = newLlmTraceId();
@@ -973,6 +1130,62 @@ var GoogleAgent = class extends LLMAgent {
       outputTokens: interaction.usage?.total_output_tokens ?? 0
     };
   }
+  async callVision(model, systemPrompt, userPrompt, image, maxTokens) {
+    const client = await this.getClient();
+    const traceId = newLlmTraceId();
+    const startedAt = Date.now();
+    const resolvedModel = this.resolveModel(model);
+    try {
+      const response = await this.apiCall(
+        () => client.models.generateContent({
+          model: resolvedModel,
+          contents: [
+            {
+              role: "user",
+              parts: [
+                { inlineData: { mimeType: image.mediaType, data: image.base64 } },
+                { text: userPrompt }
+              ]
+            }
+          ],
+          config: {
+            systemInstruction: systemPrompt,
+            ...maxTokens ? { maxOutputTokens: maxTokens } : {}
+          }
+        })
+      );
+      const text = response.candidates?.[0]?.content?.parts?.map((p) => p.text || "").join("") || "";
+      const usage = response.usageMetadata;
+      const result = {
+        text,
+        inputTokens: usage?.promptTokenCount ?? 0,
+        outputTokens: usage?.candidatesTokenCount ?? 0
+      };
+      const endedAt = Date.now();
+      emitLlmTraceEvent({
+        id: traceId,
+        at: startedAt,
+        endedAt,
+        durationMs: endedAt - startedAt,
+        provider: "google",
+        model: resolvedModel,
+        kind: "callVision",
+        systemPrompt,
+        userPrompt,
+        result: {
+          inputTokens: result.inputTokens,
+          outputTokens: result.outputTokens,
+          text
+        }
+      });
+      return result;
+    } catch (e) {
+      emitLlmTraceEvent({
+        error: { message: e instanceof Error ? e.message : String(e) }
+      });
+      throw e;
+    }
+  }
   async callTools(model, systemPrompt, userPrompt, tools, _toolChoice = "required", scopedTools) {
     const client = await this.getClient();
     const interactionTools = tools.map((t) => ({
@@ -1304,15 +1517,24 @@ var OpenRouterAgent = class extends LLMAgent {
 function createAgent(providerOrConfig) {
   const provider = typeof providerOrConfig === "string" ? providerOrConfig : providerOrConfig.provider;
   const routeOverride = typeof providerOrConfig === "string" ? void 0 : providerOrConfig.routeOverride;
+  const onRetry = typeof providerOrConfig === "string" ? void 0 : providerOrConfig.onRetry;
   switch (provider) {
     case "anthropic":
-      return new AnthropicAgent(routeOverride);
+      return new AnthropicAgent(routeOverride, onRetry);
     case "openai":
-      return new OpenAIAgent(routeOverride);
+      return new OpenAIAgent(routeOverride, onRetry);
     case "google":
-      return new GoogleAgent(routeOverride);
+      return new GoogleAgent(routeOverride, onRetry);
     case "openrouter":
-      return new OpenRouterAgent(routeOverride);
+      return new OpenRouterAgent(routeOverride, onRetry);
+  }
+}
+function createVisionAgent(config) {
+  switch (config.provider) {
+    case "anthropic":
+      return new AnthropicAgent(config.routeOverride, config.onRetry);
+    case "google":
+      return new GoogleAgent(config.routeOverride, config.onRetry);
   }
 }
 
@@ -2203,6 +2425,106 @@ var STATE_MERGE_CHECKS = [
   }
 ];
 
+// src/evaluation/axis-checks/checks/state-ui-affordance.ts
+var STATE_ARIA_OR_ROLE = /aria-(checked|pressed|selected|expanded|current)\s*=|role\s*=/;
+var STATE_PRIMITIVE_IN_SUBTREE = /<(Checkbox|Toggle|Tabs|Accordion)[\s/>]/;
+var SEMANTIC_PRIMITIVES = /* @__PURE__ */ new Set(["Button", "Checkbox", "Toggle", "Link", "MenuItem"]);
+function collectSetters(src) {
+  const setters = /* @__PURE__ */ new Set();
+  for (const m of src.matchAll(/const\s*\[\s*\w+\s*,\s*(set\w+)\s*\]\s*=\s*useState/g)) {
+    setters.add(m[1]);
+  }
+  return setters;
+}
+function collectHandlers(src, setters) {
+  const handlers = /* @__PURE__ */ new Set();
+  for (const m of src.matchAll(/const\s+(\w+)\s*=\s*(?:\([^)]*\)|\w+)\s*=>\s*\{?/g)) {
+    const body = src.slice(m.index + m[0].length, m.index + m[0].length + 400);
+    for (const s of setters) {
+      if (body.includes(s)) {
+        handlers.add(m[1]);
+        break;
+      }
+    }
+  }
+  return handlers;
+}
+function attrBlock(src, start) {
+  let depth = 0;
+  for (let i = start; i < src.length; i++) {
+    const c = src[i];
+    if (c === "{") depth++;
+    else if (c === "}") depth--;
+    else if (c === ">" && depth === 0) return { block: src.slice(start, i), end: i };
+  }
+  return { block: src.slice(start, start + 900), end: start + 900 };
+}
+function onClickValue(block) {
+  const m = block.match(/onClick\s*=\s*\{/);
+  if (!m || m.index === void 0) return null;
+  const open = m.index + m[0].length - 1;
+  let depth = 0;
+  for (let i = open; i < block.length; i++) {
+    if (block[i] === "{") depth++;
+    else if (block[i] === "}") {
+      depth--;
+      if (depth === 0) return block.slice(open + 1, i);
+    }
+  }
+  return block.slice(open + 1);
+}
+function subtree(src, pos, tag) {
+  const close = src.indexOf(`</${tag}>`, pos);
+  return src.slice(pos, close === -1 ? pos + 700 : close);
+}
+function runStateAriaPresent(input) {
+  if (input.compiledCode === null) return [];
+  const src = input.sourceCode;
+  const setters = collectSetters(src);
+  const handlers = collectHandlers(src, setters);
+  const issues = [];
+  const flaggedLines = /* @__PURE__ */ new Set();
+  const localComponents = /* @__PURE__ */ new Set();
+  for (const m of src.matchAll(/(?:function\s+([A-Z]\w*)\s*\(|const\s+([A-Z]\w*)\s*=)/g)) {
+    localComponents.add(m[1] ?? m[2]);
+  }
+  for (const m of src.matchAll(/<([A-Za-z][A-Za-z0-9]*)\b/g)) {
+    const tag = m[1];
+    if (m.index === void 0) continue;
+    if (localComponents.has(tag)) continue;
+    const { block, end } = attrBlock(src, m.index + m[0].length);
+    const clickValue = onClickValue(block);
+    if (clickValue === null) continue;
+    const mutatesAffordance = [...setters].some((s) => clickValue.includes(s)) || [...handlers].some((h) => new RegExp(`\\b${h}\\b`).test(clickValue)) || // Extracted-subcomponent shape: the toggle handler arrives as an
+    // `on[A-Z]…` callback prop and is invoked from onClick — inside a
+    // ui-affordance component that IS the affordance mutation.
+    /\bon[A-Z]\w*\b/.test(clickValue);
+    if (!mutatesAffordance) continue;
+    if (SEMANTIC_PRIMITIVES.has(tag)) continue;
+    if (STATE_ARIA_OR_ROLE.test(block)) continue;
+    if (STATE_PRIMITIVE_IN_SUBTREE.test(subtree(src, end, tag))) continue;
+    const line = src.slice(0, m.index).split("\n").length;
+    if (flaggedLines.has(line)) continue;
+    flaggedLines.add(line);
+    issues.push(
+      mkIssue(
+        "state.ui_affordance.state_aria_present",
+        `Line ${line}: <${tag}> toggles the UI affordance from its onClick but exposes no state ARIA \u2014 its checked/selected/expanded state is invisible to assistive technology (styling is the only carrier).`,
+        `Add the matching state attribute to the SAME element that carries the onClick: \`role="checkbox"\` + \`aria-checked={\u2026}\` for a toggle row, \`aria-pressed\` for a toggle button, \`aria-selected\` for a selection, \`aria-expanded\` for an open/close affordance \u2014 or replace the hand-rolled control with the state-carrying primitive (Checkbox / Toggle / Tabs / Accordion). \`as={Clickable}\` adds button + keyboard semantics but NOT state.`
+      )
+    );
+  }
+  return issues;
+}
+var STATE_UI_AFFORDANCE_CHECKS = [
+  {
+    id: "state.ui_affordance.state_aria_present",
+    axis: "state",
+    values: ["ui-affordance"],
+    run: runStateAriaPresent
+  }
+];
+
 // src/evaluation/axis-checks/checks/realtime.ts
 var REALTIME_ACTIVE = ["merge", "append", "status", "presence", "mixed"];
 function runStreamHandlerPerEvent(input) {
@@ -2892,6 +3214,7 @@ var EXTRA_CHECKS = [
 var REGISTRY = [
   ...UNIVERSAL_CHECKS,
   ...STATE_MERGE_CHECKS,
+  ...STATE_UI_AFFORDANCE_CHECKS,
   ...REALTIME_CHECKS,
   ...WRITES_CHECKS,
   ...STATE_PAYLOAD_CHECKS,
@@ -2919,7 +3242,25 @@ function runAxisChecks(classification, input) {
   }
   return issues;
 }
+async function callMultimodalLLM(config, model, prompt, screenshot, originalPrompt) {
+  const userPrompt = `## Original Request
+${originalPrompt}
+
+Evaluate the screenshot of the generated component.`;
+  const agent = createVisionAgent({
+    provider: config.provider === "claude" ? "anthropic" : config.provider,
+    routeOverride: config.routeOverride,
+    onRetry: config.onRetry
+  });
+  return agent.callVision(
+    model,
+    prompt,
+    userPrompt,
+    { mediaType: "image/png", base64: screenshot.toString("base64") },
+    1500
+  );
+}
 
-export { CRITERIA, DEFAULT_EVAL_MAX_ROUNDS, DEFAULT_QUALITY_CONFIG, MAX_EVAL_ROUNDS_HARD_LIMIT, buildCodingCriteriaSummary, buildFixPrompt, computeEvaluationScore, createEvaluationToolsServer, extractCompiledCode, extractCompiledCodeFromMessage, extractEvalResult, extractSourceCode, extractSourceCodeFromMessage, extractToolResultTexts, getActionableIssues, getCriteriaByPriority, getCriterionById, getEvaluatorSystemPrompt, getLLMCriteria, isBlocked, matches, priorityForIssue, runAxisChecks, runEvaluation, runEvaluationLoop };
+export { CRITERIA, DEFAULT_EVAL_MAX_ROUNDS, DEFAULT_QUALITY_CONFIG, MAX_EVAL_ROUNDS_HARD_LIMIT, buildCodingCriteriaSummary, buildFixPrompt, callMultimodalLLM, computeEvaluationScore, createEvaluationToolsServer, extractCompiledCode, extractCompiledCodeFromMessage, extractEvalResult, extractSourceCode, extractSourceCodeFromMessage, extractToolResultTexts, getActionableIssues, getCriteriaByPriority, getCriterionById, getEvaluatorSystemPrompt, getLLMCriteria, isBlocked, matches, priorityForIssue, runAxisChecks, runEvaluation, runEvaluationLoop };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map