@iamu-dev/iamu 0.2.0-beta.1 → 0.2.0-beta.10

package/dist/eval/gemini-eval-cli.js

@@ -0,0 +1,976 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+// src/eval/walkthrough-evaluator.ts
+var walkthrough_evaluator_exports = {};
+__export(walkthrough_evaluator_exports, {
+  evaluateWalkthroughRun: () => evaluateWalkthroughRun,
+  formatEvaluationScorecardTable: () => formatEvaluationScorecardTable
+});
+function evaluateWalkthroughRun(canonical, contract) {
+  const findings = canonical.findings ?? [];
+  const steps = canonical.steps ?? [];
+  const unseededKeywords = contract.unseededDataKeywords ?? DEFAULT_UNSEEDED_KEYWORDS;
+  const expectedDefects = contract.expectedDefects ?? [];
+  let matchedDefects = 0;
+  for (const expected of expectedDefects) {
+    const isMatched = findings.some((f) => {
+      const text = `${f.title} ${f.ui_fact ?? ""} ${f.consequence ?? ""}`.toLowerCase();
+      return expected.keywords.some((kw) => text.includes(kw.toLowerCase()));
+    });
+    if (isMatched) {
+      matchedDefects++;
+    }
+  }
+  const defectRecallRate = expectedDefects.length > 0 ? matchedDefects / expectedDefects.length : 1;
+  let environmentalFalsePositives = 0;
+  for (const f of findings) {
+    const text = `${f.title} ${f.ui_fact ?? ""} ${f.consequence ?? ""}`.toLowerCase();
+    const isEnvBug = unseededKeywords.some(
+      (kw) => text.includes(kw.toLowerCase())
+    );
+    if (isEnvBug && contract.expectedState === "empty") {
+      environmentalFalsePositives++;
+    }
+  }
+  const fpreRate = findings.length > 0 ? environmentalFalsePositives / findings.length : 0;
+  const legitimateDefects = findings.length - environmentalFalsePositives;
+  const precisionRate = findings.length > 0 ? legitimateDefects / findings.length : 1;
+  let adhered = true;
+  let parReason = "Preconditions respected.";
+  if (contract.expectedState === "empty") {
+    if (environmentalFalsePositives > 0) {
+      adhered = false;
+      parReason = `Walker logged ${environmentalFalsePositives} defect(s) on unseeded empty state instead of honoring precondition.`;
+    }
+  } else if (contract.expectedState === "mismatch") {
+    const target = contract.expectedApproximationTarget?.toLowerCase();
+    if (target) {
+      const selectedTarget = steps.some((s) => {
+        const reasoning = s.persona_reasoning ?? s.note ?? "";
+        const text = `${s.action ?? ""} ${reasoning}`.toLowerCase();
+        return text.includes(target);
+      });
+      if (!selectedTarget) {
+        adhered = false;
+        parReason = `Walker failed to select approximation target "${contract.expectedApproximationTarget}".`;
+      }
+    }
+  }
+  const allowedSurfaces = contract.allowedSurfaces;
+  let driftCount = 0;
+  if (allowedSurfaces && allowedSurfaces.length > 0) {
+    for (const s of steps) {
+      if (s.surface && s.surface.toLowerCase() !== "page" && !allowedSurfaces.some(
+        (allowed) => s.surface?.toLowerCase().includes(allowed.toLowerCase())
+      )) {
+        driftCount++;
+      }
+    }
+  }
+  const totalSteps = steps.length;
+  const optimalSteps = contract.optimalSteps;
+  const stepWasteIndex = optimalSteps > 0 ? Math.max(0, (totalSteps - optimalSteps) / optimalSteps) : 0;
+  let authDetected = false;
+  let authTransitionSteps = 0;
+  let authSuccess = true;
+  if (contract.authRequired) {
+    const loginStepIndex = steps.findIndex((s) => {
+      const reasoning = s.persona_reasoning ?? s.note ?? "";
+      const text = `${s.action ?? ""} ${reasoning} ${s.url ?? ""}`.toLowerCase();
+      return text.includes("log") || text.includes("auth") || text.includes("inlogg");
+    });
+    if (loginStepIndex >= 0) {
+      authDetected = true;
+      authTransitionSteps = loginStepIndex + 1;
+      authSuccess = steps.length > authTransitionSteps;
+    } else {
+      authSuccess = false;
+    }
+  }
+  const passed = defectRecallRate >= 0.8 && precisionRate >= 0.8 && environmentalFalsePositives === 0 && adhered && (!contract.authRequired || authSuccess) && driftCount === 0;
+  const summary = passed ? `PASSED: Walkthrough satisfied all ground-truth contracts (Recall: ${(defectRecallRate * 100).toFixed(0)}%, Prec: ${(precisionRate * 100).toFixed(0)}%, FPR-E: 0%)` : `FAILED: Walkthrough violated contracts (FPR-E: ${(fpreRate * 100).toFixed(0)}%, PAR: ${adhered ? "Pass" : parReason}, Drift: ${driftCount})`;
+  return {
+    scenarioSlug: contract.scenarioSlug,
+    runId: canonical.run_id ?? "unknown",
+    expectedState: contract.expectedState,
+    defectRecall: {
+      matched: matchedDefects,
+      expected: expectedDefects.length,
+      rate: defectRecallRate
+    },
+    defectPrecision: {
+      legitimate: legitimateDefects,
+      total: findings.length,
+      rate: precisionRate
+    },
+    falsePositiveRateEnvironment: {
+      count: environmentalFalsePositives,
+      totalFindings: findings.length,
+      rate: fpreRate
+    },
+    preconditionAdherence: {
+      adhered,
+      reason: parReason
+    },
+    trajectory: {
+      totalSteps,
+      optimalSteps,
+      driftCount,
+      stepWasteIndex
+    },
+    authTransition: {
+      detected: authDetected,
+      transitionSteps: authTransitionSteps,
+      success: authSuccess
+    },
+    passed,
+    summary
+  };
+}
+function formatEvaluationScorecardTable(scorecards) {
+  const lines = [];
+  lines.push(
+    "\u250C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510"
+  );
+  lines.push(
+    "\u2502 Benchmark Scenario                \u2502 State   \u2502 Recall   \u2502 Prec.  \u2502 FPR-E  \u2502 PAR   \u2502 Verdict\u2502"
+  );
+  lines.push(
+    "\u251C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524"
+  );
+  for (const s of scorecards) {
+    const name = s.scenarioSlug.padEnd(33).slice(0, 33);
+    const state = s.expectedState.padEnd(7).slice(0, 7);
+    const recallStr = s.defectRecall.expected > 0 ? `${s.defectRecall.matched}/${s.defectRecall.expected} (${(s.defectRecall.rate * 100).toFixed(0)}%)`.padEnd(
+      8
+    ) : "N/A     ";
+    const precStr = s.defectPrecision.total > 0 ? `${(s.defectPrecision.rate * 100).toFixed(0)}%`.padEnd(6) : "N/A   ";
+    const fpreStr = `${(s.falsePositiveRateEnvironment.rate * 100).toFixed(0)}%`.padEnd(6);
+    const parStr = (s.preconditionAdherence.adhered ? "100%" : "0%").padEnd(5);
+    const verdict = s.passed ? "PASS   " : "FAIL   ";
+    lines.push(
+      `\u2502 ${name} \u2502 ${state} \u2502 ${recallStr} \u2502 ${precStr} \u2502 ${fpreStr} \u2502 ${parStr} \u2502 ${verdict}\u2502`
+    );
+  }
+  lines.push(
+    "\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518"
+  );
+  return lines.join("\n");
+}
+var DEFAULT_UNSEEDED_KEYWORDS;
+var init_walkthrough_evaluator = __esm({
+  "src/eval/walkthrough-evaluator.ts"() {
+    "use strict";
+    DEFAULT_UNSEEDED_KEYWORDS = [
+      "empty",
+      "tom lista",
+      "inga poster",
+      "inga reser\xE4kningar",
+      "saknas data",
+      "no records",
+      "no expenses",
+      "no reports",
+      "missing data",
+      "unseeded"
+    ];
+  }
+});
+
+// src/eval/gemini-eval-cli.ts
+init_walkthrough_evaluator();
+import { existsSync, promises as fs2 } from "fs";
+import http from "http";
+import os from "os";
+import path2 from "path";
+import { fileURLToPath as fileURLToPath2 } from "url";
+import { GoogleAuth as GoogleAuth2 } from "google-auth-library";
+
+// src/eval/gemini-walker.ts
+import { execFile } from "child_process";
+import { promises as fs } from "fs";
+import path from "path";
+import { promisify } from "util";
+import { fileURLToPath } from "url";
+import { GoogleAuth } from "google-auth-library";
+var execFileAsync = promisify(execFile);
+var __dirname = path.dirname(fileURLToPath(import.meta.url));
+var GEMINI_TOOLS = [
+  {
+    functionDeclarations: [
+      {
+        name: "browse_action",
+        description: "Interact with the web browser surface: click, fill, select, or snapshot the page. Read ARIA roles and names from previous snapshots. NOTE: Call record_step to record meaningful persona steps and observations.",
+        parameters: {
+          type: "OBJECT",
+          properties: {
+            action: {
+              type: "STRING",
+              enum: [
+                "click",
+                "fill",
+                "select",
+                "press",
+                "scroll",
+                "back",
+                "snapshot"
+              ],
+              description: "The browser action verb to perform"
+            },
+            role: {
+              type: "STRING",
+              description: "ARIA role of target element (e.g. 'button', 'link', 'combobox', 'textbox')"
+            },
+            name: {
+              type: "STRING",
+              description: "Exact or closest accessible name from the snapshot (e.g. 'Ny reser\xE4kning', 'Till', 'Fr\xE5n')"
+            },
+            value: {
+              type: "STRING",
+              description: "Text to type into an input, or option text to choose in a select/combobox"
+            },
+            nth: {
+              type: "INTEGER",
+              description: "0-based index if multiple elements match the same role or name"
+            }
+          },
+          required: ["action"]
+        }
+      },
+      {
+        name: "record_step",
+        description: "Log an official step for this persona walkthrough report documenting actions and observations.",
+        parameters: {
+          type: "OBJECT",
+          properties: {
+            action: {
+              type: "STRING",
+              description: "What action the persona took (e.g. 'Selected Gothenburg destination')"
+            },
+            note: {
+              type: "STRING",
+              description: "Persona's internal thoughts, reasons, and observations (e.g. observed missing options, evaluated rates)"
+            },
+            surface: {
+              type: "STRING",
+              description: "UI section or surface name (e.g. 'Expense Form', 'Trip List', 'Login Portal')"
+            }
+          },
+          required: ["action", "note"]
+        }
+      },
+      {
+        name: "record_finding",
+        description: "Record a genuine product defect, regulatory compliance violation, or usability barrier discovered during the walk.",
+        parameters: {
+          type: "OBJECT",
+          properties: {
+            title: {
+              type: "STRING",
+              description: "Clear defect title summarizing what is wrong"
+            },
+            type: {
+              type: "STRING",
+              enum: ["missing", "incorrect", "hinders", "helps", "irrelevant"],
+              description: "Defect taxonomy type"
+            },
+            severity: {
+              type: "STRING",
+              enum: ["critical", "major", "minor", "observation"],
+              description: "Severity rating"
+            },
+            consequence: {
+              type: "STRING",
+              description: "Impact on persona tasks or regulatory risk (e.g. Skatteverket tax compliance penalty)"
+            },
+            ui_fact: {
+              type: "STRING",
+              description: "Direct factual description of what the UI displays or is missing"
+            }
+          },
+          required: ["title", "type", "severity", "consequence", "ui_fact"]
+        }
+      },
+      {
+        name: "complete_walkthrough",
+        description: "Conclude the walkthrough when outline objectives are complete, or when unseeded preconditions halt testing.",
+        parameters: {
+          type: "OBJECT",
+          properties: {
+            summary: {
+              type: "STRING",
+              description: "Final wrap-up reasoning for concluding the walkthrough"
+            }
+          }
+        }
+      }
+    ]
+  }
+];
+async function runGeminiWalkthrough(opts) {
+  const apiKey = opts.apiKey ?? process.env.GEMINI_API_KEY;
+  const project = opts.googleCloudProject ?? process.env.GOOGLE_CLOUD_PROJECT ?? "plasma-inquiry-492121-f7";
+  const location = opts.googleCloudLocation ?? process.env.GOOGLE_CLOUD_LOCATION ?? "global";
+  const model = opts.model ?? "gemini-2.5-flash";
+  const maxTurns = opts.maxTurns ?? 12;
+  let requestUrl;
+  let getHeaders;
+  if (apiKey) {
+    requestUrl = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${apiKey}`;
+    getHeaders = async () => ({ "Content-Type": "application/json" });
+  } else {
+    const auth = new GoogleAuth({
+      scopes: "https://www.googleapis.com/auth/cloud-platform"
+    });
+    const client = await auth.getClient();
+    const host = location === "global" ? "aiplatform.googleapis.com" : `${location}-aiplatform.googleapis.com`;
+    requestUrl = `https://${host}/v1/projects/${project}/locations/${location}/publishers/google/models/${model}:generateContent`;
+    getHeaders = async () => {
+      const token = (await client.getAccessToken()).token;
+      if (!token) {
+        throw new Error(
+          "Vertex AI authentication failed: GCP client returned empty access token. Run 'gcloud auth application-default login' to authenticate."
+        );
+      }
+      return {
+        Authorization: `Bearer ${token}`,
+        "Content-Type": "application/json"
+      };
+    };
+  }
+  const runCli = async (args) => {
+    return execFileAsync(process.execPath, [opts.binPath, ...args], {
+      cwd: opts.projectDir,
+      env: {
+        ...process.env,
+        NODE_ENV: "test",
+        IAMU_HOME: path.join(opts.projectDir, ".iamu")
+      }
+    });
+  };
+  const { stdout: startOut } = await runCli([
+    "start-run",
+    opts.scenarioSlug,
+    "--target",
+    opts.targetUrl,
+    "--no-check",
+    "--json"
+  ]);
+  const { run_id: runId } = JSON.parse(startOut);
+  await runCli([
+    "browse",
+    "start",
+    "--headless",
+    "--port",
+    String(opts.cdpPort),
+    "--run",
+    runId,
+    "--json"
+  ]);
+  await runCli(["browse", "goto", opts.targetUrl, "--json"]);
+  const { stdout: initialSnapOut } = await runCli([
+    "browse",
+    "snapshot",
+    "--json"
+  ]);
+  const initialSnapData = JSON.parse(initialSnapOut);
+  let currentSnapshot = initialSnapData.snapshot ?? "";
+  let currentUrl = initialSnapData.url ?? opts.targetUrl;
+  const { stdout: promptOut } = await runCli([
+    "instructions",
+    "walk",
+    "--scenario",
+    opts.scenarioSlug,
+    "--run",
+    runId,
+    "--json"
+  ]);
+  const parsedPrompt = JSON.parse(promptOut);
+  const systemInstruction = parsedPrompt.walker_prompt ?? parsedPrompt.prompt ?? "";
+  const contents = [
+    {
+      role: "user",
+      parts: [
+        {
+          text: `You are starting your persona walkthrough now.
+
+Initial URL: ${currentUrl}
+
+Initial Page ARIA Snapshot:
+\`\`\`yaml
+${currentSnapshot}
+\`\`\`
+
+Take your first action using the available tools. Inspect roles, click, fill, record steps, record all distinct defects found across all form sections, and call complete_walkthrough when the journey is finished.`
+        }
+      ]
+    }
+  ];
+  let isCompleted = false;
+  let turns = 0;
+  while (!isCompleted && turns < maxTurns) {
+    turns++;
+    let response = null;
+    for (let attempt = 1; attempt <= 4; attempt++) {
+      const headers = await getHeaders();
+      response = await fetch(requestUrl, {
+        method: "POST",
+        headers,
+        body: JSON.stringify({
+          systemInstruction: {
+            parts: [{ text: systemInstruction }]
+          },
+          contents,
+          tools: GEMINI_TOOLS,
+          generationConfig: {
+            temperature: 0.1
+          }
+        })
+      });
+      if (response.status === 429 && attempt < 4) {
+        const delay = attempt * 3e3;
+        console.log(
+          `    [Rate Limit 429] Retrying turn ${turns} in ${delay}ms...`
+        );
+        await new Promise((r) => setTimeout(r, delay));
+        continue;
+      }
+      break;
+    }
+    if (!response || !response.ok) {
+      const errBody = response ? await response.text() : "No response";
+      throw new Error(
+        `Gemini API error (${response?.status ?? 500}): ${errBody}`
+      );
+    }
+    const data = await response.json();
+    const candidate = data.candidates?.[0]?.content;
+    if (!candidate || !candidate.parts) {
+      break;
+    }
+    contents.push({
+      role: "model",
+      parts: candidate.parts.map((p) => {
+        const thoughtSig = "thoughtSignature" in p && typeof p.thoughtSignature === "string" ? p.thoughtSignature : void 0;
+        if ("functionCall" in p && p.functionCall) {
+          return {
+            functionCall: p.functionCall,
+            ...thoughtSig ? { thoughtSignature: thoughtSig } : {}
+          };
+        }
+        return {
+          text: "text" in p && p.text ? p.text : "",
+          ...thoughtSig ? { thoughtSignature: thoughtSig } : {}
+        };
+      })
+    });
+    const toolCalls = candidate.parts.filter(
+      (p) => "functionCall" in p && Boolean(p.functionCall)
+    );
+    if (toolCalls.length === 0) {
+      contents.push({
+        role: "user",
+        parts: [
+          {
+            text: "Please use the tools (browse_action, record_step, record_finding, complete_walkthrough) to continue driving the walkthrough."
+          }
+        ]
+      });
+      continue;
+    }
+    console.log(
+      `    [Turn ${turns}] ${toolCalls.map(
+        (t) => `${t.functionCall.name}(${JSON.stringify(t.functionCall.args)})`
+      ).join(", ")}`
+    );
+    const toolResponses = [];
+    for (const { functionCall } of toolCalls) {
+      const { name, args } = functionCall;
+      if (name === "browse_action") {
+        const action = String(args.action ?? "snapshot");
+        let role = args.role ? String(args.role) : void 0;
+        let targetName = args.name ? String(args.name) : void 0;
+        const value = args.value ? String(args.value) : void 0;
+        if (action === "click") {
+          if (!role && targetName) {
+            role = "button";
+          } else if (!targetName && role) {
+            targetName = role;
+            role = "button";
+          }
+        } else if (action === "fill") {
+          if (!role && targetName) {
+            role = "textbox";
+          }
+        } else if (action === "select") {
+          if (!role && targetName) {
+            role = "combobox";
+          }
+        }
+        const cliArgs = ["browse", action];
+        if (action === "goto") {
+          cliArgs.push(value ?? targetName ?? opts.targetUrl);
+        } else {
+          if (role && targetName) {
+            cliArgs.push(role, targetName);
+          }
+          if (value) {
+            cliArgs.push(value);
+          }
+        }
+        if (args.nth !== void 0) {
+          cliArgs.push("--nth", String(args.nth));
+        }
+        cliArgs.push("--json");
+        try {
+          const { stdout: actionOut } = await runCli(cliArgs);
+          const parsed = JSON.parse(actionOut);
+          currentUrl = parsed.url ?? currentUrl;
+          const { stdout: snapOut } = await runCli([
+            "browse",
+            "snapshot",
+            "--json"
+          ]);
+          const snapObj = JSON.parse(snapOut);
+          currentSnapshot = snapObj.snapshot ?? "";
+          toolResponses.push({
+            functionResponse: {
+              name,
+              response: {
+                status: "success",
+                url: currentUrl,
+                snapshot: currentSnapshot,
+                instruction: "Action executed. If this was a meaningful step in your scenario walkthrough, call record_step to record your observation."
+              }
+            }
+          });
+        } catch (err) {
+          const errMsg = err instanceof Error ? err.message : String(err);
+          console.log(`      -> Action error: ${errMsg.slice(0, 120)}`);
+          toolResponses.push({
+            functionResponse: {
+              name,
+              response: {
+                status: "error",
+                message: errMsg,
+                snapshot: currentSnapshot
+              }
+            }
+          });
+        }
+      } else if (name === "record_step") {
+        const stepAction = String(args.action ?? "");
+        const stepNote = String(args.note ?? "");
+        const surface = String(args.surface ?? "Page");
+        const cliArgs = [
+          "step",
+          runId,
+          "--action",
+          stepAction,
+          "--note",
+          stepNote,
+          "--url",
+          currentUrl,
+          "--surface",
+          surface,
+          "--json"
+        ];
+        try {
+          const { stdout: stepOut } = await runCli(cliArgs);
+          toolResponses.push({
+            functionResponse: {
+              name,
+              response: {
+                status: "success",
+                result: JSON.parse(stepOut)
+              }
+            }
+          });
+        } catch (err) {
+          toolResponses.push({
+            functionResponse: {
+              name,
+              response: {
+                status: "error",
+                message: err instanceof Error ? err.message : String(err)
+              }
+            }
+          });
+        }
+      } else if (name === "record_finding") {
+        const title = String(args.title ?? "");
+        const type = String(args.type ?? "missing");
+        const severity = String(args.severity ?? "major");
+        const consequence = String(args.consequence ?? "");
+        const uiFact = String(args.ui_fact ?? "");
+        const cliArgs = [
+          "new-finding",
+          "--run",
+          runId,
+          "--scenario",
+          opts.scenarioSlug,
+          "--title",
+          title,
+          "--type",
+          type,
+          "--severity",
+          severity,
+          "--consequence",
+          consequence,
+          "--ui-fact",
+          uiFact
+        ];
+        if (args.task_impact) {
+          cliArgs.push("--task-impact", String(args.task_impact));
+        }
+        if (args.confidence) {
+          cliArgs.push("--confidence", String(args.confidence));
+        }
+        cliArgs.push("--json");
+        try {
+          const { stdout: findingOut } = await runCli(cliArgs);
+          console.log(
+            `      -> record_finding success: ${findingOut.trim().slice(0, 140)}`
+          );
+          toolResponses.push({
+            functionResponse: {
+              name,
+              response: {
+                status: "success",
+                finding: JSON.parse(findingOut),
+                instruction: "Finding successfully logged. Do not terminate the walkthrough early \u2014 continue through the remaining form sections (including per diem meal deductions and expenses) to record all distinct compliance defects and complete the journey."
+              }
+            }
+          });
+        } catch (err) {
+          const errMsg = err instanceof Error ? err.message : String(err);
+          console.log(`      -> record_finding error: ${errMsg}`);
+          toolResponses.push({
+            functionResponse: {
+              name,
+              response: {
+                status: "error",
+                message: errMsg
+              }
+            }
+          });
+        }
+      } else if (name === "complete_walkthrough") {
+        isCompleted = true;
+        try {
+          await runCli(["complete", runId, "--yes", "--no-prompt", "--json"]);
+          toolResponses.push({
+            functionResponse: {
+              name,
+              response: {
+                status: "success",
+                completed: true,
+                run_id: runId
+              }
+            }
+          });
+        } catch (err) {
+          toolResponses.push({
+            functionResponse: {
+              name,
+              response: {
+                status: "error",
+                message: err instanceof Error ? err.message : String(err)
+              }
+            }
+          });
+        }
+      }
+    }
+    contents.push({
+      role: "user",
+      parts: toolResponses
+    });
+  }
+  try {
+    await runCli(["browse", "stop", "--json"]);
+  } catch {
+  }
+  if (!isCompleted) {
+    try {
+      await runCli(["complete", runId, "--yes", "--no-prompt", "--json"]);
+    } catch {
+    }
+  }
+  const canonicalFile = path.join(
+    opts.projectDir,
+    "iamu",
+    ".runs",
+    runId,
+    "canonical.json"
+  );
+  let canonicalData;
+  try {
+    canonicalData = JSON.parse(await fs.readFile(canonicalFile, "utf-8"));
+  } catch {
+    const fallbackFile = path.join(
+      opts.projectDir,
+      "iamu",
+      ".runs",
+      runId,
+      "walkthrough.json"
+    );
+    canonicalData = JSON.parse(await fs.readFile(fallbackFile, "utf-8"));
+  }
+  return {
+    runId,
+    canonical: canonicalData,
+    turns
+  };
+}
+
+// src/eval/gemini-eval-cli.ts
+var __dirname2 = path2.dirname(fileURLToPath2(import.meta.url));
+var BENCHMARK_DIR = path2.resolve(
+  __dirname2,
+  "..",
+  "..",
+  "..",
+  "..",
+  "tests",
+  "benchmarks",
+  "reserakning"
+);
+var BENCHMARK_SCENARIOS = [
+  {
+    slug: "submit-gothenburg-client-trip",
+    queryParam: "",
+    contract: {
+      scenarioSlug: "submit-gothenburg-client-trip",
+      expectedState: "populated",
+      optimalSteps: 1,
+      expectedDefects: [
+        {
+          id: "def-departure-time",
+          category: "compliance",
+          keywords: ["departure", "arrival", "avresa", "tid"],
+          description: "Missing departure/arrival times for per diem"
+        },
+        {
+          id: "def-meal-deduction",
+          category: "compliance",
+          keywords: [
+            "meal deduction",
+            "traktamente",
+            "breakfast",
+            "kostavdrag"
+          ],
+          description: "Missing meal deductions and outdated rate"
+        }
+      ]
+    }
+  },
+  {
+    slug: "audit-empty-queue",
+    queryParam: "?state=empty",
+    contract: {
+      scenarioSlug: "audit-empty-queue",
+      expectedState: "empty",
+      optimalSteps: 1
+    }
+  },
+  {
+    slug: "submit-malmo-client-trip",
+    queryParam: "?state=mismatch",
+    contract: {
+      scenarioSlug: "submit-malmo-client-trip",
+      expectedState: "mismatch",
+      optimalSteps: 1,
+      expectedApproximationTarget: "G\xF6teborg",
+      allowedSurfaces: [
+        "Expense Form",
+        "Mina reser\xE4kningar",
+        "Ny reser\xE4kning",
+        "Form",
+        "Dashboard",
+        "New Expense Report",
+        "Reser\xE4kning",
+        "Reseinformation",
+        "Page",
+        "Formul\xE4r",
+        "Utl\xE4gg",
+        "Traktamente"
+      ]
+    }
+  },
+  {
+    slug: "submit-gated-client-trip",
+    queryParam: "?auth=required",
+    contract: {
+      scenarioSlug: "submit-gated-client-trip",
+      expectedState: "auth_gated",
+      optimalSteps: 2,
+      authRequired: true
+    }
+  },
+  {
+    slug: "submit-trip-methodical-esg",
+    queryParam: "",
+    contract: {
+      scenarioSlug: "submit-trip-methodical-esg",
+      expectedState: "populated",
+      optimalSteps: 2,
+      expectedDefects: [
+        {
+          id: "def-departure-time",
+          category: "compliance",
+          keywords: ["departure", "arrival", "avresa", "tid"],
+          description: "Missing departure/arrival times for per diem"
+        },
+        {
+          id: "def-meal-deduction",
+          category: "compliance",
+          keywords: [
+            "meal deduction",
+            "traktamente",
+            "breakfast",
+            "kostavdrag"
+          ],
+          description: "Missing meal deductions and outdated rate"
+        }
+      ]
+    }
+  }
+];
+async function main() {
+  const apiKey = process.env.GEMINI_API_KEY;
+  let authDescription = "Google AI Studio (API Key)";
+  if (!apiKey) {
+    try {
+      const auth = new GoogleAuth2({
+        scopes: "https://www.googleapis.com/auth/cloud-platform"
+      });
+      const client = await auth.getClient();
+      const tokenRes = await client.getAccessToken();
+      if (!tokenRes.token) {
+        throw new Error("Empty access token");
+      }
+      const project = process.env.GOOGLE_CLOUD_PROJECT || "plasma-inquiry-492121-f7";
+      const location = process.env.GOOGLE_CLOUD_LOCATION || "global";
+      authDescription = `Google Cloud Vertex AI (project: ${project}, location: ${location})`;
+    } catch {
+      console.error(`
+\x1B[31m[ERROR] No Gemini credentials found.\x1B[0m
+
+To run autonomous LLM walker evaluations with Gemini 2.5 Flash, either:
+  1. Authenticate with Google Cloud Vertex AI (Recommended):
+     gcloud auth application-default login
+  2. Or provide an AI Studio API key:
+     GEMINI_API_KEY=your_key_here pnpm eval:gemini
+`);
+      process.exit(1);
+    }
+  }
+  const binPath = path2.resolve(__dirname2, "..", "bin.js");
+  if (!existsSync(binPath)) {
+    console.error(
+      "\x1B[31m[ERROR] CLI binary not built. Run 'pnpm build' first.\x1B[0m"
+    );
+    process.exit(1);
+  }
+  const htmlContent = await fs2.readFile(
+    path2.join(BENCHMARK_DIR, "index.html"),
+    "utf-8"
+  );
+  const htmlBuffer = Buffer.from(htmlContent, "utf-8");
+  const server = http.createServer((req, res) => {
+    if (req.url === "/favicon.ico") {
+      res.writeHead(204);
+      res.end();
+      return;
+    }
+    res.writeHead(200, {
+      "Content-Type": "text/html; charset=utf-8",
+      "Content-Length": htmlBuffer.length
+    });
+    res.end(htmlBuffer);
+  });
+  const serverPort = 31234;
+  await new Promise((resolve) => {
+    server.listen(serverPort, "127.0.0.1", () => resolve());
+  });
+  const filterArg = process.argv.slice(2).find((a) => !a.startsWith("-"));
+  const scenariosToRun = filterArg ? BENCHMARK_SCENARIOS.filter((s) => s.slug.includes(filterArg)) : BENCHMARK_SCENARIOS;
+  if (scenariosToRun.length === 0) {
+    console.error(
+      `\x1B[31m[ERROR] No scenarios matched filter: "${filterArg}"\x1B[0m`
+    );
+    process.exit(1);
+  }
+  console.log(
+    `
+\x1B[36m=== Autonomous Gemini Walker Benchmark Suite ===\x1B[0m`
+  );
+  console.log(`Provider: ${authDescription}`);
+  console.log(`Model: gemini-2.5-flash`);
+  console.log(`Local Fixture: http://127.0.0.1:${serverPort}`);
+  console.log(
+    `Scenarios: ${scenariosToRun.length}${filterArg ? ` (filtered by "${filterArg}")` : ""}
+`
+  );
+  const scorecards = [];
+  let cdpPortCounter = 19550;
+  try {
+    for (const { slug, queryParam, contract } of scenariosToRun) {
+      console.log(
+        `\x1B[33m\u25B6 Running autonomous walker for scenario:\x1B[0m ${slug}...`
+      );
+      const tmpDir = await fs2.mkdtemp(
+        path2.join(os.tmpdir(), `iamu-eval-gemini-${slug}-`)
+      );
+      await fs2.cp(path2.join(BENCHMARK_DIR, "iamu"), path2.join(tmpDir, "iamu"), {
+        recursive: true
+      });
+      const cdpPort = cdpPortCounter++;
+      const targetUrl = `http://127.0.0.1:${serverPort}/${queryParam}`;
+      try {
+        const { canonical, turns } = await runGeminiWalkthrough({
+          scenarioSlug: slug,
+          targetUrl,
+          projectDir: tmpDir,
+          binPath,
+          cdpPort,
+          apiKey,
+          model: "gemini-2.5-flash",
+          maxTurns: 24
+        });
+        const { evaluateWalkthroughRun: evaluateWalkthroughRun2 } = await Promise.resolve().then(() => (init_walkthrough_evaluator(), walkthrough_evaluator_exports));
+        const scorecard = evaluateWalkthroughRun2(canonical, contract);
+        scorecards.push(scorecard);
+        console.log(
+          `  \x1B[32m\u2713\x1B[0m Completed in ${turns} turns. Verdict: ${scorecard.passed ? "\x1B[32mPASS\x1B[0m" : "\x1B[31mFAIL\x1B[0m"} - ${scorecard.summary}
+`
+        );
+      } catch (err) {
+        console.error(
+          `  \x1B[31m\u2717\x1B[0m Run error:`,
+          err instanceof Error ? err.message : String(err)
+        );
+      } finally {
+        await fs2.rm(tmpDir, { recursive: true, force: true });
+      }
+    }
+    if (scorecards.length > 0) {
+      console.log(formatEvaluationScorecardTable(scorecards));
+    }
+  } finally {
+    server.closeAllConnections?.();
+    server.close();
+  }
+}
+main().catch((err) => {
+  console.error("Fatal error running Gemini walker benchmark:", err);
+  process.exit(1);
+});