@bbearai/ai-executor 0.2.0 → 0.2.1

package/dist/cli.js

@@ -28,10 +28,88 @@ var import_supabase_js = require("@supabase/supabase-js");
 
 // src/runner.ts
 var import_sdk = __toESM(require("@anthropic-ai/sdk"));
-var import_zod = require("zod");
 
 // src/browser.ts
 var import_stagehand = require("@browserbasehq/stagehand");
+
+// src/supabase-auth.ts
+function extractProjectRef(supabaseUrl) {
+  const url = new URL(supabaseUrl);
+  const hostname = url.hostname;
+  const ref = hostname.split(".")[0];
+  return ref;
+}
+async function authenticateSupabase(auth) {
+  const url = `${auth.supabaseUrl.replace(/\/$/, "")}/auth/v1/token?grant_type=password`;
+  const response = await fetch(url, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "apikey": auth.anonKey
+    },
+    body: JSON.stringify({
+      email: auth.email,
+      password: auth.password
+    })
+  });
+  if (!response.ok) {
+    const body = await response.text().catch(() => "");
+    throw new Error(
+      `Supabase auth failed (${response.status}): ${body.slice(0, 200)}`
+    );
+  }
+  const session = await response.json();
+  if (!session.access_token) {
+    throw new Error("Supabase auth returned no access_token");
+  }
+  return session;
+}
+async function injectSupabaseAuth(page, auth, session) {
+  const ref = extractProjectRef(auth.supabaseUrl);
+  const storageKey = `sb-${ref}-auth-token`;
+  const storageValue = JSON.stringify({
+    access_token: session.access_token,
+    refresh_token: session.refresh_token,
+    expires_in: session.expires_in,
+    expires_at: session.expires_at,
+    token_type: session.token_type,
+    user: session.user
+  });
+  const currentUrl = page.url();
+  if (currentUrl === "about:blank" || !currentUrl) {
+    await page.goto(auth.supabaseUrl.replace(/\/$/, ""), {
+      waitUntil: "domcontentloaded",
+      timeoutMs: 1e4
+    }).catch(() => {
+    });
+  }
+  await page.evaluate(
+    ({ key, value }) => {
+      localStorage.setItem(key, value);
+    },
+    { key: storageKey, value: storageValue }
+  );
+}
+async function verifySupabaseSession(auth, accessToken) {
+  const url = `${auth.supabaseUrl.replace(/\/$/, "")}/auth/v1/user`;
+  const response = await fetch(url, {
+    headers: {
+      "Authorization": `Bearer ${accessToken}`,
+      "apikey": auth.anonKey
+    }
+  });
+  return response.ok;
+}
+async function performSupabaseAuth(page, auth) {
+  const session = await authenticateSupabase(auth);
+  await injectSupabaseAuth(page, auth, session);
+  const valid = await verifySupabaseSession(auth, session.access_token);
+  if (!valid) {
+    throw new Error("Supabase auth verification failed \u2014 session token rejected");
+  }
+}
+
+// src/browser.ts
 var DEFAULT_MODEL = "anthropic/claude-sonnet-4-20250514";
 async function createStagehandSession(config, anthropicApiKey) {
   const modelName = config.model ?? DEFAULT_MODEL;
@@ -44,6 +122,11 @@ async function createStagehandSession(config, anthropicApiKey) {
       modelName,
       apiKey: anthropicApiKey
     },
+    // Bypass pino logger — its pino-pretty transport uses worker threads
+    // which fail in Vercel's serverless environment
+    logger: (msg) => {
+      if ((msg.level ?? 0) >= 40) console.warn("[Stagehand]", msg.message);
+    },
     localBrowserLaunchOptions: config.provider === "local" ? {
       headless: config.headless ?? true,
       viewport
@@ -67,6 +150,21 @@ async function createStagehandSession(config, anthropicApiKey) {
     }
   };
 }
+async function suppressBugBearWidget(stagehand) {
+  try {
+    const ctx = stagehand.context;
+    if (ctx?.addInitScript) {
+      await ctx.addInitScript(() => {
+        window.__bugbear_suppress = true;
+        try {
+          localStorage.setItem("__bugbear_suppress", "true");
+        } catch {
+        }
+      });
+    }
+  } catch {
+  }
+}
 async function injectAuth(page, auth, stagehand) {
   if (auth.type === "cookie") {
     for (const c of auth.cookies) {
@@ -92,23 +190,27 @@ async function injectAuth(page, auth, stagehand) {
     }, auth.items);
   } else if (auth.type === "form-login") {
     await performFormLogin(page, auth, stagehand);
+  } else if (auth.type === "supabase-native") {
+    await performSupabaseAuth(page, auth);
   }
 }
 async function performFormLogin(page, auth, stagehand) {
   await page.goto(auth.loginUrl, { waitUntil: "domcontentloaded" });
   await page.waitForLoadState("networkidle", 15e3).catch(() => {
   });
+  await fillLoginCredentials(page, auth);
   if (stagehand) {
     await stagehand.act(
-      `Fill in the email/username field with "${auth.email}" and the password field with "${auth.password}", then click the login/sign-in button to submit the form.`
-    );
+      "Click the login, sign-in, or submit button to submit the form."
+    ).catch(() => {
+    });
   } else {
-    await manualFormLogin(page, auth);
+    await clickSubmitButton(page);
   }
   await page.waitForLoadState("networkidle", 15e3).catch(() => {
   });
 }
-async function manualFormLogin(page, auth) {
+async function fillLoginCredentials(page, auth) {
   await page.waitForSelector(
     'input[type="email"], input[type="text"][name*="email"], input[name*="user"], input[type="text"]',
     { timeout: 15e3 }
@@ -142,6 +244,8 @@ async function manualFormLogin(page, auth) {
   } else {
     throw new Error("Could not find password input on login page");
   }
+}
+async function clickSubmitButton(page) {
   const submitSelectors = [
     'button[type="submit"]',
     'input[type="submit"]'
@@ -166,21 +270,23 @@ async function generateRunSummary(anthropic, testTitle, steps, model) {
     (s) => `Step ${s.stepNumber}: ${s.action}
   Expected: ${s.expectedResult}
   Actual: ${s.actualResult}
-  Result: ${s.passed ? "PASS" : "FAIL"} (confidence: ${Math.round(s.confidence * 100)}%)${s.error ? `
+  Result: ${s.skipped ? "SKIPPED" : s.passed ? "PASS" : "FAIL"} (confidence: ${Math.round(s.confidence * 100)}%)${s.error ? `
   Error: ${s.error}` : ""}`
   ).join("\n\n");
-  const passCount = steps.filter((s) => s.passed).length;
-  const failCount = steps.filter((s) => !s.passed).length;
+  const passCount = steps.filter((s) => s.passed && !s.skipped).length;
+  const failCount = steps.filter((s) => !s.passed && !s.skipped).length;
+  const skipCount = steps.filter((s) => s.skipped).length;
+  const skipNote = skipCount > 0 ? " Some steps were skipped due to page state recovery \u2014 these are not failures, just steps that could not be executed." : "";
   const response = await anthropic.messages.create({
     model,
     max_tokens: 512,
     messages: [
       {
         role: "user",
-        content: `Summarize this AI test execution in 2-3 sentences. Focus on what was tested, what passed, and what failed (if anything). Be concise and factual.
+        content: `Summarize this AI test execution in 2-3 sentences. Focus on what was tested, what passed, and what failed (if anything).${skipNote} Be concise and factual.
 
 Test: ${testTitle}
-Results: ${passCount} passed, ${failCount} failed out of ${steps.length} steps
+Results: ${passCount} passed, ${failCount} failed, ${skipCount} skipped out of ${steps.length} steps
 
 ${stepsText}`
       }
@@ -189,7 +295,317 @@ ${stepsText}`
   return response.content.filter((block) => block.type === "text").map((block) => block.text).join("");
 }
 
+// src/vision-evaluator.ts
+var DEFAULT_MODEL2 = "claude-sonnet-4-20250514";
+async function evaluateStep(input) {
+  const model = input.model ?? DEFAULT_MODEL2;
+  const hintClause = input.evaluationHint ? `
+EVALUATION HINT: ${input.evaluationHint}` : "";
+  const response = await input.anthropic.messages.create({
+    model,
+    max_tokens: 512,
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "BEFORE screenshot (page state before the action):"
+          },
+          {
+            type: "image",
+            source: {
+              type: "base64",
+              media_type: "image/png",
+              data: input.screenshotBefore.toString("base64")
+            }
+          },
+          {
+            type: "text",
+            text: "AFTER screenshot (page state after the action):"
+          },
+          {
+            type: "image",
+            source: {
+              type: "base64",
+              media_type: "image/png",
+              data: input.screenshotAfter.toString("base64")
+            }
+          },
+          {
+            type: "text",
+            text: `You are a QA test evaluator. Compare the BEFORE and AFTER screenshots to evaluate this test step.
+
+ACTION PERFORMED: ${input.action}
+EXPECTED RESULT: ${input.expectedResult}${hintClause}
+
+Analyze the visual differences between the two screenshots and determine if the expected result was achieved.
+
+Respond with ONLY a JSON object (no markdown, no explanation outside the JSON):
+{
+  "passed": true/false,
+  "confidence": 0.0-1.0,
+  "actualResult": "Brief description of what actually changed between the screenshots"
+}
+
+Confidence guide:
+- 0.95-1.0: Clearly achieved/not achieved, obvious visual evidence
+- 0.8-0.94: Very likely, strong visual indicators
+- 0.6-0.79: Probable but some ambiguity
+- Below 0.6: Uncertain, hard to tell from screenshots alone`
+          }
+        ]
+      }
+    ]
+  });
+  const text = response.content.filter((block) => block.type === "text").map((block) => block.text).join("");
+  return parseEvaluation(text);
+}
+function parseEvaluation(text) {
+  try {
+    const parsed = JSON.parse(text.trim());
+    return validateEvaluation(parsed);
+  } catch {
+    const jsonMatch = text.match(/\{[\s\S]*"passed"[\s\S]*"confidence"[\s\S]*"actualResult"[\s\S]*\}/);
+    if (jsonMatch) {
+      try {
+        const parsed = JSON.parse(jsonMatch[0]);
+        return validateEvaluation(parsed);
+      } catch {
+      }
+    }
+  }
+  return {
+    passed: false,
+    confidence: 0.3,
+    actualResult: `Vision evaluation returned unparseable response: ${text.slice(0, 200)}`
+  };
+}
+function validateEvaluation(parsed) {
+  return {
+    passed: typeof parsed.passed === "boolean" ? parsed.passed : false,
+    confidence: typeof parsed.confidence === "number" ? Math.max(0, Math.min(1, parsed.confidence)) : 0.5,
+    actualResult: typeof parsed.actualResult === "string" ? parsed.actualResult : "No description provided"
+  };
+}
+
+// src/action-executor.ts
+async function executeAction(page, stagehand, step) {
+  if (step.selector && step.actionType) {
+    try {
+      await executePlaywrightAction(page, step);
+      return { deterministic: true };
+    } catch (err) {
+      const fallbackResult = await executeStagehandAction(stagehand, step);
+      return {
+        deterministic: false,
+        error: fallbackResult.error ? `Playwright failed (${err instanceof Error ? err.message : String(err)}), Stagehand fallback also failed: ${fallbackResult.error}` : void 0
+      };
+    }
+  }
+  return executeStagehandAction(stagehand, step);
+}
+async function executePlaywrightAction(page, step) {
+  const { actionType, selector, value, waitMs } = step;
+  switch (actionType) {
+    case "click": {
+      const locator = page.locator(selector);
+      await locator.click();
+      break;
+    }
+    case "fill": {
+      const locator = page.locator(selector);
+      await locator.fill(value ?? "");
+      break;
+    }
+    case "select": {
+      await page.evaluate(
+        ({ sel, val }) => {
+          const el = document.querySelector(sel);
+          if (!el) throw new Error(`Select element not found: ${sel}`);
+          el.value = val;
+          el.dispatchEvent(new Event("change", { bubbles: true }));
+        },
+        { sel: selector, val: value ?? "" }
+      );
+      break;
+    }
+    case "navigate": {
+      const url = value ?? selector ?? "";
+      if (!url) throw new Error("Navigate action requires a value or selector with the URL");
+      await page.goto(url, { waitUntil: "domcontentloaded", timeoutMs: 15e3 });
+      break;
+    }
+    case "scroll": {
+      await page.evaluate((sel) => {
+        const el = document.querySelector(sel);
+        if (el) el.scrollIntoView({ behavior: "smooth", block: "center" });
+      }, selector);
+      break;
+    }
+    case "wait": {
+      if (selector) {
+        await page.waitForSelector(selector, { timeout: waitMs ?? 1e4 });
+      } else if (waitMs) {
+        await page.waitForTimeout(waitMs);
+      }
+      break;
+    }
+    case "assert": {
+      break;
+    }
+    default: {
+      throw new Error(`Unknown actionType: ${actionType}`);
+    }
+  }
+  if (waitMs && actionType !== "wait") {
+    await page.waitForTimeout(waitMs);
+  }
+}
+async function executeStagehandAction(stagehand, step) {
+  try {
+    await stagehand.act(step.action);
+    return { deterministic: false };
+  } catch (err) {
+    return {
+      deterministic: false,
+      error: err instanceof Error ? err.message : String(err)
+    };
+  }
+}
+
+// src/selector-discovery.ts
+async function discoverSelector(page) {
+  try {
+    const result = await page.evaluate(() => {
+      const el = document.__bbLastClicked ?? document.activeElement;
+      if (!el || el === document.body || el === document.documentElement) return null;
+      const tagName = el.tagName?.toLowerCase() ?? "unknown";
+      const textContent = (el.textContent ?? "").trim().slice(0, 100);
+      let selector = "";
+      let strategy = "css-path";
+      const testId = el.getAttribute("data-testid") ?? el.getAttribute("data-test-id");
+      if (testId) {
+        selector = `[data-testid="${testId}"]`;
+        strategy = "data-testid";
+      } else if (el.id && !/^:r[0-9a-z]+:?$/.test(el.id) && !/^react-/.test(el.id)) {
+        selector = `#${el.id}`;
+        strategy = "id";
+      } else if (el.getAttribute("role")) {
+        const role = el.getAttribute("role");
+        const name = el.getAttribute("aria-label") ?? el.getAttribute("name") ?? "";
+        if (name) {
+          selector = `[role="${role}"][aria-label="${name}"]`;
+          strategy = "role";
+        } else {
+          selector = `[role="${role}"]`;
+          strategy = "role";
+        }
+      } else if (el.getAttribute("aria-label")) {
+        selector = `[aria-label="${el.getAttribute("aria-label")}"]`;
+        strategy = "aria-label";
+      } else {
+        const parts = [];
+        let current = el;
+        while (current && current !== document.body) {
+          let part = current.tagName.toLowerCase();
+          if (current.className && typeof current.className === "string") {
+            const classes = current.className.split(/\s+/).filter(
+              (c) => c && !c.startsWith("_") && c.length < 30
+            );
+            if (classes.length > 0) {
+              part += `.${classes[0]}`;
+            }
+          }
+          parts.unshift(part);
+          current = current.parentElement;
+          if (parts.length >= 4) break;
+        }
+        selector = parts.join(" > ");
+        strategy = "css-path";
+      }
+      let suggestedActionType;
+      if (tagName === "button" || tagName === "a" || el.getAttribute("role") === "button") {
+        suggestedActionType = "click";
+      } else if (tagName === "input" || tagName === "textarea") {
+        const type = el.getAttribute("type") ?? "text";
+        if (type === "checkbox" || type === "radio") {
+          suggestedActionType = "click";
+        } else {
+          suggestedActionType = "fill";
+        }
+      } else if (tagName === "select") {
+        suggestedActionType = "select";
+      }
+      return { selector, strategy, suggestedActionType, tagName, textContent };
+    });
+    return result;
+  } catch {
+    return null;
+  }
+}
+async function installClickTracker(page) {
+  try {
+    await page.evaluate(() => {
+      document.addEventListener("click", (e) => {
+        document.__bbLastClicked = e.target;
+      }, { capture: true });
+    });
+  } catch {
+  }
+}
+
+// src/cost.ts
+var TOKEN_PROFILE = {
+  /** act() — screenshot + DOM context → action decision */
+  actInput: 2e3,
+  actOutput: 200,
+  /** extract() — screenshot + extraction schema → structured result */
+  extractInput: 3e3,
+  extractOutput: 500,
+  /** summary — all step results → narrative summary (once per run) */
+  summaryInput: 2e3,
+  summaryOutput: 500
+};
+function getTokenEstimate(stepCount) {
+  return {
+    inputTokens: stepCount * (TOKEN_PROFILE.actInput + TOKEN_PROFILE.extractInput) + TOKEN_PROFILE.summaryInput,
+    outputTokens: stepCount * (TOKEN_PROFILE.actOutput + TOKEN_PROFILE.extractOutput) + TOKEN_PROFILE.summaryOutput
+  };
+}
+
 // src/runner.ts
+var AI_OPERATION_TIMEOUT_MS = 3e4;
+var DEFAULT_MAX_RETRIES = 2;
+var DEFAULT_RETRY_DELAY_MS = 2e3;
+function isRetryableError(error) {
+  const patterns = [
+    /timed?\s*out/i,
+    /ECONNREFUSED/i,
+    /ECONNRESET/i,
+    /ENOTFOUND/i,
+    /net::ERR_/i,
+    /navigation failed/i,
+    /page crashed/i,
+    /context was destroyed/i,
+    /target closed/i,
+    /session closed/i,
+    /browser disconnected/i,
+    /execution context/i
+  ];
+  return patterns.some((p) => p.test(error));
+}
+async function withTimeout(promise, timeoutMs, operation) {
+  let timeoutId;
+  const timeoutPromise = new Promise((_, reject) => {
+    timeoutId = setTimeout(() => reject(new Error(`${operation} timed out after ${timeoutMs}ms`)), timeoutMs);
+  });
+  try {
+    return await Promise.race([promise, timeoutPromise]);
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
 async function runTest(config) {
   const anthropic = new import_sdk.default({ apiKey: config.anthropicApiKey });
   const startTime = Date.now();
@@ -198,60 +614,71 @@ async function runTest(config) {
     headless: true
   };
   config.onStatusChange?.("initializing");
-  const session = await createStagehandSession(browserConfig, config.anthropicApiKey);
-  const { stagehand, page } = session;
+  let session;
   const stepResults = [];
   let pendingConsoleLogs = [];
   let pendingNetworkErrors = [];
   let stepStartTime = Date.now();
-  const rawPage = page;
-  rawPage.on("console", (msg) => {
-    const level = msg.type?.() ?? msg.type ?? "log";
-    const mappedLevel = level === "error" ? "error" : level === "warn" || level === "warning" ? "warning" : level === "info" ? "info" : level === "debug" ? "debug" : "log";
-    pendingConsoleLogs.push({
-      level: mappedLevel,
-      text: (typeof msg.text === "function" ? msg.text() : String(msg.text ?? msg)).slice(0, 2e3),
-      source: typeof msg.location === "function" ? msg.location()?.url : void 0,
-      timestamp: Date.now() - stepStartTime
-    });
-  });
-  rawPage.on("requestfailed", (req) => {
-    const url = typeof req.url === "function" ? req.url() : String(req.url ?? "");
-    const method = typeof req.method === "function" ? req.method() : String(req.method ?? "GET");
-    const failure = typeof req.failure === "function" ? req.failure() : req.failure;
-    pendingNetworkErrors.push({
-      method,
-      url: url.slice(0, 500),
-      status: 0,
-      statusText: failure?.errorText ?? "Request failed",
-      timestamp: Date.now() - stepStartTime
-    });
-  });
-  rawPage.on("response", (res) => {
-    const status = typeof res.status === "function" ? res.status() : Number(res.status ?? 0);
-    if (status >= 400) {
-      const url = typeof res.url === "function" ? res.url() : String(res.url ?? "");
-      const statusText = typeof res.statusText === "function" ? res.statusText() : String(res.statusText ?? "");
-      const req = typeof res.request === "function" ? res.request() : res.request;
-      const method = req ? typeof req.method === "function" ? req.method() : String(req.method ?? "GET") : "GET";
-      pendingNetworkErrors.push({
-        method,
-        url: url.slice(0, 500),
-        status,
-        statusText,
-        timestamp: Date.now() - stepStartTime
+  try {
+    session = await createStagehandSession(browserConfig, config.anthropicApiKey);
+    const { stagehand, page } = session;
+    await suppressBugBearWidget(stagehand);
+    const rawPage = page;
+    try {
+      rawPage.on("console", (msg) => {
+        const level = msg.type?.() ?? msg.type ?? "log";
+        const mappedLevel = level === "error" ? "error" : level === "warn" || level === "warning" ? "warning" : level === "info" ? "info" : level === "debug" ? "debug" : "log";
+        pendingConsoleLogs.push({
+          level: mappedLevel,
+          text: (typeof msg.text === "function" ? msg.text() : String(msg.text ?? msg)).slice(0, 2e3),
+          source: typeof msg.location === "function" ? msg.location()?.url : void 0,
+          timestamp: Date.now() - stepStartTime
+        });
       });
+    } catch {
     }
-  });
-  try {
-    if (config.auth?.type === "form-login") {
+    try {
+      rawPage.on("requestfailed", (req) => {
+        const url = typeof req.url === "function" ? req.url() : String(req.url ?? "");
+        const method = typeof req.method === "function" ? req.method() : String(req.method ?? "GET");
+        const failure = typeof req.failure === "function" ? req.failure() : req.failure;
+        pendingNetworkErrors.push({
+          method,
+          url: url.slice(0, 500),
+          status: 0,
+          statusText: failure?.errorText ?? "Request failed",
+          timestamp: Date.now() - stepStartTime
+        });
+      });
+    } catch {
+    }
+    try {
+      rawPage.on("response", (res) => {
+        const status = typeof res.status === "function" ? res.status() : Number(res.status ?? 0);
+        if (status >= 400) {
+          const url = typeof res.url === "function" ? res.url() : String(res.url ?? "");
+          const statusText = typeof res.statusText === "function" ? res.statusText() : String(res.statusText ?? "");
+          const req = typeof res.request === "function" ? res.request() : res.request;
+          const method = req ? typeof req.method === "function" ? req.method() : String(req.method ?? "GET") : "GET";
+          pendingNetworkErrors.push({
+            method,
+            url: url.slice(0, 500),
+            status,
+            statusText,
+            timestamp: Date.now() - stepStartTime
+          });
+        }
+      });
+    } catch {
+    }
+    if (config.auth?.type === "form-login" || config.auth?.type === "supabase-native") {
       config.onStatusChange?.("authenticating");
       await injectAuth(page, config.auth, stagehand);
     }
     config.onStatusChange?.("navigating");
     const targetUrl = config.testCase.targetRoute ? `${config.targetUrl.replace(/\/$/, "")}${config.testCase.targetRoute}` : config.targetUrl;
     await page.goto(targetUrl, { waitUntil: "domcontentloaded", timeoutMs: 3e4 });
-    if (config.auth && config.auth.type !== "form-login") {
+    if (config.auth && config.auth.type !== "form-login" && config.auth.type !== "supabase-native") {
       config.onStatusChange?.("authenticating");
       await injectAuth(page, config.auth, stagehand);
       if (config.auth.type === "localStorage") {
@@ -265,79 +692,143 @@ async function runTest(config) {
     }
     await page.waitForLoadState("networkidle").catch(() => {
     });
+    await page.evaluate(() => {
+      window.__bugbear_suppress = true;
+      try {
+        localStorage.setItem("__bugbear_suppress", "true");
+      } catch {
+      }
+    }).catch(() => {
+    });
+    await installClickTracker(page);
     pendingConsoleLogs = [];
     pendingNetworkErrors = [];
     config.onStatusChange?.("executing");
     const steps = config.testCase.steps;
+    const maxRetries = config.retry?.maxRetries ?? DEFAULT_MAX_RETRIES;
+    const retryDelayMs = config.retry?.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS;
+    const resilientMode = config.resilientMode ?? true;
     for (let i = 0; i < steps.length; i++) {
       const step = steps[i];
-      stepStartTime = Date.now();
-      pendingConsoleLogs = [];
-      pendingNetworkErrors = [];
-      const screenshotBefore = await page.screenshot({ type: "png" });
-      let error;
-      let screenshotAfter = screenshotBefore;
-      let actSucceeded = false;
-      try {
-        await stagehand.act(step.action);
-        actSucceeded = true;
-        await page.waitForLoadState("networkidle").catch(() => {
-        });
-        await page.waitForTimeout(500);
-        screenshotAfter = await page.screenshot({ type: "png" });
-      } catch (err) {
-        error = err instanceof Error ? err.message : String(err);
+      const retryHistory = [];
+      let finalResult;
+      for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        stepStartTime = Date.now();
+        pendingConsoleLogs = [];
+        pendingNetworkErrors = [];
+        const screenshotBefore = await page.screenshot({ type: "png" });
+        let error;
+        let screenshotAfter = screenshotBefore;
+        let actSucceeded = false;
+        const actionResult = await executeAction(page, stagehand, step);
+        error = actionResult.error;
+        actSucceeded = !error;
+        if (actSucceeded) {
+          await page.waitForLoadState("networkidle").catch(() => {
+          });
+          await page.waitForTimeout(step.waitMs ?? 500);
+        }
         screenshotAfter = await page.screenshot({ type: "png" }).catch(() => screenshotBefore);
+        let evaluation = {
+          passed: false,
+          confidence: 0,
+          actualResult: error ?? "Action execution failed"
+        };
+        if (actSucceeded) {
+          try {
+            const visionResult = await withTimeout(
+              evaluateStep({
+                anthropic,
+                screenshotBefore,
+                screenshotAfter,
+                action: step.action,
+                expectedResult: step.expectedResult,
+                evaluationHint: step.evaluationHint,
+                model: config.model
+              }),
+              AI_OPERATION_TIMEOUT_MS,
+              "Vision evaluation"
+            );
+            evaluation = {
+              passed: visionResult.passed,
+              confidence: visionResult.confidence,
+              actualResult: visionResult.actualResult
+            };
+          } catch (evalErr) {
+            evaluation = {
+              passed: false,
+              confidence: 0.2,
+              actualResult: `Vision evaluation error: ${evalErr instanceof Error ? evalErr.message : String(evalErr)}`
+            };
+          }
+        }
+        let discoveredActions = [];
+        if (actSucceeded && !actionResult.deterministic) {
+          const discovered = await discoverSelector(page);
+          if (discovered) {
+            discoveredActions = [{
+              type: discovered.suggestedActionType ?? "click",
+              selector: discovered.selector,
+              description: `Discovered via ${discovered.strategy}: ${discovered.tagName}${discovered.textContent ? ` "${discovered.textContent.slice(0, 50)}"` : ""}`
+            }];
+          }
+        }
+        const consoleLogs = pendingConsoleLogs.slice(0, 50);
+        const networkErrors = pendingNetworkErrors.slice(0, 30);
+        finalResult = {
+          stepNumber: step.stepNumber,
+          action: step.action,
+          expectedResult: step.expectedResult,
+          actualResult: evaluation.actualResult,
+          passed: evaluation.passed,
+          confidence: evaluation.confidence,
+          screenshotBefore,
+          screenshotAfter,
+          actionsTaken: discoveredActions,
+          error,
+          durationMs: Date.now() - stepStartTime,
+          consoleLogs,
+          networkErrors,
+          retryCount: attempt,
+          retryHistory,
+          skipped: false
+        };
+        const shouldRetry = !evaluation.passed && error && isRetryableError(error) && attempt < maxRetries;
+        if (!shouldRetry) break;
+        retryHistory.push({
+          attempt,
+          error,
+          confidence: evaluation.confidence,
+          timestamp: Date.now()
+        });
+        await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
       }
-      let evaluation = {
-        passed: false,
-        confidence: 0,
-        actualResult: error ?? "Action execution failed"
-      };
-      if (actSucceeded) {
+      if (resilientMode && finalResult && !finalResult.passed) {
+        finalResult.skipped = true;
+        finalResult.skipReason = "Step failed, recovered page state";
         try {
-          const verificationSchema = import_zod.z.object({
-            passed: import_zod.z.boolean().describe("Whether the expected result was achieved"),
-            confidence: import_zod.z.number().min(0).max(1).describe("Confidence in the assessment (0.9+ = very sure, 0.7-0.9 = likely, below 0.7 = uncertain)"),
-            actualResult: import_zod.z.string().describe("Description of what actually happened on the page")
+          config.onStatusChange?.("navigating");
+          await page.goto(targetUrl, { waitUntil: "domcontentloaded", timeoutMs: 3e4 });
+          await page.waitForLoadState("networkidle").catch(() => {
           });
-          const verification = await stagehand.extract(
-            `You are evaluating a QA test step. The action "${step.action}" was just performed. Check if this expected result was achieved: "${step.expectedResult}". Look at the current page state and describe what actually happened. Be precise and factual in your assessment.`,
-            verificationSchema
-          );
-          evaluation = {
-            passed: verification.passed,
-            confidence: verification.confidence,
-            actualResult: verification.actualResult
-          };
-        } catch (evalErr) {
-          evaluation = {
-            passed: false,
-            confidence: 0.2,
-            actualResult: `Verification error: ${evalErr instanceof Error ? evalErr.message : String(evalErr)}`
-          };
+          await installClickTracker(page);
+          await page.evaluate(() => {
+            window.__bugbear_suppress = true;
+            try {
+              localStorage.setItem("__bugbear_suppress", "true");
+            } catch {
+            }
+          }).catch(() => {
+          });
+          pendingConsoleLogs = [];
+          pendingNetworkErrors = [];
+          config.onStatusChange?.("executing");
+        } catch (recoveryErr) {
+          finalResult.skipReason = `Step failed, recovery also failed: ${recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)}`;
         }
       }
-      const consoleLogs = pendingConsoleLogs.slice(0, 50);
-      const networkErrors = pendingNetworkErrors.slice(0, 30);
-      const result = {
-        stepNumber: step.stepNumber,
-        action: step.action,
-        expectedResult: step.expectedResult,
-        actualResult: evaluation.actualResult,
-        passed: evaluation.passed,
-        confidence: evaluation.confidence,
-        screenshotBefore,
-        screenshotAfter,
-        actionsTaken: [],
-        // Stagehand handles actions internally
-        error,
-        durationMs: Date.now() - stepStartTime,
-        consoleLogs,
-        networkErrors
-      };
-      stepResults.push(result);
-      config.onStepComplete?.(result, i, steps.length);
+      stepResults.push(finalResult);
+      config.onStepComplete?.(finalResult, i, steps.length);
     }
     config.onStatusChange?.("completed");
     const model = config.model ?? "claude-sonnet-4-20250514";
@@ -351,11 +842,7 @@ async function runTest(config) {
       totalDurationMs: Date.now() - startTime,
       summary,
       screenshotUrls: [],
-      tokenUsage: {
-        // Stagehand tracks tokens internally; these are approximate
-        inputTokens: steps.length * 3e3,
-        outputTokens: steps.length * 500
-      },
+      tokenUsage: getTokenEstimate(steps.length),
       browserSessionId: session.sessionId
     };
   } catch (err) {
@@ -367,23 +854,29 @@ async function runTest(config) {
       totalDurationMs: Date.now() - startTime,
       summary: `Test execution failed: ${err instanceof Error ? err.message : String(err)}`,
       screenshotUrls: [],
-      tokenUsage: {
-        inputTokens: stepResults.length * 3e3,
-        outputTokens: stepResults.length * 500
-      },
-      browserSessionId: session.sessionId
+      tokenUsage: getTokenEstimate(stepResults.length),
+      browserSessionId: session?.sessionId ?? "unknown"
     };
   } finally {
-    await session.close();
+    if (session?.page) {
+      const rawPage = session.page;
+      rawPage.removeAllListeners?.("console");
+      rawPage.removeAllListeners?.("requestfailed");
+      rawPage.removeAllListeners?.("response");
+    }
+    await session?.close();
   }
 }
 function determineOverallResult(steps) {
   if (steps.length === 0) return "error";
-  const allPassed = steps.every((s) => s.passed);
-  const allFailed = steps.every((s) => !s.passed);
-  const hasErrors = steps.some((s) => s.error);
-  if (allPassed) return "passed";
-  if (allFailed || hasErrors) return "failed";
+  const nonSkipped = steps.filter((s) => !s.skipped);
+  const skippedCount = steps.length - nonSkipped.length;
+  if (nonSkipped.length === 0) return "error";
+  const allNonSkippedPassed = nonSkipped.every((s) => s.passed);
+  const hasErrors = nonSkipped.some((s) => s.error);
+  if (skippedCount > 0 && allNonSkippedPassed) return "passed_with_skips";
+  if (allNonSkippedPassed) return "passed";
+  if (nonSkipped.every((s) => !s.passed) || hasErrors) return "failed";
   return "partial";
 }