@bbearai/ai-executor 0.2.0

package/dist/cli.js

@@ -0,0 +1,591 @@
+#!/usr/bin/env node
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+
+// src/cli.ts
+var import_supabase_js = require("@supabase/supabase-js");
+
+// src/runner.ts
+var import_sdk = __toESM(require("@anthropic-ai/sdk"));
+var import_zod = require("zod");
+
+// src/browser.ts
+var import_stagehand = require("@browserbasehq/stagehand");
+var DEFAULT_MODEL = "anthropic/claude-sonnet-4-20250514";
+async function createStagehandSession(config, anthropicApiKey) {
+  const modelName = config.model ?? DEFAULT_MODEL;
+  const viewport = config.viewport ?? { width: 1280, height: 720 };
+  const stagehand = new import_stagehand.Stagehand({
+    env: config.provider === "browserbase" ? "BROWSERBASE" : "LOCAL",
+    apiKey: config.provider === "browserbase" ? config.browserbaseApiKey : void 0,
+    projectId: config.provider === "browserbase" ? config.browserbaseProjectId : void 0,
+    model: {
+      modelName,
+      apiKey: anthropicApiKey
+    },
+    localBrowserLaunchOptions: config.provider === "local" ? {
+      headless: config.headless ?? true,
+      viewport
+    } : void 0,
+    browserbaseSessionCreateParams: config.provider === "browserbase" ? { projectId: config.browserbaseProjectId } : void 0
+  });
+  await stagehand.init();
+  const page = stagehand.context.activePage();
+  await page.setViewportSize(viewport.width, viewport.height);
+  let sessionId = `local-${Date.now()}`;
+  if (config.provider === "browserbase" && stagehand.browserbaseSessionID) {
+    sessionId = stagehand.browserbaseSessionID;
+  }
+  return {
+    stagehand,
+    page,
+    sessionId,
+    close: async () => {
+      await stagehand.close().catch(() => {
+      });
+    }
+  };
+}
+async function injectAuth(page, auth, stagehand) {
+  if (auth.type === "cookie") {
+    for (const c of auth.cookies) {
+      await page.sendCDP("Network.setCookie", {
+        name: c.name,
+        value: c.value,
+        domain: c.domain,
+        path: c.path ?? "/",
+        secure: c.secure ?? false,
+        httpOnly: c.httpOnly ?? false,
+        sameSite: c.sameSite ?? "Lax"
+      });
+    }
+  } else if (auth.type === "localStorage") {
+    const currentUrl = page.url();
+    if (currentUrl === "about:blank") {
+      return;
+    }
+    await page.evaluate((items) => {
+      for (const [key, value] of Object.entries(items)) {
+        localStorage.setItem(key, value);
+      }
+    }, auth.items);
+  } else if (auth.type === "form-login") {
+    await performFormLogin(page, auth, stagehand);
+  }
+}
+async function performFormLogin(page, auth, stagehand) {
+  await page.goto(auth.loginUrl, { waitUntil: "domcontentloaded" });
+  await page.waitForLoadState("networkidle", 15e3).catch(() => {
+  });
+  if (stagehand) {
+    await stagehand.act(
+      `Fill in the email/username field with "${auth.email}" and the password field with "${auth.password}", then click the login/sign-in button to submit the form.`
+    );
+  } else {
+    await manualFormLogin(page, auth);
+  }
+  await page.waitForLoadState("networkidle", 15e3).catch(() => {
+  });
+}
+async function manualFormLogin(page, auth) {
+  await page.waitForSelector(
+    'input[type="email"], input[type="text"][name*="email"], input[name*="user"], input[type="text"]',
+    { timeout: 15e3 }
+  ).catch(() => {
+  });
+  const emailSelectors = [
+    'input[type="email"]',
+    'input[name="email"]',
+    'input[name="username"]',
+    'input[autocomplete="email"]',
+    'input[autocomplete="username"]',
+    'input[type="text"][name*="email"]',
+    'input[type="text"][name*="user"]',
+    'input[type="text"]'
+  ];
+  let emailFilled = false;
+  for (const sel of emailSelectors) {
+    const locator = page.locator(sel);
+    if (await locator.count() > 0 && await locator.isVisible()) {
+      await locator.fill(auth.email);
+      emailFilled = true;
+      break;
+    }
+  }
+  if (!emailFilled) {
+    throw new Error("Could not find email/username input on login page");
+  }
+  const passwordLocator = page.locator('input[type="password"]');
+  if (await passwordLocator.count() > 0 && await passwordLocator.isVisible()) {
+    await passwordLocator.fill(auth.password);
+  } else {
+    throw new Error("Could not find password input on login page");
+  }
+  const submitSelectors = [
+    'button[type="submit"]',
+    'input[type="submit"]'
+  ];
+  let submitted = false;
+  for (const sel of submitSelectors) {
+    const locator = page.locator(sel);
+    if (await locator.count() > 0 && await locator.isVisible()) {
+      await locator.click();
+      submitted = true;
+      break;
+    }
+  }
+  if (!submitted) {
+    await page.locator('input[type="password"]').type("\n");
+  }
+}
+
+// src/evaluator.ts
+async function generateRunSummary(anthropic, testTitle, steps, model) {
+  const stepsText = steps.map(
+    (s) => `Step ${s.stepNumber}: ${s.action}
+  Expected: ${s.expectedResult}
+  Actual: ${s.actualResult}
+  Result: ${s.passed ? "PASS" : "FAIL"} (confidence: ${Math.round(s.confidence * 100)}%)${s.error ? `
+  Error: ${s.error}` : ""}`
+  ).join("\n\n");
+  const passCount = steps.filter((s) => s.passed).length;
+  const failCount = steps.filter((s) => !s.passed).length;
+  const response = await anthropic.messages.create({
+    model,
+    max_tokens: 512,
+    messages: [
+      {
+        role: "user",
+        content: `Summarize this AI test execution in 2-3 sentences. Focus on what was tested, what passed, and what failed (if anything). Be concise and factual.
+
+Test: ${testTitle}
+Results: ${passCount} passed, ${failCount} failed out of ${steps.length} steps
+
+${stepsText}`
+      }
+    ]
+  });
+  return response.content.filter((block) => block.type === "text").map((block) => block.text).join("");
+}
+
+// src/runner.ts
+async function runTest(config) {
+  const anthropic = new import_sdk.default({ apiKey: config.anthropicApiKey });
+  const startTime = Date.now();
+  const browserConfig = config.browser ?? {
+    provider: "local",
+    headless: true
+  };
+  config.onStatusChange?.("initializing");
+  const session = await createStagehandSession(browserConfig, config.anthropicApiKey);
+  const { stagehand, page } = session;
+  const stepResults = [];
+  let pendingConsoleLogs = [];
+  let pendingNetworkErrors = [];
+  let stepStartTime = Date.now();
+  const rawPage = page;
+  rawPage.on("console", (msg) => {
+    const level = msg.type?.() ?? msg.type ?? "log";
+    const mappedLevel = level === "error" ? "error" : level === "warn" || level === "warning" ? "warning" : level === "info" ? "info" : level === "debug" ? "debug" : "log";
+    pendingConsoleLogs.push({
+      level: mappedLevel,
+      text: (typeof msg.text === "function" ? msg.text() : String(msg.text ?? msg)).slice(0, 2e3),
+      source: typeof msg.location === "function" ? msg.location()?.url : void 0,
+      timestamp: Date.now() - stepStartTime
+    });
+  });
+  rawPage.on("requestfailed", (req) => {
+    const url = typeof req.url === "function" ? req.url() : String(req.url ?? "");
+    const method = typeof req.method === "function" ? req.method() : String(req.method ?? "GET");
+    const failure = typeof req.failure === "function" ? req.failure() : req.failure;
+    pendingNetworkErrors.push({
+      method,
+      url: url.slice(0, 500),
+      status: 0,
+      statusText: failure?.errorText ?? "Request failed",
+      timestamp: Date.now() - stepStartTime
+    });
+  });
+  rawPage.on("response", (res) => {
+    const status = typeof res.status === "function" ? res.status() : Number(res.status ?? 0);
+    if (status >= 400) {
+      const url = typeof res.url === "function" ? res.url() : String(res.url ?? "");
+      const statusText = typeof res.statusText === "function" ? res.statusText() : String(res.statusText ?? "");
+      const req = typeof res.request === "function" ? res.request() : res.request;
+      const method = req ? typeof req.method === "function" ? req.method() : String(req.method ?? "GET") : "GET";
+      pendingNetworkErrors.push({
+        method,
+        url: url.slice(0, 500),
+        status,
+        statusText,
+        timestamp: Date.now() - stepStartTime
+      });
+    }
+  });
+  try {
+    if (config.auth?.type === "form-login") {
+      config.onStatusChange?.("authenticating");
+      await injectAuth(page, config.auth, stagehand);
+    }
+    config.onStatusChange?.("navigating");
+    const targetUrl = config.testCase.targetRoute ? `${config.targetUrl.replace(/\/$/, "")}${config.testCase.targetRoute}` : config.targetUrl;
+    await page.goto(targetUrl, { waitUntil: "domcontentloaded", timeoutMs: 3e4 });
+    if (config.auth && config.auth.type !== "form-login") {
+      config.onStatusChange?.("authenticating");
+      await injectAuth(page, config.auth, stagehand);
+      if (config.auth.type === "localStorage") {
+        await page.evaluate((items) => {
+          for (const [key, value] of Object.entries(items)) {
+            localStorage.setItem(key, value);
+          }
+        }, config.auth.items);
+        await page.reload({ waitUntil: "domcontentloaded" });
+      }
+    }
+    await page.waitForLoadState("networkidle").catch(() => {
+    });
+    pendingConsoleLogs = [];
+    pendingNetworkErrors = [];
+    config.onStatusChange?.("executing");
+    const steps = config.testCase.steps;
+    for (let i = 0; i < steps.length; i++) {
+      const step = steps[i];
+      stepStartTime = Date.now();
+      pendingConsoleLogs = [];
+      pendingNetworkErrors = [];
+      const screenshotBefore = await page.screenshot({ type: "png" });
+      let error;
+      let screenshotAfter = screenshotBefore;
+      let actSucceeded = false;
+      try {
+        await stagehand.act(step.action);
+        actSucceeded = true;
+        await page.waitForLoadState("networkidle").catch(() => {
+        });
+        await page.waitForTimeout(500);
+        screenshotAfter = await page.screenshot({ type: "png" });
+      } catch (err) {
+        error = err instanceof Error ? err.message : String(err);
+        screenshotAfter = await page.screenshot({ type: "png" }).catch(() => screenshotBefore);
+      }
+      let evaluation = {
+        passed: false,
+        confidence: 0,
+        actualResult: error ?? "Action execution failed"
+      };
+      if (actSucceeded) {
+        try {
+          const verificationSchema = import_zod.z.object({
+            passed: import_zod.z.boolean().describe("Whether the expected result was achieved"),
+            confidence: import_zod.z.number().min(0).max(1).describe("Confidence in the assessment (0.9+ = very sure, 0.7-0.9 = likely, below 0.7 = uncertain)"),
+            actualResult: import_zod.z.string().describe("Description of what actually happened on the page")
+          });
+          const verification = await stagehand.extract(
+            `You are evaluating a QA test step. The action "${step.action}" was just performed. Check if this expected result was achieved: "${step.expectedResult}". Look at the current page state and describe what actually happened. Be precise and factual in your assessment.`,
+            verificationSchema
+          );
+          evaluation = {
+            passed: verification.passed,
+            confidence: verification.confidence,
+            actualResult: verification.actualResult
+          };
+        } catch (evalErr) {
+          evaluation = {
+            passed: false,
+            confidence: 0.2,
+            actualResult: `Verification error: ${evalErr instanceof Error ? evalErr.message : String(evalErr)}`
+          };
+        }
+      }
+      const consoleLogs = pendingConsoleLogs.slice(0, 50);
+      const networkErrors = pendingNetworkErrors.slice(0, 30);
+      const result = {
+        stepNumber: step.stepNumber,
+        action: step.action,
+        expectedResult: step.expectedResult,
+        actualResult: evaluation.actualResult,
+        passed: evaluation.passed,
+        confidence: evaluation.confidence,
+        screenshotBefore,
+        screenshotAfter,
+        actionsTaken: [],
+        // Stagehand handles actions internally
+        error,
+        durationMs: Date.now() - stepStartTime,
+        consoleLogs,
+        networkErrors
+      };
+      stepResults.push(result);
+      config.onStepComplete?.(result, i, steps.length);
+    }
+    config.onStatusChange?.("completed");
+    const model = config.model ?? "claude-sonnet-4-20250514";
+    const summary = await generateRunSummary(anthropic, config.testCase.title, stepResults, model);
+    const overallResult = determineOverallResult(stepResults);
+    return {
+      testCaseId: config.testCase.id,
+      testCaseTitle: config.testCase.title,
+      overallResult,
+      steps: stepResults,
+      totalDurationMs: Date.now() - startTime,
+      summary,
+      screenshotUrls: [],
+      tokenUsage: {
+        // Stagehand tracks tokens internally; these are approximate
+        inputTokens: steps.length * 3e3,
+        outputTokens: steps.length * 500
+      },
+      browserSessionId: session.sessionId
+    };
+  } catch (err) {
+    return {
+      testCaseId: config.testCase.id,
+      testCaseTitle: config.testCase.title,
+      overallResult: "error",
+      steps: stepResults,
+      totalDurationMs: Date.now() - startTime,
+      summary: `Test execution failed: ${err instanceof Error ? err.message : String(err)}`,
+      screenshotUrls: [],
+      tokenUsage: {
+        inputTokens: stepResults.length * 3e3,
+        outputTokens: stepResults.length * 500
+      },
+      browserSessionId: session.sessionId
+    };
+  } finally {
+    await session.close();
+  }
+}
+function determineOverallResult(steps) {
+  if (steps.length === 0) return "error";
+  const allPassed = steps.every((s) => s.passed);
+  const allFailed = steps.every((s) => !s.passed);
+  const hasErrors = steps.some((s) => s.error);
+  if (allPassed) return "passed";
+  if (allFailed || hasErrors) return "failed";
+  return "partial";
+}
+
+// src/cli.ts
+function parseArgs() {
+  const args = process.argv.slice(2);
+  const parsed = {
+    url: "",
+    headless: true,
+    provider: "local"
+  };
+  for (let i = 0; i < args.length; i++) {
+    switch (args[i]) {
+      case "--url":
+        parsed.url = args[++i] || "";
+        break;
+      case "--test-case-id":
+        parsed.testCaseId = args[++i];
+        break;
+      case "--test-file":
+        parsed.testFile = args[++i];
+        break;
+      case "--project-id":
+        parsed.projectId = args[++i];
+        break;
+      case "--headless":
+        parsed.headless = args[++i] !== "false";
+        break;
+      case "--provider":
+        parsed.provider = args[++i];
+        break;
+      case "--cookies":
+        parsed.cookies = args[++i];
+        break;
+      case "--local-storage":
+        parsed.localStorage = args[++i];
+        break;
+      case "--help":
+        printHelp();
+        process.exit(0);
+    }
+  }
+  return parsed;
+}
+function printHelp() {
+  console.log(`
+@bbearai/ai-executor - AI-powered QA test executor
+
+Usage:
+  bbear-execute --url <target-url> --test-case-id <id> [options]
+  bbear-execute --url <target-url> --test-file <path> [options]
+
+Required:
+  --url <url>              Target application URL
+
+Test Source (one required):
+  --test-case-id <id>      Fetch test case from BugBear (requires SUPABASE_URL, SUPABASE_ANON_KEY)
+  --test-file <path>       Path to a JSON test case file
+
+Options:
+  --project-id <id>        BugBear project ID (for fetching test cases)
+  --headless <true|false>  Run browser headlessly (default: true)
+  --provider <provider>    Browser provider: local or browserbase (default: local)
+  --cookies <json>         JSON array of cookies for authentication
+  --local-storage <json>   JSON object of localStorage items for authentication
+  --help                   Show this help message
+
+Environment Variables:
+  ANTHROPIC_API_KEY        Required. Claude API key for AI interpretation.
+  SUPABASE_URL             Required when using --test-case-id. BugBear Supabase URL.
+  SUPABASE_ANON_KEY        Required when using --test-case-id. BugBear Supabase anon key.
+  BROWSERBASE_API_KEY      Required when --provider=browserbase.
+  BROWSERBASE_PROJECT_ID   Required when --provider=browserbase.
+
+Examples:
+  # Run with a local test file
+  bbear-execute --url https://staging.myapp.com --test-file ./login-test.json --headless false
+
+  # Run with a BugBear test case
+  bbear-execute --url https://staging.myapp.com --test-case-id abc-123 --project-id xyz-789
+
+  # Run with cookie auth
+  bbear-execute --url https://staging.myapp.com --test-file ./test.json \\
+    --cookies '[{"name":"session","value":"abc123","domain":".myapp.com"}]'
+`);
+}
+async function fetchTestCase(testCaseId) {
+  const supabaseUrl = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL;
+  const supabaseKey = process.env.SUPABASE_ANON_KEY || process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY;
+  if (!supabaseUrl || !supabaseKey) {
+    throw new Error("SUPABASE_URL and SUPABASE_ANON_KEY are required to fetch test cases");
+  }
+  const supabase = (0, import_supabase_js.createClient)(supabaseUrl, supabaseKey);
+  const { data, error } = await supabase.from("test_cases").select("id, title, description, steps, expected_result, preconditions, target_route, estimated_minutes").eq("id", testCaseId).single();
+  if (error || !data) {
+    throw new Error(`Failed to fetch test case ${testCaseId}: ${error?.message ?? "Not found"}`);
+  }
+  return {
+    id: data.id,
+    title: data.title,
+    description: data.description ?? void 0,
+    steps: data.steps,
+    preconditions: data.preconditions ?? void 0,
+    targetRoute: data.target_route ?? void 0,
+    estimatedMinutes: data.estimated_minutes ?? void 0
+  };
+}
+async function loadTestFile(path) {
+  const fs = await import("fs");
+  const content = fs.readFileSync(path, "utf-8");
+  return JSON.parse(content);
+}
+function parseAuth(args) {
+  if (args.cookies) {
+    try {
+      return { type: "cookie", cookies: JSON.parse(args.cookies) };
+    } catch {
+      console.error("Failed to parse --cookies JSON");
+      process.exit(1);
+    }
+  }
+  if (args.localStorage) {
+    try {
+      return { type: "localStorage", items: JSON.parse(args.localStorage) };
+    } catch {
+      console.error("Failed to parse --local-storage JSON");
+      process.exit(1);
+    }
+  }
+  return void 0;
+}
+async function main() {
+  const args = parseArgs();
+  if (!args.url) {
+    console.error("Error: --url is required");
+    printHelp();
+    process.exit(1);
+  }
+  if (!args.testCaseId && !args.testFile) {
+    console.error("Error: Either --test-case-id or --test-file is required");
+    printHelp();
+    process.exit(1);
+  }
+  const anthropicApiKey = process.env.ANTHROPIC_API_KEY;
+  if (!anthropicApiKey) {
+    console.error("Error: ANTHROPIC_API_KEY environment variable is required");
+    process.exit(1);
+  }
+  console.log("Loading test case...");
+  const testCase = args.testCaseId ? await fetchTestCase(args.testCaseId) : await loadTestFile(args.testFile);
+  console.log(`Test: ${testCase.title}`);
+  console.log(`Steps: ${testCase.steps.length}`);
+  console.log(`Target: ${args.url}${testCase.targetRoute ?? ""}`);
+  console.log("");
+  const browser = {
+    provider: args.provider,
+    headless: args.headless,
+    browserbaseApiKey: process.env.BROWSERBASE_API_KEY,
+    browserbaseProjectId: process.env.BROWSERBASE_PROJECT_ID
+  };
+  const result = await runTest({
+    targetUrl: args.url,
+    testCase,
+    auth: parseAuth(args),
+    browser,
+    anthropicApiKey,
+    onStepComplete: (step, index, total) => {
+      const icon = step.passed ? "\u2705" : "\u274C";
+      const confidence = Math.round(step.confidence * 100);
+      console.log(
+        `${icon} Step ${index + 1}/${total}: ${step.action.slice(0, 60)}... [${step.passed ? "PASS" : "FAIL"} ${confidence}%] (${step.durationMs}ms)`
+      );
+      if (!step.passed) {
+        console.log(`   Expected: ${step.expectedResult.slice(0, 80)}`);
+        console.log(`   Actual:   ${step.actualResult.slice(0, 80)}`);
+      }
+      if (step.error) {
+        console.log(`   Error:    ${step.error.slice(0, 80)}`);
+      }
+    },
+    onStatusChange: (status) => {
+      if (status !== "executing" && status !== "evaluating") {
+        console.log(`[${status}]`);
+      }
+    }
+  });
+  console.log("");
+  console.log("=".repeat(60));
+  console.log(`Result: ${result.overallResult.toUpperCase()}`);
+  console.log(`Duration: ${Math.round(result.totalDurationMs / 1e3)}s`);
+  console.log(
+    `Steps: ${result.steps.filter((s) => s.passed).length} passed, ${result.steps.filter((s) => !s.passed).length} failed`
+  );
+  console.log(`Tokens: ${result.tokenUsage.inputTokens} in / ${result.tokenUsage.outputTokens} out`);
+  console.log("");
+  console.log("Summary:");
+  console.log(result.summary);
+  console.log("=".repeat(60));
+  process.exit(result.overallResult === "passed" ? 0 : 1);
+}
+main().catch((err) => {
+  console.error("Fatal error:", err);
+  process.exit(1);
+});
+//# sourceMappingURL=cli.js.map