@testdriverai/agent 7.8.0-canary.10

package/manual/flake-shared.mjs

@@ -0,0 +1,51 @@
+/**
+ * Shared test logic for popup-loading variants.
+ * Each variant file imports this and calls it with specific options.
+ */
+import { describe, expect, it } from "vitest";
+import { TestDriver } from "../lib/vitest/hooks.mjs";
+
+export function popupLoadingTest(label, options = {}) {
+  describe(`Popup with Loading (${label})`, () => {
+    it("should accept cookies and wait for completion", async (context) => {
+      const testdriver = TestDriver(context, {
+        preview: 'ide',
+        ip: context.ip || process.env.TD_IP,
+        ...options,
+      });
+
+      await testdriver.provision.chrome({
+        url: "https://v0-popup-with-loading-bar.vercel.app/",
+      });
+      await testdriver.screenshot();
+
+      // Accept the cookie banner to trigger the loading process
+      let acceptButton = await testdriver.find("Accept All button on the cookie banner", {timeout: 10000});
+
+      if (await acceptButton.found()) {
+        await acceptButton.click();
+      } else {
+        console.log('no cookie banner found, proceeding without accepting cookies');
+      }
+
+      await testdriver.find('Start button').click();
+
+      // Wait for "All done!" to appear with 120s timeout
+      const allDone = await testdriver.find("All done! text or heading in a modal or popup", { timeout: 120000 });
+      await testdriver.screenshot();
+
+      const result = await testdriver.assert("The text 'All done!' is visible on the page");
+      expect(result).toBeTruthy();
+
+      // Click Continue to proceed to the image grid
+      await testdriver.find("Continue button in the modal").click();
+
+      // Wait for the 5x5 grid of images to fully load (up to 60s) and click the rocket
+      await testdriver.find("The icon of a rocket in the 5x5 grid of images", { timeout: 60000, zoom: true }).click();
+
+      // Assert the success message appears
+      const rocketResult = await testdriver.assert("The text 'You found the rocket!' is visible on the page");
+      expect(rocketResult).toBeTruthy();
+    });
+  });
+}

package/manual/no-provision.test.mjs

@@ -0,0 +1,31 @@
+/**
+ * TestDriver SDK - No-Provision Test with Dashcam (Vitest)
+ * 
+ * Demonstrates manual dashcam control without using provision methods.
+ * When not using provision.chrome(), provision.vscode(), etc., you need
+ * to manually start and stop dashcam recording.
+ */
+
+import { describe, it } from "vitest";
+import { TestDriver } from "../lib/vitest/hooks.mjs";
+import { getDefaults } from "./config.mjs";
+
+describe("No-Provision with Dashcam", () => {
+  it("should record dashcam while asserting desktop is visible", async (context) => {
+    const testdriver = TestDriver(context, { ...getDefaults(context) });
+
+    // Start dashcam recording manually (provision methods do this automatically)
+    await testdriver.dashcam.start();
+    
+    await testdriver.exec('sh', 'gedit >/dev/null 2>&1 &'); // Example command to keep the test running for a bit
+
+    await testdriver.assert('untitled document is visible');
+
+    // Stop dashcam and get the recording URL
+    const dashcamUrl = await testdriver.dashcam.stop();
+    if (dashcamUrl) {
+      console.log(`🎥 Dashcam recording: ${dashcamUrl}`);
+    }
+  });
+});
+

package/manual/packer-hover-image.test.mjs

@@ -0,0 +1,176 @@
+/**
+ * Packer AMI Integration Test
+ *
+ * Builds a fresh AMI via `packer build`, then runs the hover-image test suite
+ * against it. This is an end-to-end validation that a newly built runner image
+ * can provision a sandbox, execute commands, and interact with a browser.
+ *
+ * Usage:
+ *   TD_API_ROOT=http://localhost:1337 \
+ *   TD_API_KEY=<key> \
+ *   npx vitest run examples/packer-hover-image.test.mjs
+ *
+ * The packer build takes ~25 minutes, and the hover-image test ~5-10 minutes,
+ * so the overall test timeout is set to 60 minutes.
+ *
+ * Set TD_SANDBOX_AMI to skip the packer build and use an existing AMI.
+ */
+
+import { describe, expect, it, beforeAll, afterAll } from "vitest";
+import { TestDriver } from "../lib/vitest/hooks.mjs";
+import { getDefaults } from "../examples/config.mjs";
+import { execSync } from "child_process";
+import path from "path";
+import { fileURLToPath } from "url";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const PACKER_DIR = path.resolve(__dirname, "../../runner/packer");
+
+// 60 minute timeout for the entire suite (packer build + test)
+const SUITE_TIMEOUT = 60 * 60 * 1000;
+
+/**
+ * Build a fresh AMI using packer.
+ * Returns the AMI ID string (e.g. "ami-0337d8cd7cff854a4").
+ */
+function buildAmi() {
+  console.log("[packer] Starting AMI build — this takes ~25 minutes...");
+  const startTime = Date.now();
+
+  // packer build outputs "us-east-2: ami-XXXX" on the last relevant line
+  // Use -machine-readable for reliable parsing
+  const output = execSync("packer build -machine-readable .", {
+    cwd: PACKER_DIR,
+    encoding: "utf-8",
+    timeout: 45 * 60 * 1000, // 45 minute hard limit on packer
+    stdio: ["ignore", "pipe", "pipe"],
+    env: { ...process.env },
+  });
+
+  // Machine-readable output has lines like:
+  //   timestamp,target,type,data
+  //   ...,artifact,0,id,us-east-2:ami-XXXXXXXXXXXX
+  const amiMatch = output.match(/,artifact,\d+,id,[\w-]+:(ami-[a-f0-9]+)/);
+  if (!amiMatch) {
+    // Fallback: try human-readable format
+    const humanMatch = output.match(/[\w-]+:\s*(ami-[a-f0-9]+)/);
+    if (!humanMatch) {
+      console.error("[packer] Build output (last 2000 chars):", output.slice(-2000));
+      throw new Error("Failed to extract AMI ID from packer build output");
+    }
+    const elapsed = ((Date.now() - startTime) / 1000 / 60).toFixed(1);
+    console.log(`[packer] AMI built: ${humanMatch[1]} (${elapsed} min)`);
+    return humanMatch[1];
+  }
+
+  const amiId = amiMatch[1];
+  const elapsed = ((Date.now() - startTime) / 1000 / 60).toFixed(1);
+  console.log(`[packer] AMI built: ${amiId} (${elapsed} min)`);
+  return amiId;
+}
+
+/**
+ * Deregister an AMI and delete its backing snapshot.
+ * Best-effort — failures are logged but don't fail the test.
+ */
+function cleanupAmi(amiId) {
+  if (!amiId) return;
+  try {
+    console.log(`[cleanup] Deregistering AMI ${amiId}...`);
+    // Get snapshot IDs before deregistering
+    const describeOutput = execSync(
+      `aws ec2 describe-images --image-ids ${amiId} --query "Images[0].BlockDeviceMappings[*].Ebs.SnapshotId" --output text`,
+      { encoding: "utf-8", timeout: 15000 },
+    ).trim();
+
+    execSync(`aws ec2 deregister-image --image-id ${amiId}`, {
+      timeout: 15000,
+    });
+
+    // Delete backing snapshots
+    if (describeOutput && describeOutput !== "None") {
+      for (const snapId of describeOutput.split(/\s+/)) {
+        if (snapId.startsWith("snap-")) {
+          console.log(`[cleanup] Deleting snapshot ${snapId}...`);
+          execSync(`aws ec2 delete-snapshot --snapshot-id ${snapId}`, {
+            timeout: 15000,
+          });
+        }
+      }
+    }
+    console.log(`[cleanup] AMI ${amiId} cleaned up`);
+  } catch (err) {
+    console.warn(`[cleanup] Failed to clean up AMI ${amiId}: ${err.message}`);
+  }
+}
+
+// ── Hover-Image login helper ─────────────────────────────────────────────
+
+async function performLogin(client, username = "standard_user") {
+  await client.focusApplication("Google Chrome");
+  const password = await client.extract("the password");
+  const usernameField = await client.find("username input");
+  await usernameField.click();
+  await client.type(username);
+  await client.pressKeys(["tab"]);
+  await client.type(password, { secret: true });
+  await client.pressKeys(["tab"]);
+  await client.pressKeys(["enter"]);
+}
+
+// ── Test Suite ───────────────────────────────────────────────────────────
+
+describe("Packer AMI → Hover Image", { timeout: SUITE_TIMEOUT }, () => {
+  let amiId;
+  let builtAmi = false;
+
+  beforeAll(() => {
+    // If TD_SANDBOX_AMI is set, skip the packer build
+    if (process.env.TD_SANDBOX_AMI) {
+      amiId = process.env.TD_SANDBOX_AMI;
+      console.log(`[packer] Using existing AMI: ${amiId}`);
+    } else {
+      amiId = buildAmi();
+      builtAmi = true;
+    }
+  });
+
+  afterAll(() => {
+    // Only clean up AMIs we built (don't delete pre-existing ones)
+    if (builtAmi && amiId && process.env.TD_PACKER_CLEANUP !== "false") {
+      cleanupAmi(amiId);
+    }
+  });
+
+  it(
+    "should click on shopping cart icon and verify empty cart",
+    { timeout: SUITE_TIMEOUT },
+    async (context) => {
+      const testdriver = TestDriver(context, {
+        ...getDefaults(context),
+        os: "windows",
+        sandboxAmi: amiId,
+      });
+
+      // Provision Chrome on the freshly-built AMI
+      await testdriver.provision.chrome({
+        url: "http://testdriver-sandbox.vercel.app/login",
+      });
+
+      // Perform login
+      await performLogin(testdriver);
+
+      // Click on the shopping cart icon
+      await testdriver.focusApplication("Google Chrome");
+      const cartIcon = await testdriver.find(
+        "shopping cart icon next to the Cart text in the top right corner",
+      );
+      await cartIcon.click();
+
+      // Assert that you see an empty shopping cart
+      const result = await testdriver.assert("Your cart is empty");
+      expect(result).toBeTruthy();
+    },
+  );
+});

package/manual/scroll-until-text.test.mjs

@@ -0,0 +1,68 @@
+/**
+ * TestDriver SDK - Scroll Until Text Test (Vitest)
+ * Converted from: testdriver/acceptance/scroll-until-text.yaml
+ */
+
+import { describe, expect, it } from "vitest";
+import { TestDriver } from "../lib/vitest/hooks.mjs";
+import { getDefaults } from "./config.mjs";
+
+/**
+ * Perform login flow for SauceLabs demo app
+ * @param {TestDriver} client - TestDriver client
+ * @param {string} username - Username (default: 'standard_user')
+ */
+async function performLogin(client, username = "standard_user") {
+  await client.focusApplication("Google Chrome");
+  const password = await client.extract("the password");
+  const usernameField = await client.find(
+    "username input",
+  );
+  await usernameField.click();
+  await client.type(username);
+  await client.pressKeys(["tab"]);
+  await client.type(password, { secret: true });
+  await client.pressKeys(["tab"]);
+  await client.pressKeys(["enter"]);
+}
+
+describe("Scroll Until Text Test", () => {
+  it('should scroll until "testdriver socks" appears', async (context) => {
+    const testdriver = TestDriver(context, { ...getDefaults(context), headless: true });
+    await testdriver.provision.chrome({ url: 'http://testdriver-sandbox.vercel.app/login' });
+
+    //
+    // Perform login first
+    await performLogin(testdriver);
+
+    // Scroll until text appears
+    await testdriver.focusApplication("Google Chrome");
+
+    await testdriver.find('TestDriver.ai Sandbox heading').click();
+
+    // Scroll until text appears
+    let found = false;
+    let scrollCount = 0;
+    const maxScrolls = 10;
+    
+    while (!found && scrollCount < maxScrolls) {
+      const findResult = await testdriver.find("testdriver socks product text is fully visible");
+
+      if (findResult.found()) {
+        found = true;
+      } else {
+        await testdriver.scroll();
+        scrollCount++;
+      }
+    }
+    
+    if (!found) {
+      throw new Error(`Failed to find "testdriver socks" after ${maxScrolls} scrolls`);
+    }
+
+    // Assert testdriver socks appears on screen
+    await testdriver.focusApplication("Google Chrome");
+    const result = await testdriver.assert("TestDriver Socks appears on screen");
+    expect(result).toBeTruthy();
+  });
+});

package/manual/test-init-command.js

@@ -0,0 +1,223 @@
+#!/usr/bin/env node
+
+/**
+ * Test Init Command
+ * 
+ * This script tests the `testdriverai init` command by:
+ * 1. Creating a temporary test project
+ * 2. Running the init command
+ * 3. Verifying all files were created correctly
+ * 4. Running the generated test
+ * 5. Cleaning up
+ * 
+ * Usage:
+ *   node manual/test-init-command.js
+ * 
+ * Requirements:
+ *   - TD_API_KEY environment variable must be set
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { execSync } = require('child_process');
+const os = require('os');
+
+// Colors for terminal output
+const colors = {
+  reset: '\x1b[0m',
+  green: '\x1b[32m',
+  red: '\x1b[31m',
+  cyan: '\x1b[36m',
+  yellow: '\x1b[33m',
+  gray: '\x1b[90m',
+};
+
+function log(message, color = 'reset') {
+  console.log(`${colors[color]}${message}${colors.reset}`);
+}
+
+function success(message) {
+  log(`✓ ${message}`, 'green');
+}
+
+function error(message) {
+  log(`✗ ${message}`, 'red');
+}
+
+function info(message) {
+  log(message, 'cyan');
+}
+
+function step(message) {
+  log(`\n${message}`, 'cyan');
+}
+
+// Main test function
+async function testInitCommand() {
+  // Check for API key
+  if (!process.env.TD_API_KEY) {
+    error('TD_API_KEY environment variable is required');
+    process.exit(1);
+  }
+
+  const testDir = path.join(os.tmpdir(), `test-init-${Date.now()}`);
+  const cliPath = path.join(__dirname, '..');
+  
+  try {
+    step('📦 Creating test directory...');
+    fs.mkdirSync(testDir, { recursive: true });
+    success(`Created: ${testDir}`);
+
+    step('🔧 Setting up .env file...');
+    const envContent = `TD_API_KEY=${process.env.TD_API_KEY}\n`;
+    fs.writeFileSync(path.join(testDir, '.env'), envContent);
+    success('Created .env with API key');
+
+    step('🚀 Running init command...');
+    try {
+      execSync(`node ${path.join(cliPath, 'bin/testdriverai.js')} init`, {
+        cwd: testDir,
+        stdio: 'inherit',
+        env: { ...process.env, TD_API_KEY: process.env.TD_API_KEY }
+      });
+      success('Init command completed');
+    } catch (err) {
+      error('Init command failed');
+      throw err;
+    }
+
+    step('🔍 Verifying project structure...');
+    
+    const expectedFiles = [
+      'package.json',
+      'vitest.config.js',
+      'tests/example.test.js',
+      'tests/login.js',
+      '.env',
+      '.gitignore',
+      '.github/workflows/testdriver.yml'
+    ];
+
+    for (const file of expectedFiles) {
+      const filePath = path.join(testDir, file);
+      if (!fs.existsSync(filePath)) {
+        error(`Missing file: ${file}`);
+        throw new Error(`Expected file not found: ${file}`);
+      }
+      success(`Found: ${file}`);
+    }
+
+    step('📝 Verifying vitest.config.js contents...');
+    const vitestConfig = fs.readFileSync(path.join(testDir, 'vitest.config.js'), 'utf8');
+    
+    if (!vitestConfig.includes('TestDriver()')) {
+      error('TestDriver reporter not found in vitest.config.js');
+      console.log(vitestConfig);
+      throw new Error('TestDriver reporter not configured');
+    }
+    success('TestDriver reporter is configured');
+
+    if (!vitestConfig.includes('setupFiles') || !vitestConfig.includes('testdriverai/vitest/setup')) {
+      error('setupFiles not configured correctly');
+      console.log(vitestConfig);
+      throw new Error('setupFiles not configured');
+    }
+    success('setupFiles is configured correctly');
+
+    if (!vitestConfig.includes('reporters')) {
+      error('reporters array not found');
+      console.log(vitestConfig);
+      throw new Error('reporters not configured');
+    }
+    success('reporters array includes TestDriver');
+
+    step('📝 Verifying example test contents...');
+    const testFile = fs.readFileSync(path.join(testDir, 'tests/example.test.js'), 'utf8');
+    
+    if (!testFile.includes('.provision.chrome')) {
+      error('Test does not use .provision.chrome');
+      console.log(testFile);
+      throw new Error('Test does not use .provision pattern');
+    }
+    success('Test uses .provision.chrome');
+
+    if (!testFile.includes("from 'testdriverai/vitest/hooks'")) {
+      error('Test does not import from testdriverai/vitest/hooks');
+      console.log(testFile);
+      throw new Error('Test does not import TestDriver from vitest/hooks');
+    }
+    success('Test imports TestDriver from vitest/hooks');
+
+    if (!testFile.includes("from './login.js'")) {
+      error('Test does not import login from ./login.js');
+      console.log(testFile);
+      throw new Error('Test does not import login snippet');
+    }
+    success('Test imports login snippet');
+
+    step('📝 Verifying login snippet contents...');
+    const loginFile = fs.readFileSync(path.join(testDir, 'tests/login.js'), 'utf8');
+
+    if (!loginFile.includes('.extract(')) {
+      error('Login snippet does not use .extract()');
+      console.log(loginFile);
+      throw new Error('Login snippet does not demonstrate .extract()');
+    }
+    success('Login snippet demonstrates .extract()');
+
+    if (!loginFile.includes('secret: true')) {
+      error('Login snippet does not use secret option');
+      console.log(loginFile);
+      throw new Error('Login snippet does not demonstrate secret typing');
+    }
+    success('Login snippet demonstrates secret typing');
+
+    step('🧪 Running generated test...');
+    try {
+      execSync('npm test', {
+        cwd: testDir,
+        stdio: 'inherit',
+        env: { ...process.env, TD_API_KEY: process.env.TD_API_KEY }
+      });
+      success('Test execution completed successfully');
+    } catch (err) {
+      error('Test execution failed');
+      throw err;
+    }
+
+    step('✅ All checks passed!');
+    log('\nTest summary:', 'green');
+    log('  • Init command executed successfully', 'green');
+    log('  • All expected files created', 'green');
+    log('  • Configuration files are correct', 'green');
+    log('  • Example test has proper patterns', 'green');
+    log('  • Generated test runs successfully', 'green');
+
+  } catch (err) {
+    step('❌ Test failed!');
+    error(err.message);
+    if (err.stack) {
+      log(err.stack, 'gray');
+    }
+    process.exit(1);
+  } finally {
+    // Optional: Clean up test directory
+    // Commented out so you can inspect the generated files
+    // step('🧹 Cleaning up...');
+    // if (fs.existsSync(testDir)) {
+    //   fs.rmSync(testDir, { recursive: true, force: true });
+    //   success('Cleaned up test directory');
+    // }
+    
+    info(`\nTest project preserved at: ${testDir}`);
+    info('To clean up manually, run:');
+    log(`  rm -rf ${testDir}`, 'gray');
+  }
+}
+
+// Run the test
+testInitCommand().catch(err => {
+  error('Unexpected error:');
+  console.error(err);
+  process.exit(1);
+});