@testdriverai/agent 7.8.0-canary.15 → 7.8.0-canary.16

package/lib/vitest/hooks.mjs

@@ -403,6 +403,11 @@ export function TestDriver(context, options = {}) {
     config.apiRoot = process.env.TD_API_ROOT;
   }
 
+  // Use TD_E2B_TEMPLATE_ID from environment if not provided in config
+  if (!config.e2bTemplateId && process.env.TD_E2B_TEMPLATE_ID) {
+    config.e2bTemplateId = process.env.TD_E2B_TEMPLATE_ID;
+  }
+
   const testdriver = new TestDriverSDK(apiKey, config);
   testdriver.__vitestContext = context.task;
   testdriver._debugOnFailure = mergedOptions.debugOnFailure || false;
@@ -644,6 +649,61 @@ export function TestDriver(context, options = {}) {
       // Clean up console spies
       cleanupConsoleSpy(currentInstance);
 
+      // Build test result metadata for JSON report output
+      {
+        const sdkPkg = require("../../package.json");
+        const inst = currentInstance.getInstance?.() || {};
+        const sbx = currentInstance.sandbox || {};
+        const apiRoot = currentInstance.config?.TD_API_ROOT || null;
+
+        context.task.meta.testResult = {
+          // Versions
+          sdkVersion: sdkPkg.version || null,
+          apiVersion: currentInstance._apiVersion || null,
+          runnerVersionBefore: inst.runnerVersionBefore || null,
+          runnerVersionAfter: inst.runnerVersionAfter || null,
+          wasUpdated: inst.wasUpdated || false,
+
+          // URLs
+          apiUrl: apiRoot,
+          vncUrl: inst.vncUrl || inst.url || null,
+
+          // Dates
+          date: new Date().toISOString(),
+
+          // Team / session
+          teamId: sbx._teamId || null,
+          sessionId: currentInstance.getSessionId?.() || null,
+
+          // Test info
+          testFile: context.task.meta.testFile || null,
+          testName: context.task.name || null,
+          suiteName: context.task.suite?.name || null,
+
+          // Test result
+          testPassed: context.task.result?.state === "pass",
+          error: context.task.result?.errors?.[0]?.message || null,
+          errorStack: context.task.result?.errors?.[0]?.stack || null,
+
+          // Infrastructure
+          sandboxId: inst.sandboxId || inst.instanceId || null,
+          instanceId: inst.instanceId || null,
+          os: currentInstance.os || inst.os || null,
+          amiId: inst.amiId || null,
+          e2bTemplateId: inst.e2bTemplateId || null,
+          imageVersion: inst.imageVersion || null,
+
+          // Realtime
+          realtimeChannel: inst.channelName || sbx._channelName || null,
+          realtimeMessageCount: typeof sbx.getPublishCount === "function" ? sbx.getPublishCount() : 0,
+
+          // Interactions
+          interactions: currentInstance._interactionStats
+            ? { ...currentInstance._interactionStats, byType: { ...currentInstance._interactionStats.byType } }
+            : { total: 0, cached: 0, byType: {} },
+        };
+      }
+
       // Wait for connection to finish if it was initiated
       if (currentInstance.__connectionPromise) {
         await currentInstance.__connectionPromise.catch(() => { }); // Ignore connection errors during cleanup

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@testdriverai/agent",
-  "version": "7.8.0-canary.15",
+  "version": "7.8.0-canary.16",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "sdk.js",
   "types": "sdk.d.ts",

package/sdk.d.ts

@@ -273,6 +273,8 @@ export interface TestDriverOptions {
   sandboxAmi?: string;
   /** EC2 instance type for sandbox (e.g., 'i3.metal') */
   sandboxInstance?: string;
+  /** E2B template ID to use when creating the sandbox (e.g., 'my-template-id') */
+  e2bTemplateId?: string;
   /** Cache key for element finding operations. If provided, enables caching tied to this key */
   cacheKey?: string;
   /** Reconnect to the last used sandbox instead of creating a new one. When true, provision methods (chrome, vscode, installer, etc.) will be skipped since the application is already running. Throws error if no previous sandbox exists. */
@@ -327,6 +329,8 @@ export interface ConnectOptions {
   sandboxAmi?: string;
   /** EC2 instance type for sandbox (e.g., 'i3.metal') */
   sandboxInstance?: string;
+  /** E2B template ID to use when creating the sandbox (e.g., 'my-template-id') */
+  e2bTemplateId?: string;
   /** Operating system for the sandbox (default: 'linux') */
   os?: "windows" | "linux";
   /**

package/sdk.js

@@ -481,7 +481,7 @@ class Element {
       let cacheKey = null;
       let cacheThreshold = null;
       let perCommandThresholds = null; // Per-command { screen, element } override
-      let zoom = false; // Default to disabled, enable with zoom: true
+      let zoom = true; // Default to enabled
       let perCommandAi = null; // Per-command AI config override
 
       let minConfidence = null; // Minimum confidence threshold
@@ -494,8 +494,8 @@ class Element {
         // New: options is an object with cacheKey and/or cacheThreshold
         cacheKey = options.cacheKey || null;
         cacheThreshold = options.cacheThreshold ?? null;
-        // zoom defaults to false unless explicitly set to true
-        zoom = options.zoom === true;
+        // zoom defaults to true unless explicitly set to false
+        zoom = options.zoom !== false;
         // Minimum confidence threshold: fail find if AI confidence is below this value
         minConfidence = options.confidence ?? null;
         // Element type hint for prompt wrapping
@@ -568,7 +568,7 @@ class Element {
         cacheKey: cacheKey,
         os: this.sdk.os,
         resolution: this.sdk.resolution,
-        zoom: zoom,
+        zoom: zoom === true ? 1 : zoom === false ? 0 : zoom,
         confidence: minConfidence,
         type: elementType,
         ai: {
@@ -623,6 +623,11 @@ class Element {
 
     // Track find interaction once at the end (fire-and-forget, don't block)
     const sessionId = this.sdk.getSessionId();
+    const findCacheHit = response?.cacheHit || response?.cache_hit || response?.cached || false;
+    // Increment local interaction counters
+    this.sdk._interactionStats.total++;
+    this.sdk._interactionStats.byType.find = (this.sdk._interactionStats.byType.find || 0) + 1;
+    if (findCacheHit) this.sdk._interactionStats.cached++;
     if (sessionId && this.sdk.apiClient) {
       this.sdk.apiClient
         .req("interaction/track", {
@@ -632,11 +637,7 @@ class Element {
           timestamp: absoluteTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
           success: this._found,
           error: findError,
-          cacheHit:
-            response?.cacheHit ||
-            response?.cache_hit ||
-            response?.cached ||
-            false,
+          cacheHit: findCacheHit,
           selector: response?.selector,
           selectorUsed: !!response?.selector,
           confidence: response?.confidence ?? null,
@@ -1498,6 +1499,7 @@ class TestDriverSDK {
     // Store sandbox configuration options
     this.sandboxAmi = options.sandboxAmi || null;
     this.sandboxInstance = options.sandboxInstance || null;
+    this.e2bTemplateId = options.e2bTemplateId || null;
 
     // Store reconnect preference from options
     this.reconnect =
@@ -1616,6 +1618,12 @@ class TestDriverSDK {
     // Uploaded to S3 at cleanup so they can be displayed alongside dashcam replays.
     this._logBuffer = [];
 
+    // API version discovered by _logEnvironmentInfo()
+    this._apiVersion = null;
+
+    // Local interaction counters — incremented at each interaction/track call site
+    this._interactionStats = { total: 0, cached: 0, byType: {} };
+
     // Set up event listeners once (they live for the lifetime of the SDK instance)
     this._setupLogging();
 
@@ -2716,6 +2724,7 @@ CAPTCHA_SOLVER_EOF`,
    * @param {string} options.ip - Direct IP address to connect to
    * @param {string} options.sandboxAmi - AMI to use for the sandbox
    * @param {string} options.sandboxInstance - Instance type for the sandbox
+   * @param {string} options.e2bTemplateId - E2B template ID to use when creating the sandbox
    * @param {string} options.os - Operating system for the sandbox (windows or linux)
    * @param {boolean} options.reuseConnection - Reuse recent connection if available (default: true)
    * @returns {Promise<Object>} Sandbox instance details
@@ -2804,6 +2813,12 @@ CAPTCHA_SOLVER_EOF`,
     } else if (this.sandboxInstance) {
       this.agent.sandboxInstance = this.sandboxInstance;
     }
+    // Use e2bTemplateId from connectOptions if provided, otherwise fall back to constructor value
+    if (connectOptions.e2bTemplateId !== undefined) {
+      this.agent.e2bTemplateId = connectOptions.e2bTemplateId;
+    } else if (this.e2bTemplateId) {
+      this.agent.e2bTemplateId = this.e2bTemplateId;
+    }
     // Use os from connectOptions if provided, otherwise fall back to this.os
     if (connectOptions.os !== undefined) {
       this.agent.sandboxOs = connectOptions.os;
@@ -3193,6 +3208,11 @@ CAPTCHA_SOLVER_EOF`,
 
         // Track successful findAll interaction (fire-and-forget, don't block)
         const sessionId = this.getSessionId();
+        const findAllCacheHit = response.cached || false;
+        // Increment local interaction counters
+        this._interactionStats.total++;
+        this._interactionStats.byType.findAll = (this._interactionStats.byType.findAll || 0) + 1;
+        if (findAllCacheHit) this._interactionStats.cached++;
         if (sessionId && this.apiClient) {
           this.apiClient
             .req("interaction/track", {
@@ -3202,7 +3222,7 @@ CAPTCHA_SOLVER_EOF`,
               timestamp: absoluteTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
               success: true,
               input: { count: elements.length },
-              cacheHit: response.cached || false,
+              cacheHit: findAllCacheHit,
               selector: response.selector,
               selectorUsed: !!response.selector,
               screenshotUrl: response.screenshotKey ?? null,
@@ -3248,6 +3268,11 @@ CAPTCHA_SOLVER_EOF`,
 
         // No elements found - track interaction (fire-and-forget, don't block)
         const sessionId = this.getSessionId();
+        const noResultCacheHit = response?.cached || false;
+        // Increment local interaction counters
+        this._interactionStats.total++;
+        this._interactionStats.byType.findAll = (this._interactionStats.byType.findAll || 0) + 1;
+        if (noResultCacheHit) this._interactionStats.cached++;
         if (sessionId && this.apiClient) {
           this.apiClient
             .req("interaction/track", {
@@ -3258,7 +3283,7 @@ CAPTCHA_SOLVER_EOF`,
               success: false,
               error: "No elements found",
               input: { count: 0 },
-              cacheHit: response?.cached || false,
+              cacheHit: noResultCacheHit,
               selector: response?.selector,
               selectorUsed: !!response?.selector,
               screenshotUrl: response?.screenshotKey ?? null,
@@ -3292,6 +3317,9 @@ CAPTCHA_SOLVER_EOF`,
 
       // Track findAll error interaction (fire-and-forget, don't block)
       const sessionId = this.getSessionId();
+      // Increment local interaction counters
+      this._interactionStats.total++;
+      this._interactionStats.byType.findAll = (this._interactionStats.byType.findAll || 0) + 1;
       if (sessionId && this.apiClient) {
         this.apiClient
           .req("interaction/track", {
@@ -3843,6 +3871,8 @@ CAPTCHA_SOLVER_EOF`,
       res.on('end', () => {
         try {
           const info = JSON.parse(data);
+          // Persist API version for test result metadata
+          this._apiVersion = info.version || null;
           const commit = info.commit || 'unknown';
           const shortCommit = commit.substring(0, 7);
           const commitUrl = commit !== 'unknown'

package/setup/aws/spawn-runner.sh

@@ -237,24 +237,55 @@ PARAMS_EOF
     --output json)
   rm -f "$PARAMS_FILE"
 else
-  echo "Installing @testdriverai/runner@${RUNNER_VERSION} via npm..."
+  echo "Installing @testdriverai/runner@${RUNNER_VERSION} via npm pack + extract..."
+
+  # Build SSM parameters JSON in a temp file (same approach as dev mode)
+  PARAMS_FILE=$(mktemp)
+  cat > "$PARAMS_FILE" << PARAMS_EOF
+{
+  "commands": [
+    "Write-Host '=== Starting runner install (npm pack) ==='",
+    "Write-Host 'Stopping existing runner processes...'",
+    "Stop-ScheduledTask -TaskName RunTestDriverAgent -ErrorAction SilentlyContinue",
+    "Stop-Process -Name node -Force -ErrorAction SilentlyContinue",
+    "Start-Sleep -Seconds 2",
+    "Write-Host 'Current runner version:'",
+    "Get-Content 'C:\\\\testdriver\\\\sandbox-agent\\\\package.json' | ConvertFrom-Json | Select-Object -ExpandProperty version",
+    "Set-Location 'C:\\\\Windows\\\\Temp'",
+    "Write-Host 'Downloading @testdriverai/runner@${RUNNER_VERSION} via npm pack...'",
+    "npm pack @testdriverai/runner@${RUNNER_VERSION} 2>&1 | Write-Host",
+    "\$tarball = (Get-ChildItem 'C:\\\\Windows\\\\Temp\\\\testdriverai-runner-*.tgz' | Sort-Object LastWriteTime -Descending | Select-Object -First 1).FullName",
+    "Write-Host \"Downloaded tarball: \$tarball\"",
+    "Write-Host 'Extracting runner...'",
+    "tar -xzf \$tarball -C 'C:\\\\Windows\\\\Temp'",
+    "Write-Host 'New runner version in package:'",
+    "Get-Content 'C:\\\\Windows\\\\Temp\\\\package\\\\package.json' | ConvertFrom-Json | Select-Object -ExpandProperty version",
+    "Write-Host 'Clearing old lib folder...'",
+    "Remove-Item 'C:\\\\testdriver\\\\sandbox-agent\\\\lib' -Recurse -Force -ErrorAction SilentlyContinue",
+    "Write-Host 'Copying files to sandbox-agent...'",
+    "xcopy 'C:\\\\Windows\\\\Temp\\\\package\\\\*' 'C:\\\\testdriver\\\\sandbox-agent\\\\' /E /Y /I",
+    "Write-Host 'Runner version after copy:'",
+    "Get-Content 'C:\\\\testdriver\\\\sandbox-agent\\\\package.json' | ConvertFrom-Json | Select-Object -ExpandProperty version",
+    "Remove-Item 'C:\\\\Windows\\\\Temp\\\\package' -Recurse -Force -ErrorAction SilentlyContinue",
+    "Remove-Item \$tarball -Force -ErrorAction SilentlyContinue",
+    "Set-Location 'C:\\\\testdriver\\\\sandbox-agent'",
+    "Write-Host 'Installing npm dependencies...'",
+    "npm install --omit=dev 2>&1 | Write-Host",
+    "Write-Host 'Restarting RunTestDriverAgent scheduled task...'",
+    "Start-ScheduledTask -TaskName RunTestDriverAgent -ErrorAction SilentlyContinue",
+    "Write-Host '=== Runner install complete (npm pack) ==='"
+  ]
+}
+PARAMS_EOF
+
   INSTALL_CMD=$(aws ssm send-command \
     --region "$AWS_REGION" \
     --instance-ids "$INSTANCE_ID" \
     --document-name "AWS-RunPowerShellScript" \
-    --parameters "commands=[
-      \"Set-Location 'C:\\\\testdriver\\\\sandbox-agent'\",
-      \"Write-Host 'Installing @testdriverai/runner@${RUNNER_VERSION}...'\",
-      \"npm install @testdriverai/runner@${RUNNER_VERSION} --omit=dev 2>&1 | Write-Host\",
-      \"Write-Host 'Stopping runner (config not yet provisioned)...'\",
-      \"Stop-Process -Name node -Force -ErrorAction SilentlyContinue\",
-      \"Stop-ScheduledTask -TaskName RunTestDriverAgent -ErrorAction SilentlyContinue\",
-      \"Start-Sleep -Seconds 2\",
-      \"Start-ScheduledTask -TaskName RunTestDriverAgent -ErrorAction SilentlyContinue\",
-      \"Write-Host 'Runner install complete'\"
-    ]" \
-    --timeout-seconds 120 \
+    --parameters "file://$PARAMS_FILE" \
+    --timeout-seconds 180 \
     --output json)
+  rm -f "$PARAMS_FILE"
 fi
 
 INSTALL_CMD_ID=$(jq -r '.Command.CommandId' <<<"$INSTALL_CMD")