@browserbasehq/stagehand 3.0.8-alpha-3d5af07f66d6d26d1f5ac4bd9be7183c3381dd92 → 3.0.8-alpha-16d72fb4c4081dd33bf45605d75c27644ea4c00e

package/dist/index.d.ts

@@ -3403,6 +3403,63 @@ interface StreamingAgentInstance {
 interface NonStreamingAgentInstance {
     execute: (instructionOrOptions: string | AgentExecuteOptions) => Promise<AgentResult>;
 }
+/**
+ * Content item type for toModelOutput return values.
+ * Used in tool definitions to return text and/or media to the model.
+ */
+type ModelOutputContentItem = {
+    type: "text";
+    text: string;
+} | {
+    type: "media";
+    mediaType: string;
+    data: string;
+};
+interface ClickToolResult {
+    success: boolean;
+    describe?: string;
+    coordinates?: number[];
+    error?: string;
+    screenshotBase64?: string;
+}
+interface TypeToolResult {
+    success: boolean;
+    describe?: string;
+    text?: string;
+    error?: string;
+    screenshotBase64?: string;
+}
+interface DragAndDropToolResult {
+    success: boolean;
+    describe?: string;
+    error?: string;
+    screenshotBase64?: string;
+}
+interface FillFormField {
+    action: string;
+    value: string;
+    coordinates: {
+        x: number;
+        y: number;
+    };
+}
+interface FillFormVisionToolResult {
+    success: boolean;
+    playwrightArguments?: FillFormField[];
+    error?: string;
+    screenshotBase64?: string;
+}
+interface ScrollVisionToolResult {
+    success: boolean;
+    message: string;
+    scrolledPixels: number;
+    screenshotBase64?: string;
+}
+interface WaitToolResult {
+    success: boolean;
+    waited: number;
+    screenshotBase64?: string;
+}
 
 type OpenAIClientOptions = Pick<ClientOptions$1, "baseURL" | "apiKey">;
 type AnthropicClientOptions = Pick<ClientOptions$2, "baseURL" | "apiKey">;
@@ -3815,12 +3872,9 @@ declare const screenshotTool: (v3: V3) => ai.Tool<Record<string, never>, {
     pageUrl: string;
 }>;
 
-declare const waitTool: (v3: V3) => ai.Tool<{
+declare const waitTool: (v3: V3, mode?: AgentToolMode) => ai.Tool<{
     timeMs: number;
-}, {
-    success: boolean;
-    waited: number;
-}>;
+}, WaitToolResult>;
 
 declare const navBackTool: (v3: V3) => ai.Tool<{
     reasoningText: string;
@@ -3873,11 +3927,7 @@ declare const scrollVisionTool: (v3: V3, provider?: string) => ai.Tool<{
     direction: "up" | "down";
     coordinates?: number[];
     percentage?: number;
-}, {
-    success: boolean;
-    message: string;
-    scrolledPixels: number;
-}>;
+}, ScrollVisionToolResult>;
 
 declare const extractTool: (v3: V3, executionModel?: string, logger?: (message: LogLine) => void) => ai.Tool<{
     instruction: string;
@@ -3895,47 +3945,19 @@ declare const extractTool: (v3: V3, executionModel?: string, logger?: (message:
 declare const clickTool: (v3: V3, provider?: string) => ai.Tool<{
     describe: string;
     coordinates: number[];
-}, {
-    success: boolean;
-    describe: string;
-    coordinates: number[];
-    error?: undefined;
-} | {
-    success: boolean;
-    error: string;
-    describe?: undefined;
-    coordinates?: undefined;
-}>;
+}, ClickToolResult>;
 
 declare const typeTool: (v3: V3, provider?: string) => ai.Tool<{
     describe: string;
     text: string;
     coordinates: number[];
-}, {
-    success: boolean;
-    describe: string;
-    text: string;
-    error?: undefined;
-} | {
-    success: boolean;
-    error: string;
-    describe?: undefined;
-    text?: undefined;
-}>;
+}, TypeToolResult>;
 
 declare const dragAndDropTool: (v3: V3, provider?: string) => ai.Tool<{
     describe: string;
     startCoordinates: number[];
     endCoordinates: number[];
-}, {
-    success: boolean;
-    describe: string;
-    error?: undefined;
-} | {
-    success: boolean;
-    error: string;
-    describe?: undefined;
-}>;
+}, DragAndDropToolResult>;
 
 declare const clickAndHoldTool: (v3: V3, provider?: string) => ai.Tool<{
     describe: string;
@@ -3984,22 +4006,7 @@ declare const fillFormVisionTool: (v3: V3, provider?: string) => ai.Tool<{
             y: number;
         };
     }[];
-}, {
-    success: boolean;
-    playwrightArguments: {
-        coordinates: {
-            x: number;
-            y: number;
-        };
-        action: string;
-        value: string;
-    }[];
-    error?: undefined;
-} | {
-    success: boolean;
-    error: string;
-    playwrightArguments?: undefined;
-}>;
+}, FillFormVisionToolResult>;
 
 declare const thinkTool: () => ai.Tool<{
     reasoning: string;
@@ -4187,4 +4194,4 @@ declare class V3Evaluator {
     private _evaluateWithMultipleScreenshots;
 }
 
-export { type AISDKCustomProvider, type AISDKProvider, AISdkClient, AVAILABLE_CUA_MODELS, type ActOptions, type ActResult, ActTimeoutError, type Action, type ActionExecutionResult, AgentAbortError, type AgentAction, type AgentCallbacks, type AgentConfig, type AgentContext, type AgentExecuteCallbacks, type AgentExecuteOptions, type AgentExecuteOptionsBase, type AgentExecutionOptions, type AgentHandlerOptions, type AgentInstance, type AgentModelConfig, AgentProvider, type AgentProviderType, type AgentResult, AgentScreenshotProviderError, type AgentState, type AgentStreamCallbacks, type AgentStreamExecuteOptions, type AgentStreamResult, type AgentToolCall, type AgentToolMode, type AgentToolResult, type AgentToolTypesMap, type AgentTools, type AgentType, type AgentUITools, AnnotatedScreenshotText, type AnthropicClientOptions, type AnthropicContentBlock, type AnthropicJsonSchemaObject, type AnthropicMessage, type AnthropicTextBlock, type AnthropicToolResult, type AnyPage, api as Api, type AvailableCuaModel, type AvailableModel, BrowserbaseSessionNotFoundError, CaptchaTimeoutError, type ChatCompletionOptions, type ChatMessage, type ChatMessageContent, type ChatMessageImageContent, type ChatMessageTextContent, type ClientOptions, type ComputerCallItem, ConnectionTimeoutError, type ConsoleListener, ConsoleMessage, ContentFrameNotFoundError, type CreateChatCompletionOptions, CreateChatCompletionResponseError, CuaModelRequiredError, ElementNotVisibleError, ExperimentalApiConflictError, ExperimentalNotConfiguredError, type ExtractOptions, type ExtractResult, ExtractTimeoutError, type FunctionCallItem, type GoogleServiceAccountCredentials, type GoogleVertexProviderSettings, HandlerNotInitializedError, type HistoryEntry, type InferStagehandSchema, InvalidAISDKModelFormatError, type JsonSchema, type JsonSchemaDocument, type JsonSchemaProperty, LLMClient, type LLMParsedResponse, type LLMResponse, LLMResponseError, type LLMTool, type LLMUsage, LOG_LEVEL_NAMES, type LoadState, type LocalBrowserLaunchOptions, type LogLevel, type LogLine, type Logger, MCPConnectionError, MissingEnvironmentVariableError, MissingLLMConfigurationError, type ModelConfiguration, type ModelProvider, type NonStreamingAgentInstance, type ObserveOptions, ObserveTimeoutError, type OpenAIClientOptions, Page, PageNotFoundError, Response$1 as Response, ResponseBodyError, type ResponseInputItem, type ResponseItem, ResponseParseError, type SafetyCheck, type SafetyConfirmationHandler, type SafetyConfirmationResponse, V3 as Stagehand, StagehandAPIError, StagehandAPIUnauthorizedError, StagehandClickError, StagehandClosedError, StagehandDefaultError, StagehandDomProcessError, StagehandElementNotFoundError, StagehandEnvironmentError, StagehandError, StagehandEvalError, StagehandHttpError, StagehandIframeError, StagehandInitError, StagehandInvalidArgumentError, type StagehandMetrics, StagehandMissingArgumentError, StagehandNotInitializedError, StagehandResponseBodyError, StagehandResponseParseError, StagehandServerError, StagehandShadowRootMissingError, StagehandShadowSegmentEmptyError, StagehandShadowSegmentNotFoundError, type StagehandZodObject, type StagehandZodSchema, type StreamingAgentInstance, StreamingCallbacksInNonStreamingModeError, TimeoutError, type ToolUseItem, UnsupportedAISDKModelProviderError, UnsupportedModelError, UnsupportedModelProviderError, V3, type V3Env, V3Evaluator, V3FunctionName, type V3Options, XPathResolutionError, ZodSchemaValidationError, connectToMCPServer, defaultExtractSchema, getZodType, injectUrls, isRunningInBun, isZod3Schema, isZod4Schema, jsonSchemaToZod, loadApiKeyFromEnv, localBrowserLaunchOptionsSchema, modelToAgentProviderMap, pageTextSchema, providerEnvVarMap, toGeminiSchema, toJsonSchema, transformSchema, trimTrailingTextNode, validateZodSchema };
+export { type AISDKCustomProvider, type AISDKProvider, AISdkClient, AVAILABLE_CUA_MODELS, type ActOptions, type ActResult, ActTimeoutError, type Action, type ActionExecutionResult, AgentAbortError, type AgentAction, type AgentCallbacks, type AgentConfig, type AgentContext, type AgentExecuteCallbacks, type AgentExecuteOptions, type AgentExecuteOptionsBase, type AgentExecutionOptions, type AgentHandlerOptions, type AgentInstance, type AgentModelConfig, AgentProvider, type AgentProviderType, type AgentResult, AgentScreenshotProviderError, type AgentState, type AgentStreamCallbacks, type AgentStreamExecuteOptions, type AgentStreamResult, type AgentToolCall, type AgentToolMode, type AgentToolResult, type AgentToolTypesMap, type AgentTools, type AgentType, type AgentUITools, AnnotatedScreenshotText, type AnthropicClientOptions, type AnthropicContentBlock, type AnthropicJsonSchemaObject, type AnthropicMessage, type AnthropicTextBlock, type AnthropicToolResult, type AnyPage, api as Api, type AvailableCuaModel, type AvailableModel, BrowserbaseSessionNotFoundError, CaptchaTimeoutError, type ChatCompletionOptions, type ChatMessage, type ChatMessageContent, type ChatMessageImageContent, type ChatMessageTextContent, type ClickToolResult, type ClientOptions, type ComputerCallItem, ConnectionTimeoutError, type ConsoleListener, ConsoleMessage, ContentFrameNotFoundError, type CreateChatCompletionOptions, CreateChatCompletionResponseError, CuaModelRequiredError, type DragAndDropToolResult, ElementNotVisibleError, ExperimentalApiConflictError, ExperimentalNotConfiguredError, type ExtractOptions, type ExtractResult, ExtractTimeoutError, type FillFormField, type FillFormVisionToolResult, type FunctionCallItem, type GoogleServiceAccountCredentials, type GoogleVertexProviderSettings, HandlerNotInitializedError, type HistoryEntry, type InferStagehandSchema, InvalidAISDKModelFormatError, type JsonSchema, type JsonSchemaDocument, type JsonSchemaProperty, LLMClient, type LLMParsedResponse, type LLMResponse, LLMResponseError, type LLMTool, type LLMUsage, LOG_LEVEL_NAMES, type LoadState, type LocalBrowserLaunchOptions, type LogLevel, type LogLine, type Logger, MCPConnectionError, MissingEnvironmentVariableError, MissingLLMConfigurationError, type ModelConfiguration, type ModelOutputContentItem, type ModelProvider, type NonStreamingAgentInstance, type ObserveOptions, ObserveTimeoutError, type OpenAIClientOptions, Page, PageNotFoundError, Response$1 as Response, ResponseBodyError, type ResponseInputItem, type ResponseItem, ResponseParseError, type SafetyCheck, type SafetyConfirmationHandler, type SafetyConfirmationResponse, type ScrollVisionToolResult, V3 as Stagehand, StagehandAPIError, StagehandAPIUnauthorizedError, StagehandClickError, StagehandClosedError, StagehandDefaultError, StagehandDomProcessError, StagehandElementNotFoundError, StagehandEnvironmentError, StagehandError, StagehandEvalError, StagehandHttpError, StagehandIframeError, StagehandInitError, StagehandInvalidArgumentError, type StagehandMetrics, StagehandMissingArgumentError, StagehandNotInitializedError, StagehandResponseBodyError, StagehandResponseParseError, StagehandServerError, StagehandShadowRootMissingError, StagehandShadowSegmentEmptyError, StagehandShadowSegmentNotFoundError, type StagehandZodObject, type StagehandZodSchema, type StreamingAgentInstance, StreamingCallbacksInNonStreamingModeError, TimeoutError, type ToolUseItem, type TypeToolResult, UnsupportedAISDKModelProviderError, UnsupportedModelError, UnsupportedModelProviderError, V3, type V3Env, V3Evaluator, V3FunctionName, type V3Options, type WaitToolResult, XPathResolutionError, ZodSchemaValidationError, connectToMCPServer, defaultExtractSchema, getZodType, injectUrls, isRunningInBun, isZod3Schema, isZod4Schema, jsonSchemaToZod, loadApiKeyFromEnv, localBrowserLaunchOptionsSchema, modelToAgentProviderMap, pageTextSchema, providerEnvVarMap, toGeminiSchema, toJsonSchema, transformSchema, trimTrailingTextNode, validateZodSchema };

package/dist/index.js

@@ -31125,7 +31125,25 @@ var screenshotTool = (v3) => (0, import_ai3.tool)({
 // lib/v3/agent/tools/wait.ts
 var import_ai4 = require("ai");
 var import_zod8 = require("zod");
-var waitTool = (v3) => (0, import_ai4.tool)({
+
+// lib/v3/agent/utils/screenshotHandler.ts
+var DEFAULT_DELAY_MS = 500;
+function waitAndCaptureScreenshot(_0) {
+  return __async(this, arguments, function* (page, delayMs = DEFAULT_DELAY_MS) {
+    if (delayMs > 0) {
+      yield page.waitForTimeout(delayMs);
+    }
+    try {
+      const buffer = yield page.screenshot({ fullPage: false });
+      return buffer.toString("base64");
+    } catch (e2) {
+      return void 0;
+    }
+  });
+}
+
+// lib/v3/agent/tools/wait.ts
+var waitTool = (v3, mode) => (0, import_ai4.tool)({
   description: "Wait for a specified time",
   inputSchema: import_zod8.z.object({
     timeMs: import_zod8.z.number().describe("Time in milliseconds")
@@ -31146,8 +31164,32 @@ var waitTool = (v3) => (0, import_ai4.tool)({
     if (timeMs > 0) {
       v3.recordAgentReplayStep({ type: "wait", timeMs });
     }
+    if (mode === "hybrid") {
+      const page = yield v3.context.awaitActivePage();
+      const screenshotBase64 = yield waitAndCaptureScreenshot(page, 0);
+      return { success: true, waited: timeMs, screenshotBase64 };
+    }
     return { success: true, waited: timeMs };
-  })
+  }),
+  toModelOutput: (result) => {
+    const content = [
+      {
+        type: "text",
+        text: JSON.stringify({
+          success: result.success,
+          waited: result.waited
+        })
+      }
+    ];
+    if (result.screenshotBase64) {
+      content.push({
+        type: "media",
+        mediaType: "image/png",
+        data: result.screenshotBase64
+      });
+    }
+    return { type: "content", value: content };
+  }
 });
 
 // lib/v3/agent/tools/navback.ts
@@ -31346,7 +31388,11 @@ var scrollVisionTool = (v3, provider) => (0, import_ai9.tool)({
     ),
     percentage: import_zod13.z.number().min(1).max(200).optional()
   }),
-  execute: (_0) => __async(null, [_0], function* ({ direction, coordinates, percentage = 80 }) {
+  execute: (_0) => __async(null, [_0], function* ({
+    direction,
+    coordinates,
+    percentage = 80
+  }) {
     const page = yield v3.context.awaitActivePage();
     const { w, h: h2 } = yield page.mainFrame().evaluate("({ w: window.innerWidth, h: window.innerHeight })");
     let cx;
@@ -31382,6 +31428,7 @@ var scrollVisionTool = (v3, provider) => (0, import_ai9.tool)({
     const scrollDistance = Math.round(h2 * percentage / 100);
     const deltaY = direction === "up" ? -scrollDistance : scrollDistance;
     yield page.scroll(cx, cy, 0, deltaY);
+    const screenshotBase64 = yield waitAndCaptureScreenshot(page, 100);
     v3.recordAgentReplayStep({
       type: "scroll",
       deltaX: 0,
@@ -31391,9 +31438,30 @@ var scrollVisionTool = (v3, provider) => (0, import_ai9.tool)({
     return {
       success: true,
       message: coordinates ? `Scrolled ${percentage}% ${direction} at (${cx}, ${cy})` : `Scrolled ${percentage}% ${direction}`,
-      scrolledPixels: scrollDistance
+      scrolledPixels: scrollDistance,
+      screenshotBase64
     };
-  })
+  }),
+  toModelOutput: (result) => {
+    const content = [
+      {
+        type: "text",
+        text: JSON.stringify({
+          success: result.success,
+          message: result.message,
+          scrolledPixels: result.scrolledPixels
+        })
+      }
+    ];
+    if (result.screenshotBase64) {
+      content.push({
+        type: "media",
+        mediaType: "image/png",
+        data: result.screenshotBase64
+      });
+    }
+    return { type: "content", value: content };
+  }
 });
 
 // lib/v3/agent/tools/extract.ts
@@ -31461,9 +31529,6 @@ function ensureXPath(value) {
 }
 
 // lib/v3/agent/tools/click.ts
-function waitForTimeout(ms) {
-  return new Promise((resolve3) => setTimeout(resolve3, ms));
-}
 var clickTool = (v3, provider) => (0, import_ai11.tool)({
   description: "Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)",
   inputSchema: import_zod15.z.object({
@@ -31491,31 +31556,33 @@ var clickTool = (v3, provider) => (0, import_ai11.tool)({
           }
         }
       });
+      const shouldCollectXpath = v3.isAgentReplayActive();
       const xpath = yield page.click(processed.x, processed.y, {
-        returnXpath: true
+        returnXpath: shouldCollectXpath
       });
-      if (isGoogleProvider(provider)) {
-        yield waitForTimeout(1e3);
-      }
-      const normalizedXpath = ensureXPath(xpath);
-      if (normalizedXpath) {
-        const action = {
-          selector: normalizedXpath,
-          description: describe,
-          method: "click",
-          arguments: []
-        };
-        v3.recordAgentReplayStep({
-          type: "act",
-          instruction: describe,
-          actions: [action],
-          actionDescription: describe
-        });
+      const screenshotBase64 = yield waitAndCaptureScreenshot(page);
+      if (shouldCollectXpath) {
+        const normalizedXpath = ensureXPath(xpath);
+        if (normalizedXpath) {
+          const action = {
+            selector: normalizedXpath,
+            description: describe,
+            method: "click",
+            arguments: []
+          };
+          v3.recordAgentReplayStep({
+            type: "act",
+            instruction: describe,
+            actions: [action],
+            actionDescription: describe
+          });
+        }
       }
       return {
         success: true,
         describe,
-        coordinates: [processed.x, processed.y]
+        coordinates: [processed.x, processed.y],
+        screenshotBase64
       };
     } catch (error) {
       return {
@@ -31523,15 +31590,46 @@ var clickTool = (v3, provider) => (0, import_ai11.tool)({
         error: `Error clicking: ${error.message}`
       };
     }
-  })
+  }),
+  toModelOutput: (result) => {
+    if (result.success) {
+      const content = [
+        {
+          type: "text",
+          text: JSON.stringify({
+            success: result.success,
+            describe: result.describe,
+            coordinates: result.coordinates
+          })
+        }
+      ];
+      if (result.screenshotBase64) {
+        content.push({
+          type: "media",
+          mediaType: "image/png",
+          data: result.screenshotBase64
+        });
+      }
+      return { type: "content", value: content };
+    }
+    return {
+      type: "content",
+      value: [
+        {
+          type: "text",
+          text: JSON.stringify({
+            success: result.success,
+            error: result.error
+          })
+        }
+      ]
+    };
+  }
 });
 
 // lib/v3/agent/tools/type.ts
 var import_ai12 = require("ai");
 var import_zod16 = require("zod");
-function waitForTimeout2(ms) {
-  return new Promise((resolve3) => setTimeout(resolve3, ms));
-}
 var typeTool = (v3, provider) => (0, import_ai12.tool)({
   description: "Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)",
   inputSchema: import_zod16.z.object({
@@ -31541,7 +31639,11 @@ var typeTool = (v3, provider) => (0, import_ai12.tool)({
     text: import_zod16.z.string().describe("The text to type into the element"),
     coordinates: import_zod16.z.array(import_zod16.z.number()).describe("The (x, y) coordinates to type into the element")
   }),
-  execute: (_0) => __async(null, [_0], function* ({ describe, coordinates, text }) {
+  execute: (_0) => __async(null, [_0], function* ({
+    describe,
+    coordinates,
+    text
+  }) {
     try {
       const page = yield v3.context.awaitActivePage();
       const processed = processCoordinates(
@@ -31560,36 +31662,76 @@ var typeTool = (v3, provider) => (0, import_ai12.tool)({
           }
         }
       });
+      const shouldCollectXpath = v3.isAgentReplayActive();
       const xpath = yield page.click(processed.x, processed.y, {
-        returnXpath: true
+        returnXpath: shouldCollectXpath
       });
-      if (isGoogleProvider(provider)) {
-        yield waitForTimeout2(1e3);
-      }
       yield page.type(text);
-      const normalizedXpath = ensureXPath(xpath);
-      if (normalizedXpath) {
-        const action = {
-          selector: normalizedXpath,
-          description: describe,
-          method: "type",
-          arguments: [text]
-        };
-        v3.recordAgentReplayStep({
-          type: "act",
-          instruction: describe,
-          actions: [action],
-          actionDescription: describe
-        });
+      const screenshotBase64 = yield waitAndCaptureScreenshot(page);
+      if (shouldCollectXpath) {
+        const normalizedXpath = ensureXPath(xpath);
+        if (normalizedXpath) {
+          const action = {
+            selector: normalizedXpath,
+            description: describe,
+            method: "type",
+            arguments: [text]
+          };
+          v3.recordAgentReplayStep({
+            type: "act",
+            instruction: describe,
+            actions: [action],
+            actionDescription: describe
+          });
+        }
       }
-      return { success: true, describe, text };
+      return {
+        success: true,
+        describe,
+        text,
+        screenshotBase64
+      };
     } catch (error) {
       return {
         success: false,
         error: `Error typing: ${error.message}`
       };
     }
-  })
+  }),
+  toModelOutput: (result) => {
+    if (result.success) {
+      const content = [
+        {
+          type: "text",
+          text: JSON.stringify({
+            success: result.success,
+            describe: result.describe,
+            text: result.text
+          })
+        }
+      ];
+      if (result.screenshotBase64) {
+        content.push({
+          type: "media",
+          mediaType: "image/png",
+          data: result.screenshotBase64
+        });
+      }
+      return { type: "content", value: content };
+    }
+    return {
+      type: "content",
+      value: [
+        {
+          type: "text",
+          text: JSON.stringify({
+            success: result.success,
+            error: result.error
+          })
+        }
+      ]
+    };
+  }
 });
 
 // lib/v3/agent/tools/dragAndDrop.ts
@@ -31602,7 +31744,11 @@ var dragAndDropTool = (v3, provider) => (0, import_ai13.tool)({
     startCoordinates: import_zod17.z.array(import_zod17.z.number()).describe("The (x, y) coordinates to start the drag and drop from"),
     endCoordinates: import_zod17.z.array(import_zod17.z.number()).describe("The (x, y) coordinates to end the drag and drop at")
   }),
-  execute: (_0) => __async(null, [_0], function* ({ describe, startCoordinates, endCoordinates }) {
+  execute: (_0) => __async(null, [_0], function* ({
+    describe,
+    startCoordinates,
+    endCoordinates
+  }) {
     try {
       const page = yield v3.context.awaitActivePage();
       const processedStart = processCoordinates(
@@ -31632,37 +31778,78 @@ var dragAndDropTool = (v3, provider) => (0, import_ai13.tool)({
           }
         }
       });
+      const shouldCollectXpath = v3.isAgentReplayActive();
       const [fromXpath, toXpath] = yield page.dragAndDrop(
         processedStart.x,
         processedStart.y,
         processedEnd.x,
         processedEnd.y,
-        { returnXpath: true }
+        { returnXpath: shouldCollectXpath }
       );
-      const normalizedFrom = ensureXPath(fromXpath);
-      const normalizedTo = ensureXPath(toXpath);
-      if (normalizedFrom && normalizedTo) {
-        const action = {
-          selector: normalizedFrom,
-          description: describe,
-          method: "dragAndDrop",
-          arguments: [normalizedTo]
-        };
-        v3.recordAgentReplayStep({
-          type: "act",
-          instruction: describe,
-          actions: [action],
-          actionDescription: describe
-        });
+      const screenshotBase64 = yield waitAndCaptureScreenshot(page);
+      if (shouldCollectXpath) {
+        const normalizedFrom = ensureXPath(fromXpath);
+        const normalizedTo = ensureXPath(toXpath);
+        if (normalizedFrom && normalizedTo) {
+          const action = {
+            selector: normalizedFrom,
+            description: describe,
+            method: "dragAndDrop",
+            arguments: [normalizedTo]
+          };
+          v3.recordAgentReplayStep({
+            type: "act",
+            instruction: describe,
+            actions: [action],
+            actionDescription: describe
+          });
+        }
       }
-      return { success: true, describe };
+      return {
+        success: true,
+        describe,
+        screenshotBase64
+      };
     } catch (error) {
       return {
         success: false,
         error: `Error dragging: ${error.message}`
       };
     }
-  })
+  }),
+  toModelOutput: (result) => {
+    if (result.success) {
+      const content = [
+        {
+          type: "text",
+          text: JSON.stringify({
+            success: result.success,
+            describe: result.describe
+          })
+        }
+      ];
+      if (result.screenshotBase64) {
+        content.push({
+          type: "media",
+          mediaType: "image/png",
+          data: result.screenshotBase64
+        });
+      }
+      return { type: "content", value: content };
+    }
+    return {
+      type: "content",
+      value: [
+        {
+          type: "text",
+          text: JSON.stringify({
+            success: result.success,
+            error: result.error
+          })
+        }
+      ]
+    };
+  }
 });
 
 // lib/v3/agent/tools/clickAndHold.ts
@@ -31701,27 +31888,30 @@ var clickAndHoldTool = (v3, provider) => (0, import_ai14.tool)({
           }
         }
       });
+      const shouldCollectXpath = v3.isAgentReplayActive();
       const [xpath] = yield page.dragAndDrop(
         processed.x,
         processed.y,
         processed.x,
         processed.y,
-        { delay: duration, returnXpath: true }
+        { delay: duration, returnXpath: shouldCollectXpath }
       );
-      const normalizedXpath = ensureXPath(xpath);
-      if (normalizedXpath) {
-        const action = {
-          selector: normalizedXpath,
-          description: describe,
-          method: "clickAndHold",
-          arguments: [String(duration)]
-        };
-        v3.recordAgentReplayStep({
-          type: "act",
-          instruction: describe,
-          actions: [action],
-          actionDescription: describe
-        });
+      if (shouldCollectXpath) {
+        const normalizedXpath = ensureXPath(xpath);
+        if (normalizedXpath) {
+          const action = {
+            selector: normalizedXpath,
+            description: describe,
+            method: "clickAndHold",
+            arguments: [String(duration)]
+          };
+          v3.recordAgentReplayStep({
+            type: "act",
+            instruction: describe,
+            actions: [action],
+            actionDescription: describe
+          });
+        }
       }
       return { success: true, describe };
     } catch (error) {
@@ -31852,28 +32042,32 @@ MANDATORY USE CASES (always use fillFormVision for these):
           }
         }
       });
+      const shouldCollectXpath = v3.isAgentReplayActive();
       const actions = [];
       for (const field of processedFields) {
         const xpath = yield page.click(
           field.coordinates.x,
           field.coordinates.y,
           {
-            returnXpath: true
+            returnXpath: shouldCollectXpath
           }
         );
         yield page.type(field.value);
-        const normalizedXpath = ensureXPath(xpath);
-        if (normalizedXpath) {
-          actions.push({
-            selector: normalizedXpath,
-            description: field.action,
-            method: "type",
-            arguments: [field.value]
-          });
+        if (shouldCollectXpath) {
+          const normalizedXpath = ensureXPath(xpath);
+          if (normalizedXpath) {
+            actions.push({
+              selector: normalizedXpath,
+              description: field.action,
+              method: "type",
+              arguments: [field.value]
+            });
+          }
         }
         yield new Promise((resolve3) => setTimeout(resolve3, 100));
       }
-      if (actions.length > 0) {
+      const screenshotBase64 = yield waitAndCaptureScreenshot(page, 100);
+      if (shouldCollectXpath && actions.length > 0) {
         v3.recordAgentReplayStep({
           type: "act",
           instruction: `Fill ${fields.length} form fields`,
@@ -31883,7 +32077,8 @@ MANDATORY USE CASES (always use fillFormVision for these):
       }
       return {
         success: true,
-        playwrightArguments: processedFields
+        playwrightArguments: processedFields,
+        screenshotBase64
       };
     } catch (error) {
       return {
@@ -31891,7 +32086,41 @@ MANDATORY USE CASES (always use fillFormVision for these):
         error: `Error filling form: ${error.message}`
       };
     }
-  })
+  }),
+  toModelOutput: (result) => {
+    var _a4, _b;
+    if (result.success) {
+      const content = [
+        {
+          type: "text",
+          text: JSON.stringify({
+            success: result.success,
+            fieldsCount: (_b = (_a4 = result.playwrightArguments) == null ? void 0 : _a4.length) != null ? _b : 0
+          })
+        }
+      ];
+      if (result.screenshotBase64) {
+        content.push({
+          type: "media",
+          mediaType: "image/png",
+          data: result.screenshotBase64
+        });
+      }
+      return { type: "content", value: content };
+    }
+    return {
+      type: "content",
+      value: [
+        {
+          type: "text",
+          text: JSON.stringify({
+            success: result.success,
+            error: result.error
+          })
+        }
+      ]
+    };
+  }
 });
 
 // lib/v3/agent/tools/think.ts
@@ -32040,7 +32269,7 @@ function createAgentTools(v3, options) {
     scroll: mode === "hybrid" ? scrollVisionTool(v3, provider) : scrollTool(v3),
     think: thinkTool(),
     type: typeTool(v3, provider),
-    wait: waitTool(v3)
+    wait: waitTool(v3, mode)
   };
   if (process.env.BRAVE_API_KEY) {
     allTools.search = searchTool(v3);
@@ -32238,37 +32467,54 @@ function buildAgentSystemPrompt(options) {
 var import_ai19 = require("ai");
 
 // lib/v3/agent/utils/messageProcessing.ts
+var VISION_ACTION_TOOLS = [
+  "click",
+  "type",
+  "dragAndDrop",
+  "wait",
+  "fillFormVision",
+  "scroll"
+];
 function isToolMessage(message) {
   return !!message && typeof message === "object" && message.role === "tool" && Array.isArray(message.content);
 }
 function isScreenshotPart(part) {
   return !!part && typeof part === "object" && part.toolName === "screenshot";
 }
+function isVisionActionPart(part) {
+  if (!part || typeof part !== "object") return false;
+  const toolName = part.toolName;
+  return typeof toolName === "string" && VISION_ACTION_TOOLS.includes(toolName);
+}
+function isVisionPart(part) {
+  return isScreenshotPart(part) || isVisionActionPart(part);
+}
 function isAriaTreePart(part) {
   return !!part && typeof part === "object" && part.toolName === "ariaTree";
 }
 function processMessages(messages) {
   let compressedCount = 0;
-  const screenshotIndices = [];
+  const visionIndices = [];
   const ariaTreeIndices = [];
   for (let i2 = 0; i2 < messages.length; i2++) {
     const message = messages[i2];
     if (isToolMessage(message)) {
       const content = message.content;
-      if (content.some(isScreenshotPart)) {
-        screenshotIndices.push(i2);
+      if (content.some(isVisionPart)) {
+        visionIndices.push(i2);
       }
       if (content.some(isAriaTreePart)) {
         ariaTreeIndices.push(i2);
       }
     }
   }
-  if (screenshotIndices.length > 2) {
-    const toCompress = screenshotIndices.slice(0, screenshotIndices.length - 2);
-    for (const idx of toCompress) {
-      const message = messages[idx];
+  if (visionIndices.length > 2) {
+    const toCompress = visionIndices.slice(0, visionIndices.length - 2);
+    for (const index of toCompress) {
+      const message = messages[index];
       if (isToolMessage(message)) {
         compressScreenshotMessage(message);
+        compressVisionActionMessage(message);
         compressedCount++;
       }
     }
@@ -32300,6 +32546,24 @@ function compressScreenshotMessage(message) {
     }
   }
 }
+function compressVisionActionMessage(message) {
+  var _a4;
+  for (const part of message.content) {
+    if (isVisionActionPart(part)) {
+      const typedPart = part;
+      if (((_a4 = typedPart.output) == null ? void 0 : _a4.value) && Array.isArray(typedPart.output.value)) {
+        typedPart.output.value = typedPart.output.value.filter(
+          (item) => item && typeof item === "object" && item.type !== "media"
+        );
+      }
+      if (typedPart.result && Array.isArray(typedPart.result)) {
+        typedPart.result = typedPart.result.filter(
+          (item) => item && typeof item === "object" && item.type !== "media"
+        );
+      }
+    }
+  }
+}
 function compressAriaTreeMessage(message) {
   var _a4;
   for (const part of message.content) {
@@ -32325,6 +32589,18 @@ function compressAriaTreeMessage(message) {
 init_flowLogger();
 
 // lib/v3/agent/utils/actionMapping.ts
+var EXCLUDED_OUTPUT_KEYS = ["screenshotBase64"];
+function stripExcludedKeys(output) {
+  const result = {};
+  for (const [key, value] of Object.entries(output)) {
+    if (!EXCLUDED_OUTPUT_KEYS.includes(
+      key
+    )) {
+      result[key] = value;
+    }
+  }
+  return result;
+}
 function mapToolResultToActions({
   toolCallName,
   toolResult,
@@ -32390,8 +32666,14 @@ function createStandardAction(toolCallName, toolResult, args, reasoning) {
     return action;
   }
   if (toolCallName !== "ariaTree" && toolResult) {
-    const { output } = toolResult;
-    Object.assign(action, output);
+    const result = toolResult;
+    const output = result.output;
+    if (output && typeof output === "object" && !Array.isArray(output)) {
+      const cleanedOutput = stripExcludedKeys(
+        output
+      );
+      Object.assign(action, cleanedOutput);
+    }
   }
   return action;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserbasehq/stagehand",
-  "version": "3.0.8-alpha-3d5af07f66d6d26d1f5ac4bd9be7183c3381dd92",
+  "version": "3.0.8-alpha-16d72fb4c4081dd33bf45605d75c27644ea4c00e",
   "description": "An AI web browsing framework focused on simplicity and extensibility.",
   "main": "./dist/index.js",
   "module": "./dist/index.js",