@browserbasehq/orca 3.7.0-preview.1 → 3.7.2-preview.0

package/dist/cjs/lib/v3/types/public/agent.d.ts

@@ -411,7 +411,7 @@ export interface AgentStreamExecuteOptions extends AgentExecuteOptionsBase {
     callbacks?: AgentStreamCallbacks;
 }
 export type AgentType = "openai" | "anthropic" | "google" | "microsoft" | "bedrock" | "vertex" | "azure";
-export declare const AVAILABLE_CUA_MODELS: readonly ["openai/gpt-5.4", "openai/gpt-5.4-mini", "openai/gpt-5.5", "openai/computer-use-preview", "openai/computer-use-preview-2025-03-11", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-6", "anthropic/claude-opus-4-8", "anthropic/claude-sonnet-4-6", "anthropic/claude-haiku-4-5", "anthropic/claude-haiku-4-5-20251001", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-5-20250929", "anthropic/claude-fable-5", "google/gemini-2.5-computer-use-preview-10-2025", "google/gemini-3-flash-preview", "google/gemini-3.5-flash", "google/gemini-3-pro-preview", "microsoft/fara-7b"];
+export declare const AVAILABLE_CUA_MODELS: readonly ["openai/gpt-5.4", "openai/gpt-5.4-mini", "openai/gpt-5.5", "openai/gpt-5.6-terra", "openai/gpt-5.6-luna", "openai/gpt-5.6-sol", "openai/computer-use-preview", "openai/computer-use-preview-2025-03-11", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-6", "anthropic/claude-opus-4-8", "anthropic/claude-sonnet-4-6", "anthropic/claude-haiku-4-5", "anthropic/claude-haiku-4-5-20251001", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-5-20250929", "anthropic/claude-fable-5", "google/gemini-2.5-computer-use-preview-10-2025", "google/gemini-3-flash-preview", "google/gemini-3.5-flash", "google/gemini-3-pro-preview", "microsoft/fara-7b"];
 export type AvailableCuaModel = (typeof AVAILABLE_CUA_MODELS)[number];
 export interface AgentExecutionOptions<TOptions extends AgentExecuteOptions = AgentExecuteOptions> {
     options: TOptions;

package/dist/cjs/lib/v3/types/public/agent.js

@@ -5,6 +5,9 @@ exports.AVAILABLE_CUA_MODELS = [
     "openai/gpt-5.4",
     "openai/gpt-5.4-mini",
     "openai/gpt-5.5",
+    "openai/gpt-5.6-terra",
+    "openai/gpt-5.6-luna",
+    "openai/gpt-5.6-sol",
     "openai/computer-use-preview",
     "openai/computer-use-preview-2025-03-11",
     "anthropic/claude-opus-4-5-20251101",

package/dist/cjs/lib/v3/types/public/agent.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent.js","sourceRoot":"","sources":["../../../../../../lib/v3/types/public/agent.ts"],"names":[],"mappings":";;;AAmda,QAAA,oBAAoB,GAAG;IAClC,gBAAgB;IAChB,qBAAqB;IACrB,gBAAgB;IAChB,6BAA6B;IAC7B,wCAAwC;IACxC,oCAAoC;IACpC,2BAA2B;IAC3B,2BAA2B;IAC3B,6BAA6B;IAC7B,4BAA4B;IAC5B,qCAAqC;IACrC,oCAAoC;IACpC,sCAAsC;IACtC,0BAA0B;IAC1B,gDAAgD;IAChD,+BAA+B;IAC/B,yBAAyB;IACzB,6BAA6B;IAC7B,mBAAmB;CACX,CAAC","sourcesContent":["import type { Client } from \"@modelcontextprotocol/sdk/client/index.js\";\nimport {\n  ToolSet,\n  ModelMessage,\n  wrapLanguageModel,\n  StreamTextResult,\n  StepResult,\n  PrepareStepFunction,\n  GenerateTextOnStepFinishCallback,\n  StreamTextOnStepFinishCallback,\n  StreamTextOnErrorCallback,\n  StreamTextOnChunkCallback,\n  StreamTextOnFinishCallback,\n} from \"ai\";\nimport { LogLine } from \"./logs.js\";\nimport { ClientOptions } from \"./model.js\";\nimport { StagehandZodObject } from \"../../zodCompat.js\";\nimport type { AgentEvidenceCallback } from \"./agentEvidenceEvents.js\";\n\n// Re-export ModelMessage for consumers who want to use it for conversation continuation\nexport type { ModelMessage } from \"ai\";\n\n// Re-export Tool type for consumers who want to define custom tools\nexport type { Tool } from \"ai\";\nimport { Page as PlaywrightPage } from \"playwright-core\";\nimport { Page as PuppeteerPage } from \"puppeteer-core\";\nimport { Page as PatchrightPage } from \"patchright-core\";\nimport { Page } from \"../../understudy/page.js\";\n\n/**\n * Result of a screenshot provider: the base64-encoded image bytes plus an\n * explicit media type. Declaring the media type at the capture boundary lets\n * every CUA client pass it through instead of hardcoding or inferring it.\n */\nexport interface ScreenshotProviderResult {\n  base64: string;\n  mediaType: \"image/png\" | \"image/jpeg\";\n}\n\n// =============================================================================\n// Variable Types\n// =============================================================================\n\n/**\n * A variable value can be a simple primitive or a rich object with an optional description.\n * This unified type is shared across `act`, `agent.execute`, and other methods.\n *\n * @example Simple (backward-compatible):\n * ```typescript\n * variables: { username: \"john@example.com\" }\n * ```\n *\n * @example Rich with description (useful for agents):\n * ```typescript\n * variables: {\n *   username: { value: \"john@example.com\", description: \"The login email\" }\n * }\n * ```\n */\nexport type VariableValue =\n  | string\n  | number\n  | boolean\n  | { value: string | number | boolean; description?: string };\n\n/**\n * A collection of named variables for use in act, agent, and other methods.\n */\nexport type Variables = Record<string, VariableValue>;\n\nexport interface AgentContext {\n  options: AgentExecuteOptionsBase;\n  maxSteps: number;\n  systemPrompt: string;\n  allTools: ToolSet;\n  messages: ModelMessage[];\n  wrappedModel: ReturnType<typeof wrapLanguageModel>;\n  initialPageUrl: string;\n}\n\nexport interface AgentState {\n  collectedReasoning: string[];\n  actions: AgentAction[];\n  finalMessage: string;\n  completed: boolean;\n  currentPageUrl: string;\n}\n\nexport interface AgentAction {\n  type: string;\n  reasoning?: string;\n  taskCompleted?: boolean;\n  action?: string;\n  // Tool-specific fields\n  timeMs?: number; // wait tool\n  pageText?: string; // ariaTree tool\n  pageUrl?: string; // ariaTree tool\n  instruction?: string; // various tools\n  [key: string]: unknown;\n}\n\nexport interface AgentResult {\n  success: boolean;\n  message: string;\n  actions: AgentAction[];\n  completed: boolean;\n  metadata?: Record<string, unknown>;\n  usage?: {\n    input_tokens: number;\n    output_tokens: number;\n    reasoning_tokens?: number;\n    cached_input_tokens?: number;\n    inference_time_ms: number;\n  };\n  /**\n   * The conversation messages from this execution.\n   * Pass these to a subsequent execute() call via the `messages` option to continue the conversation.\n   * @experimental\n   */\n  messages?: ModelMessage[];\n  /**\n   * Custom output data extracted based on the `output` schema provided in execute options.\n   * Only populated if an `output` schema was provided.\n   * @experimental\n   */\n  output?: Record<string, unknown>;\n}\n\nexport type AgentStreamResult = StreamTextResult<ToolSet, never> & {\n  result: Promise<AgentResult>;\n};\n\n/**\n * Base callbacks shared between execute (non-streaming) and streaming modes.\n */\nexport interface AgentCallbacks {\n  /**\n   * Optional function called before each step to modify settings.\n   * You can change the model, tool choices, active tools, system prompt,\n   * and input messages for each step.\n   */\n  prepareStep?: PrepareStepFunction<ToolSet>;\n  /**\n   * Callback called when each step (LLM call) is finished.\n   * This is called for intermediate steps as well as the final step.\n   */\n  onStepFinish?:\n    | GenerateTextOnStepFinishCallback<ToolSet>\n    | StreamTextOnStepFinishCallback<ToolSet>;\n  /**\n   * Callback called when Stagehand captures agent-run evidence such as\n   * screenshots, completed tool/action steps, or post-action observations.\n   */\n  onEvidence?: AgentEvidenceCallback;\n}\n\n/**\n * Error message type for streaming-only callbacks used in non-streaming mode.\n * This provides a clear error message when users try to use streaming callbacks without stream: true.\n */\ntype StreamingCallbackNotAvailable =\n  \"This callback requires 'stream: true' in AgentConfig. Set stream: true to use streaming callbacks like onChunk, onFinish, onError, and onAbort.\";\n\n/**\n * Error message for safety confirmation callback misuse.\n * Safety confirmations are only available for non-streaming CUA agent executions.\n */\ntype SafetyConfirmationCallbackNotAvailable =\n  \"Safety confirmation callbacks are only available via non-streaming AgentExecuteOptions.callbacks when using mode: 'cua'.\";\n\n/**\n * Callbacks specific to the non-streaming execute method.\n */\nexport interface AgentExecuteCallbacks extends AgentCallbacks {\n  /**\n   * Callback called when each step (LLM call) is finished.\n   */\n  onStepFinish?: GenerateTextOnStepFinishCallback<ToolSet>;\n  /**\n   * Callback for handling safety confirmation requests from CUA providers.\n   * Only available when running an agent configured with mode: \"cua\".\n   */\n  onSafetyConfirmation?: SafetyConfirmationHandler;\n\n  /**\n   * NOT AVAILABLE in non-streaming mode.\n   * This callback requires `stream: true` in AgentConfig.\n   *\n   * @example\n   * ```typescript\n   * // Enable streaming to use onChunk:\n   * const agent = stagehand.agent({ stream: true });\n   * await agent.execute({\n   *   instruction: \"...\",\n   *   callbacks: { onChunk: async (chunk) => console.log(chunk) }\n   * });\n   * ```\n   */\n  onChunk?: StreamingCallbackNotAvailable;\n\n  /**\n   * NOT AVAILABLE in non-streaming mode.\n   * This callback requires `stream: true` in AgentConfig.\n   *\n   * @example\n   * ```typescript\n   * // Enable streaming to use onFinish:\n   * const agent = stagehand.agent({ stream: true });\n   * await agent.execute({\n   *   instruction: \"...\",\n   *   callbacks: { onFinish: (event) => console.log(\"Done!\", event) }\n   * });\n   * ```\n   */\n  onFinish?: StreamingCallbackNotAvailable;\n\n  /**\n   * NOT AVAILABLE in non-streaming mode.\n   * This callback requires `stream: true` in AgentConfig.\n   *\n   * @example\n   * ```typescript\n   * // Enable streaming to use onError:\n   * const agent = stagehand.agent({ stream: true });\n   * await agent.execute({\n   *   instruction: \"...\",\n   *   callbacks: { onError: ({ error }) => console.error(error) }\n   * });\n   * ```\n   */\n  onError?: StreamingCallbackNotAvailable;\n\n  /**\n   * NOT AVAILABLE in non-streaming mode.\n   * This callback requires `stream: true` in AgentConfig.\n   *\n   * @example\n   * ```typescript\n   * // Enable streaming to use onAbort:\n   * const agent = stagehand.agent({ stream: true });\n   * await agent.execute({\n   *   instruction: \"...\",\n   *   callbacks: { onAbort: (event) => console.log(\"Aborted\", event.steps) }\n   * });\n   * ```\n   */\n  onAbort?: StreamingCallbackNotAvailable;\n}\n\n/**\n * Callbacks specific to the streaming mode.\n */\nexport interface AgentStreamCallbacks extends AgentCallbacks {\n  /**\n   * Callback called when each step (LLM call) is finished during streaming.\n   */\n  onStepFinish?: StreamTextOnStepFinishCallback<ToolSet>;\n  /**\n   * Callback called when an error occurs during streaming.\n   * Use this to log errors or handle error states.\n   */\n  onError?: StreamTextOnErrorCallback;\n  /**\n   * Callback called for each chunk of the stream.\n   * Stream processing will pause until the callback promise resolves.\n   */\n  onChunk?: StreamTextOnChunkCallback<ToolSet>;\n  /**\n   * Callback called when the stream finishes.\n   */\n  onFinish?: StreamTextOnFinishCallback<ToolSet>;\n  /**\n   * Callback called when the stream is aborted.\n   */\n  onAbort?: (event: {\n    steps: Array<StepResult<ToolSet>>;\n  }) => PromiseLike<void> | void;\n  /**\n   * NOT AVAILABLE in streaming mode.\n   * Safety confirmations currently require non-streaming execute() on CUA agents.\n   */\n  onSafetyConfirmation?: SafetyConfirmationCallbackNotAvailable;\n}\n\n/**\n * Base options for agent execution (without callbacks).\n */\nexport interface AgentExecuteOptionsBase {\n  instruction: string;\n  maxSteps?: number;\n  page?: PlaywrightPage | PuppeteerPage | PatchrightPage | Page;\n  highlightCursor?: boolean;\n  /**\n   * Previous conversation messages to continue from.\n   * Pass the `messages` from a previous AgentResult to continue that conversation.\n   * @experimental\n   */\n  messages?: ModelMessage[];\n  /**\n   * An AbortSignal that can be used to cancel the agent execution.\n   * When aborted, the agent will stop and return a partial result.\n   * @experimental\n   *\n   * @example\n   * ```typescript\n   * const controller = new AbortController();\n   * setTimeout(() => controller.abort(), 30000); // 30 second timeout\n   *\n   * const result = await agent.execute({\n   *   instruction: \"...\",\n   *   signal: controller.signal\n   * });\n   * ```\n   */\n  signal?: AbortSignal;\n  /**\n   * Tools to exclude from this execution.\n   * Pass an array of tool names to prevent the agent from using those tools.\n   *\n   * **Note:** Not supported in CUA mode (`mode: \"cua\"`).\n   *\n   * **Available tools by mode:**\n   *\n   * **DOM mode (default):**\n   * - `act` - Perform semantic actions (click, type, etc.)\n   * - `fillForm` - Fill form fields using DOM selectors\n   * - `ariaTree` - Get accessibility tree of the page\n   * - `extract` - Extract structured data from page\n   * - `goto` - Navigate to a URL\n   * - `scroll` - Scroll using semantic directions (up/down/left/right)\n   * - `keys` - Press keyboard keys\n   * - `navback` - Navigate back in history\n   * - `screenshot` - Take a screenshot\n   * - `think` - Agent reasoning/planning step\n   * - `wait` - Wait for time or condition\n   * - `done` - Mark task as complete\n   * - `search` - Web search (requires useSearch: true and BROWSERBASE_API_KEY)\n   *\n   * **Hybrid mode:**\n   * - `click` - Click at specific coordinates\n   * - `type` - Type text at coordinates\n   * - `dragAndDrop` - Drag from one point to another\n   * - `clickAndHold` - Click and hold at coordinates\n   * - `fillFormVision` - Fill forms using vision/coordinates\n   * - `act` - Perform semantic actions\n   * - `ariaTree` - Get accessibility tree\n   * - `extract` - Extract data from page\n   * - `goto` - Navigate to URL\n   * - `scroll` - Scroll using coordinates\n   * - `keys` - Press keyboard keys\n   * - `navback` - Navigate back\n   * - `screenshot` - Take screenshot\n   * - `think` - Agent reasoning step\n   * - `wait` - Wait for time/condition\n   * - `done` - Mark task complete\n   * - `search` - Web search (requires useSearch: true and BROWSERBASE_API_KEY)\n   *\n   * @experimental\n   * @example\n   * ```typescript\n   * // Exclude screenshot and extract tools\n   * const result = await agent.execute({\n   *   instruction: \"Click the submit button\",\n   *   excludeTools: [\"screenshot\", \"extract\"]\n   * });\n   * ```\n   */\n  excludeTools?: string[];\n  /**\n   * A Zod schema defining custom output data to return when the task completes.\n   * The agent will populate this data in the final done tool call.\n   *\n   * @experimental\n   * @example\n   * ```typescript\n   * const result = await agent.execute({\n   *   instruction: \"Find the cheapest flight from NYC to LA\",\n   *   output: z.object({\n   *     price: z.string().describe(\"The price of the flight\"),\n   *     airline: z.string().describe(\"The airline name\"),\n   *     departureTime: z.string().describe(\"Departure time\"),\n   *   }),\n   * });\n   *\n   * console.log(result.output); // { price: \"$199\", airline: \"Delta\", departureTime: \"8:00 AM\" }\n   * ```\n   */\n  output?: StagehandZodObject;\n  /**\n   * Variables that the agent can use when filling forms or typing text.\n   * The agent will see variable names and descriptions in the system prompt,\n   * and can use them via `%variableName%` syntax in act/type/fillForm tool calls.\n   *\n   * Accepts both simple values and rich objects with descriptions (same type as `act`).\n   *\n   * **Note:** Not supported in CUA mode (`mode: \"cua\"`).\n   *\n   * @example\n   * ```typescript\n   * // Simple values\n   * variables: { username: \"john@example.com\", password: \"secret123\" }\n   *\n   * // Rich values with descriptions (helps the agent understand context)\n   * variables: {\n   *   username: { value: \"john@example.com\", description: \"The login email\" },\n   *   password: { value: \"secret123\", description: \"The login password\" },\n   * }\n   * ```\n   */\n  variables?: Variables;\n  /**\n   * Timeout in milliseconds for each agent tool call.\n   * If a tool call exceeds this duration, it will be aborted and\n   * reported back to the LLM as a timeout error so it can retry or adjust.\n   * For tools that call v3 methods (act, extract, fillForm, ariaTree), the\n   * timeout is also forwarded to the underlying v3 call for true cancellation.\n   * @default 45000 (45 seconds)\n   */\n  toolTimeout?: number;\n  /**\n   * Enable the web search tool powered by Browserbase Search API.\n   * Requires a valid Browserbase API key (BROWSERBASE_API_KEY).\n   * When set to true, the agent gains access to a `search` tool for web searches.\n   *\n   * @example\n   * ```typescript\n   * const result = await agent.execute({\n   *   instruction: \"Find the latest news about AI\",\n   *   useSearch: true,\n   * });\n   * ```\n   */\n  useSearch?: boolean;\n}\n\n/**\n * Options for non-streaming agent execution.\n * Only accepts AgentExecuteCallbacks (no streaming-specific callbacks like onChunk, onFinish).\n */\nexport interface AgentExecuteOptions extends AgentExecuteOptionsBase {\n  /**\n   * Callbacks for non-streaming agent execution.\n   * For streaming callbacks (onChunk, onFinish, onError, onAbort), use stream: true in AgentConfig.\n   */\n  callbacks?: AgentExecuteCallbacks;\n}\n\n/**\n * Options for streaming agent execution.\n * Accepts AgentStreamCallbacks including onChunk, onFinish, onError, and onAbort.\n */\nexport interface AgentStreamExecuteOptions extends AgentExecuteOptionsBase {\n  /**\n   * Callbacks for streaming agent execution.\n   * Includes streaming-specific callbacks: onChunk, onFinish, onError, onAbort.\n   */\n  callbacks?: AgentStreamCallbacks;\n}\nexport type AgentType =\n  | \"openai\"\n  | \"anthropic\"\n  | \"google\"\n  | \"microsoft\"\n  | \"bedrock\"\n  | \"vertex\"\n  | \"azure\";\n\nexport const AVAILABLE_CUA_MODELS = [\n  \"openai/gpt-5.4\",\n  \"openai/gpt-5.4-mini\",\n  \"openai/gpt-5.5\",\n  \"openai/computer-use-preview\",\n  \"openai/computer-use-preview-2025-03-11\",\n  \"anthropic/claude-opus-4-5-20251101\",\n  \"anthropic/claude-opus-4-6\",\n  \"anthropic/claude-opus-4-8\",\n  \"anthropic/claude-sonnet-4-6\",\n  \"anthropic/claude-haiku-4-5\",\n  \"anthropic/claude-haiku-4-5-20251001\",\n  \"anthropic/claude-sonnet-4-20250514\",\n  \"anthropic/claude-sonnet-4-5-20250929\",\n  \"anthropic/claude-fable-5\",\n  \"google/gemini-2.5-computer-use-preview-10-2025\",\n  \"google/gemini-3-flash-preview\",\n  \"google/gemini-3.5-flash\",\n  \"google/gemini-3-pro-preview\",\n  \"microsoft/fara-7b\",\n] as const;\nexport type AvailableCuaModel = (typeof AVAILABLE_CUA_MODELS)[number];\n\nexport interface AgentExecutionOptions<\n  TOptions extends AgentExecuteOptions = AgentExecuteOptions,\n> {\n  options: TOptions;\n  logger: (message: LogLine) => void;\n  retries?: number;\n}\n\nexport interface AgentHandlerOptions {\n  modelName: string;\n  clientOptions?: ClientOptions;\n  userProvidedInstructions?: string;\n  experimental?: boolean;\n}\n\nexport interface ActionExecutionResult {\n  success: boolean;\n  error?: string;\n  data?: unknown;\n}\n\n/**\n * Represents a safety check that requires user confirmation before proceeding.\n * These are issued by CUA providers (OpenAI, Google) when the agent attempts\n * potentially risky actions.\n */\nexport interface SafetyCheck {\n  /** Unique identifier for this safety check */\n  id: string;\n  /** Code identifying the type of safety concern */\n  code: string;\n  /** Human-readable description of the safety concern */\n  message: string;\n}\n\n/**\n * Response from the user for a safety confirmation request.\n */\nexport interface SafetyConfirmationResponse {\n  /** Whether the user acknowledged/approved the safety checks */\n  acknowledged: boolean;\n}\n\n/**\n * Callback for handling safety confirmation requests.\n * Called when the CUA provider issues safety checks that require user confirmation.\n * The callback should return a promise that resolves when the user has made a decision.\n *\n * @param safetyChecks - Array of safety checks requiring confirmation\n * @returns Promise resolving to the user's response\n *\n * @example\n * ```typescript\n * const agent = stagehand.agent({\n *   mode: \"cua\",\n * });\n * await agent.execute({\n *   instruction: \"...\",\n *   callbacks: {\n *     onSafetyConfirmation: async (checks) => {\n *       console.log(\"Safety checks:\", checks);\n *       const userApproved = await showConfirmationDialog(checks);\n *       return { acknowledged: userApproved };\n *     },\n *   },\n * });\n * ```\n */\nexport type SafetyConfirmationHandler = (\n  safetyChecks: SafetyCheck[],\n) => Promise<SafetyConfirmationResponse>;\n\n// Anthropic types:\n\nexport interface ToolUseItem extends ResponseItem {\n  type: \"tool_use\";\n  id: string; // This is the correct property name from Anthropic's API\n  name: string; // Name of the tool being used\n  input: Record<string, unknown>;\n}\n\nexport interface AnthropicMessage {\n  role: string;\n  content: string | Array<AnthropicContentBlock>;\n}\n\nexport interface AnthropicContentBlock {\n  type: string;\n  [key: string]: unknown;\n}\n\nexport interface AnthropicTextBlock extends AnthropicContentBlock {\n  type: \"text\";\n  text: string;\n}\n\nexport interface AnthropicToolResult {\n  type: \"tool_result\";\n  tool_use_id: string;\n  content: string | Array<AnthropicContentBlock>;\n}\n\n// OpenAI types:\n\nexport interface ResponseItem {\n  type: string;\n  id: string;\n  [key: string]: unknown;\n}\n\nexport interface ComputerCallItem extends ResponseItem {\n  type: \"computer_call\";\n  call_id: string;\n  action?: {\n    type: string;\n    [key: string]: unknown;\n  };\n  actions?: Array<{\n    type: string;\n    [key: string]: unknown;\n  }>;\n  pending_safety_checks?: Array<{\n    id: string;\n    code: string;\n    message: string;\n  }>;\n}\n\nexport interface FunctionCallItem extends ResponseItem {\n  type: \"function_call\";\n  call_id: string;\n  name: string;\n  arguments: string;\n}\n\nexport type ResponseInputItem =\n  | { role: string; content: string }\n  | {\n      type: \"computer_call_output\";\n      call_id: string;\n      output:\n        | {\n            type: \"input_image\" | \"computer_screenshot\";\n            image_url: string;\n            detail?: \"original\" | \"high\" | \"low\";\n            current_url?: string;\n            error?: string;\n            [key: string]: unknown;\n          }\n        | string;\n      acknowledged_safety_checks?: Array<{\n        id: string;\n        code: string;\n        message: string;\n      }>;\n    }\n  | {\n      type: \"function_call_output\";\n      call_id: string;\n      output: string;\n    };\n\nexport interface AgentInstance {\n  execute: (\n    instructionOrOptions: string | AgentExecuteOptions,\n  ) => Promise<AgentResult>;\n}\n\nexport type AgentProviderType = AgentType;\n\nexport type AgentModelConfig<TModelName extends string = string> = {\n  modelName: TModelName;\n} & Record<string, unknown>;\n\n/**\n * Agent tool mode determines which set of tools are available to the agent.\n * - 'dom': Uses DOM-based tools (act, fillForm) - better for structured page interactions\n * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, etc.) - better for visual/screenshot-based interactions\n * - 'cua': Uses Computer Use Agent (CUA) providers like Anthropic Claude or Google Gemini for screenshot-based automation\n */\nexport type AgentToolMode = \"dom\" | \"hybrid\" | \"cua\";\n\nexport type AgentConfig = {\n  /**\n   * Custom system prompt to provide to the agent. Overrides the default system prompt.\n   */\n  systemPrompt?: string;\n  /**\n   * MCP integrations - Array of Client objects\n   */\n  integrations?: (Client | string)[];\n  /**\n   * Tools passed to the agent client\n   */\n  tools?: ToolSet;\n  /**\n   * @deprecated Use `mode: \"cua\"` instead. This option will be removed in a future version.\n   * Enables Computer Use Agent (CUA) mode.\n   */\n  cua?: boolean;\n  /**\n   * The model to use for agent functionality\n   */\n  model?: string | AgentModelConfig<string>;\n  /**\n   * The model to use for tool execution (observe/act calls within agent tools).\n   * If not specified, inherits from the main model configuration.\n   * Format: \"provider/model\" (e.g., \"openai/gpt-4o-mini\", \"google/gemini-2.0-flash-exp\")\n   */\n  executionModel?: string | AgentModelConfig<string>;\n  /**\n   * Enable streaming mode for the agent.\n   * When true, execute() returns AgentStreamResult with textStream for incremental output.\n   * When false (default), execute() returns AgentResult after completion.\n   */\n  stream?: boolean;\n  /**\n   * Tool mode for the agent. Determines which set of tools are available.\n   * - 'dom' (default): Uses DOM-based tools (act, fillForm) for structured interactions\n   * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)\n   *             for visual/screenshot-based interactions\n   * - 'cua': Uses Computer Use Agent (CUA) providers for screenshot-based automation\n   */\n  mode?: AgentToolMode;\n};\n\n/**\n * Agent instance returned when stream: true is set in AgentConfig.\n * execute() returns a streaming result that can be consumed incrementally.\n * Accepts AgentStreamExecuteOptions with streaming-specific callbacks.\n */\nexport interface StreamingAgentInstance {\n  execute: (\n    instructionOrOptions: string | AgentStreamExecuteOptions,\n  ) => Promise<AgentStreamResult>;\n}\n\n/**\n * Agent instance returned when stream is false or not set in AgentConfig.\n * execute() returns a result after the agent completes.\n * Accepts AgentExecuteOptions with non-streaming callbacks only.\n */\nexport interface NonStreamingAgentInstance {\n  execute: (\n    instructionOrOptions: string | AgentExecuteOptions,\n  ) => Promise<AgentResult>;\n}\n\n// =============================================================================\n// Vision Action Tool Result Types\n// =============================================================================\n\n/**\n * Content item type for toModelOutput return values.\n * Used in tool definitions to return text and/or media to the model.\n */\nexport type ModelOutputContentItem =\n  | { type: \"text\"; text: string }\n  | { type: \"media\"; mediaType: string; data: string };\n\nexport interface ClickToolResult {\n  success: boolean;\n  describe?: string;\n  coordinates?: number[];\n  error?: string;\n  screenshotBase64?: string;\n}\n\nexport interface TypeToolResult {\n  success: boolean;\n  describe?: string;\n  text?: string;\n  error?: string;\n  screenshotBase64?: string;\n}\n\nexport interface DragAndDropToolResult {\n  success: boolean;\n  describe?: string;\n  error?: string;\n  screenshotBase64?: string;\n}\n\nexport interface FillFormField {\n  action: string;\n  value: string;\n  coordinates: { x: number; y: number };\n}\n\nexport interface FillFormVisionToolResult {\n  success: boolean;\n  playwrightArguments?: FillFormField[];\n  error?: string;\n  screenshotBase64?: string;\n}\n\nexport interface ScrollToolResult {\n  success: boolean;\n  message: string;\n  scrolledPixels: number;\n  error?: string;\n}\n\nexport interface ScrollVisionToolResult extends ScrollToolResult {\n  screenshotBase64?: string;\n}\n\nexport interface WaitToolResult {\n  success: boolean;\n  waited: number;\n  screenshotBase64?: string;\n  error?: string;\n}\n"]}
+{"version":3,"file":"agent.js","sourceRoot":"","sources":["../../../../../../lib/v3/types/public/agent.ts"],"names":[],"mappings":";;;AAmda,QAAA,oBAAoB,GAAG;IAClC,gBAAgB;IAChB,qBAAqB;IACrB,gBAAgB;IAChB,sBAAsB;IACtB,qBAAqB;IACrB,oBAAoB;IACpB,6BAA6B;IAC7B,wCAAwC;IACxC,oCAAoC;IACpC,2BAA2B;IAC3B,2BAA2B;IAC3B,6BAA6B;IAC7B,4BAA4B;IAC5B,qCAAqC;IACrC,oCAAoC;IACpC,sCAAsC;IACtC,0BAA0B;IAC1B,gDAAgD;IAChD,+BAA+B;IAC/B,yBAAyB;IACzB,6BAA6B;IAC7B,mBAAmB;CACX,CAAC","sourcesContent":["import type { Client } from \"@modelcontextprotocol/sdk/client/index.js\";\nimport {\n  ToolSet,\n  ModelMessage,\n  wrapLanguageModel,\n  StreamTextResult,\n  StepResult,\n  PrepareStepFunction,\n  GenerateTextOnStepFinishCallback,\n  StreamTextOnStepFinishCallback,\n  StreamTextOnErrorCallback,\n  StreamTextOnChunkCallback,\n  StreamTextOnFinishCallback,\n} from \"ai\";\nimport { LogLine } from \"./logs.js\";\nimport { ClientOptions } from \"./model.js\";\nimport { StagehandZodObject } from \"../../zodCompat.js\";\nimport type { AgentEvidenceCallback } from \"./agentEvidenceEvents.js\";\n\n// Re-export ModelMessage for consumers who want to use it for conversation continuation\nexport type { ModelMessage } from \"ai\";\n\n// Re-export Tool type for consumers who want to define custom tools\nexport type { Tool } from \"ai\";\nimport { Page as PlaywrightPage } from \"playwright-core\";\nimport { Page as PuppeteerPage } from \"puppeteer-core\";\nimport { Page as PatchrightPage } from \"patchright-core\";\nimport { Page } from \"../../understudy/page.js\";\n\n/**\n * Result of a screenshot provider: the base64-encoded image bytes plus an\n * explicit media type. Declaring the media type at the capture boundary lets\n * every CUA client pass it through instead of hardcoding or inferring it.\n */\nexport interface ScreenshotProviderResult {\n  base64: string;\n  mediaType: \"image/png\" | \"image/jpeg\";\n}\n\n// =============================================================================\n// Variable Types\n// =============================================================================\n\n/**\n * A variable value can be a simple primitive or a rich object with an optional description.\n * This unified type is shared across `act`, `agent.execute`, and other methods.\n *\n * @example Simple (backward-compatible):\n * ```typescript\n * variables: { username: \"john@example.com\" }\n * ```\n *\n * @example Rich with description (useful for agents):\n * ```typescript\n * variables: {\n *   username: { value: \"john@example.com\", description: \"The login email\" }\n * }\n * ```\n */\nexport type VariableValue =\n  | string\n  | number\n  | boolean\n  | { value: string | number | boolean; description?: string };\n\n/**\n * A collection of named variables for use in act, agent, and other methods.\n */\nexport type Variables = Record<string, VariableValue>;\n\nexport interface AgentContext {\n  options: AgentExecuteOptionsBase;\n  maxSteps: number;\n  systemPrompt: string;\n  allTools: ToolSet;\n  messages: ModelMessage[];\n  wrappedModel: ReturnType<typeof wrapLanguageModel>;\n  initialPageUrl: string;\n}\n\nexport interface AgentState {\n  collectedReasoning: string[];\n  actions: AgentAction[];\n  finalMessage: string;\n  completed: boolean;\n  currentPageUrl: string;\n}\n\nexport interface AgentAction {\n  type: string;\n  reasoning?: string;\n  taskCompleted?: boolean;\n  action?: string;\n  // Tool-specific fields\n  timeMs?: number; // wait tool\n  pageText?: string; // ariaTree tool\n  pageUrl?: string; // ariaTree tool\n  instruction?: string; // various tools\n  [key: string]: unknown;\n}\n\nexport interface AgentResult {\n  success: boolean;\n  message: string;\n  actions: AgentAction[];\n  completed: boolean;\n  metadata?: Record<string, unknown>;\n  usage?: {\n    input_tokens: number;\n    output_tokens: number;\n    reasoning_tokens?: number;\n    cached_input_tokens?: number;\n    inference_time_ms: number;\n  };\n  /**\n   * The conversation messages from this execution.\n   * Pass these to a subsequent execute() call via the `messages` option to continue the conversation.\n   * @experimental\n   */\n  messages?: ModelMessage[];\n  /**\n   * Custom output data extracted based on the `output` schema provided in execute options.\n   * Only populated if an `output` schema was provided.\n   * @experimental\n   */\n  output?: Record<string, unknown>;\n}\n\nexport type AgentStreamResult = StreamTextResult<ToolSet, never> & {\n  result: Promise<AgentResult>;\n};\n\n/**\n * Base callbacks shared between execute (non-streaming) and streaming modes.\n */\nexport interface AgentCallbacks {\n  /**\n   * Optional function called before each step to modify settings.\n   * You can change the model, tool choices, active tools, system prompt,\n   * and input messages for each step.\n   */\n  prepareStep?: PrepareStepFunction<ToolSet>;\n  /**\n   * Callback called when each step (LLM call) is finished.\n   * This is called for intermediate steps as well as the final step.\n   */\n  onStepFinish?:\n    | GenerateTextOnStepFinishCallback<ToolSet>\n    | StreamTextOnStepFinishCallback<ToolSet>;\n  /**\n   * Callback called when Stagehand captures agent-run evidence such as\n   * screenshots, completed tool/action steps, or post-action observations.\n   */\n  onEvidence?: AgentEvidenceCallback;\n}\n\n/**\n * Error message type for streaming-only callbacks used in non-streaming mode.\n * This provides a clear error message when users try to use streaming callbacks without stream: true.\n */\ntype StreamingCallbackNotAvailable =\n  \"This callback requires 'stream: true' in AgentConfig. Set stream: true to use streaming callbacks like onChunk, onFinish, onError, and onAbort.\";\n\n/**\n * Error message for safety confirmation callback misuse.\n * Safety confirmations are only available for non-streaming CUA agent executions.\n */\ntype SafetyConfirmationCallbackNotAvailable =\n  \"Safety confirmation callbacks are only available via non-streaming AgentExecuteOptions.callbacks when using mode: 'cua'.\";\n\n/**\n * Callbacks specific to the non-streaming execute method.\n */\nexport interface AgentExecuteCallbacks extends AgentCallbacks {\n  /**\n   * Callback called when each step (LLM call) is finished.\n   */\n  onStepFinish?: GenerateTextOnStepFinishCallback<ToolSet>;\n  /**\n   * Callback for handling safety confirmation requests from CUA providers.\n   * Only available when running an agent configured with mode: \"cua\".\n   */\n  onSafetyConfirmation?: SafetyConfirmationHandler;\n\n  /**\n   * NOT AVAILABLE in non-streaming mode.\n   * This callback requires `stream: true` in AgentConfig.\n   *\n   * @example\n   * ```typescript\n   * // Enable streaming to use onChunk:\n   * const agent = stagehand.agent({ stream: true });\n   * await agent.execute({\n   *   instruction: \"...\",\n   *   callbacks: { onChunk: async (chunk) => console.log(chunk) }\n   * });\n   * ```\n   */\n  onChunk?: StreamingCallbackNotAvailable;\n\n  /**\n   * NOT AVAILABLE in non-streaming mode.\n   * This callback requires `stream: true` in AgentConfig.\n   *\n   * @example\n   * ```typescript\n   * // Enable streaming to use onFinish:\n   * const agent = stagehand.agent({ stream: true });\n   * await agent.execute({\n   *   instruction: \"...\",\n   *   callbacks: { onFinish: (event) => console.log(\"Done!\", event) }\n   * });\n   * ```\n   */\n  onFinish?: StreamingCallbackNotAvailable;\n\n  /**\n   * NOT AVAILABLE in non-streaming mode.\n   * This callback requires `stream: true` in AgentConfig.\n   *\n   * @example\n   * ```typescript\n   * // Enable streaming to use onError:\n   * const agent = stagehand.agent({ stream: true });\n   * await agent.execute({\n   *   instruction: \"...\",\n   *   callbacks: { onError: ({ error }) => console.error(error) }\n   * });\n   * ```\n   */\n  onError?: StreamingCallbackNotAvailable;\n\n  /**\n   * NOT AVAILABLE in non-streaming mode.\n   * This callback requires `stream: true` in AgentConfig.\n   *\n   * @example\n   * ```typescript\n   * // Enable streaming to use onAbort:\n   * const agent = stagehand.agent({ stream: true });\n   * await agent.execute({\n   *   instruction: \"...\",\n   *   callbacks: { onAbort: (event) => console.log(\"Aborted\", event.steps) }\n   * });\n   * ```\n   */\n  onAbort?: StreamingCallbackNotAvailable;\n}\n\n/**\n * Callbacks specific to the streaming mode.\n */\nexport interface AgentStreamCallbacks extends AgentCallbacks {\n  /**\n   * Callback called when each step (LLM call) is finished during streaming.\n   */\n  onStepFinish?: StreamTextOnStepFinishCallback<ToolSet>;\n  /**\n   * Callback called when an error occurs during streaming.\n   * Use this to log errors or handle error states.\n   */\n  onError?: StreamTextOnErrorCallback;\n  /**\n   * Callback called for each chunk of the stream.\n   * Stream processing will pause until the callback promise resolves.\n   */\n  onChunk?: StreamTextOnChunkCallback<ToolSet>;\n  /**\n   * Callback called when the stream finishes.\n   */\n  onFinish?: StreamTextOnFinishCallback<ToolSet>;\n  /**\n   * Callback called when the stream is aborted.\n   */\n  onAbort?: (event: {\n    steps: Array<StepResult<ToolSet>>;\n  }) => PromiseLike<void> | void;\n  /**\n   * NOT AVAILABLE in streaming mode.\n   * Safety confirmations currently require non-streaming execute() on CUA agents.\n   */\n  onSafetyConfirmation?: SafetyConfirmationCallbackNotAvailable;\n}\n\n/**\n * Base options for agent execution (without callbacks).\n */\nexport interface AgentExecuteOptionsBase {\n  instruction: string;\n  maxSteps?: number;\n  page?: PlaywrightPage | PuppeteerPage | PatchrightPage | Page;\n  highlightCursor?: boolean;\n  /**\n   * Previous conversation messages to continue from.\n   * Pass the `messages` from a previous AgentResult to continue that conversation.\n   * @experimental\n   */\n  messages?: ModelMessage[];\n  /**\n   * An AbortSignal that can be used to cancel the agent execution.\n   * When aborted, the agent will stop and return a partial result.\n   * @experimental\n   *\n   * @example\n   * ```typescript\n   * const controller = new AbortController();\n   * setTimeout(() => controller.abort(), 30000); // 30 second timeout\n   *\n   * const result = await agent.execute({\n   *   instruction: \"...\",\n   *   signal: controller.signal\n   * });\n   * ```\n   */\n  signal?: AbortSignal;\n  /**\n   * Tools to exclude from this execution.\n   * Pass an array of tool names to prevent the agent from using those tools.\n   *\n   * **Note:** Not supported in CUA mode (`mode: \"cua\"`).\n   *\n   * **Available tools by mode:**\n   *\n   * **DOM mode (default):**\n   * - `act` - Perform semantic actions (click, type, etc.)\n   * - `fillForm` - Fill form fields using DOM selectors\n   * - `ariaTree` - Get accessibility tree of the page\n   * - `extract` - Extract structured data from page\n   * - `goto` - Navigate to a URL\n   * - `scroll` - Scroll using semantic directions (up/down/left/right)\n   * - `keys` - Press keyboard keys\n   * - `navback` - Navigate back in history\n   * - `screenshot` - Take a screenshot\n   * - `think` - Agent reasoning/planning step\n   * - `wait` - Wait for time or condition\n   * - `done` - Mark task as complete\n   * - `search` - Web search (requires useSearch: true and BROWSERBASE_API_KEY)\n   *\n   * **Hybrid mode:**\n   * - `click` - Click at specific coordinates\n   * - `type` - Type text at coordinates\n   * - `dragAndDrop` - Drag from one point to another\n   * - `clickAndHold` - Click and hold at coordinates\n   * - `fillFormVision` - Fill forms using vision/coordinates\n   * - `act` - Perform semantic actions\n   * - `ariaTree` - Get accessibility tree\n   * - `extract` - Extract data from page\n   * - `goto` - Navigate to URL\n   * - `scroll` - Scroll using coordinates\n   * - `keys` - Press keyboard keys\n   * - `navback` - Navigate back\n   * - `screenshot` - Take screenshot\n   * - `think` - Agent reasoning step\n   * - `wait` - Wait for time/condition\n   * - `done` - Mark task complete\n   * - `search` - Web search (requires useSearch: true and BROWSERBASE_API_KEY)\n   *\n   * @experimental\n   * @example\n   * ```typescript\n   * // Exclude screenshot and extract tools\n   * const result = await agent.execute({\n   *   instruction: \"Click the submit button\",\n   *   excludeTools: [\"screenshot\", \"extract\"]\n   * });\n   * ```\n   */\n  excludeTools?: string[];\n  /**\n   * A Zod schema defining custom output data to return when the task completes.\n   * The agent will populate this data in the final done tool call.\n   *\n   * @experimental\n   * @example\n   * ```typescript\n   * const result = await agent.execute({\n   *   instruction: \"Find the cheapest flight from NYC to LA\",\n   *   output: z.object({\n   *     price: z.string().describe(\"The price of the flight\"),\n   *     airline: z.string().describe(\"The airline name\"),\n   *     departureTime: z.string().describe(\"Departure time\"),\n   *   }),\n   * });\n   *\n   * console.log(result.output); // { price: \"$199\", airline: \"Delta\", departureTime: \"8:00 AM\" }\n   * ```\n   */\n  output?: StagehandZodObject;\n  /**\n   * Variables that the agent can use when filling forms or typing text.\n   * The agent will see variable names and descriptions in the system prompt,\n   * and can use them via `%variableName%` syntax in act/type/fillForm tool calls.\n   *\n   * Accepts both simple values and rich objects with descriptions (same type as `act`).\n   *\n   * **Note:** Not supported in CUA mode (`mode: \"cua\"`).\n   *\n   * @example\n   * ```typescript\n   * // Simple values\n   * variables: { username: \"john@example.com\", password: \"secret123\" }\n   *\n   * // Rich values with descriptions (helps the agent understand context)\n   * variables: {\n   *   username: { value: \"john@example.com\", description: \"The login email\" },\n   *   password: { value: \"secret123\", description: \"The login password\" },\n   * }\n   * ```\n   */\n  variables?: Variables;\n  /**\n   * Timeout in milliseconds for each agent tool call.\n   * If a tool call exceeds this duration, it will be aborted and\n   * reported back to the LLM as a timeout error so it can retry or adjust.\n   * For tools that call v3 methods (act, extract, fillForm, ariaTree), the\n   * timeout is also forwarded to the underlying v3 call for true cancellation.\n   * @default 45000 (45 seconds)\n   */\n  toolTimeout?: number;\n  /**\n   * Enable the web search tool powered by Browserbase Search API.\n   * Requires a valid Browserbase API key (BROWSERBASE_API_KEY).\n   * When set to true, the agent gains access to a `search` tool for web searches.\n   *\n   * @example\n   * ```typescript\n   * const result = await agent.execute({\n   *   instruction: \"Find the latest news about AI\",\n   *   useSearch: true,\n   * });\n   * ```\n   */\n  useSearch?: boolean;\n}\n\n/**\n * Options for non-streaming agent execution.\n * Only accepts AgentExecuteCallbacks (no streaming-specific callbacks like onChunk, onFinish).\n */\nexport interface AgentExecuteOptions extends AgentExecuteOptionsBase {\n  /**\n   * Callbacks for non-streaming agent execution.\n   * For streaming callbacks (onChunk, onFinish, onError, onAbort), use stream: true in AgentConfig.\n   */\n  callbacks?: AgentExecuteCallbacks;\n}\n\n/**\n * Options for streaming agent execution.\n * Accepts AgentStreamCallbacks including onChunk, onFinish, onError, and onAbort.\n */\nexport interface AgentStreamExecuteOptions extends AgentExecuteOptionsBase {\n  /**\n   * Callbacks for streaming agent execution.\n   * Includes streaming-specific callbacks: onChunk, onFinish, onError, onAbort.\n   */\n  callbacks?: AgentStreamCallbacks;\n}\nexport type AgentType =\n  | \"openai\"\n  | \"anthropic\"\n  | \"google\"\n  | \"microsoft\"\n  | \"bedrock\"\n  | \"vertex\"\n  | \"azure\";\n\nexport const AVAILABLE_CUA_MODELS = [\n  \"openai/gpt-5.4\",\n  \"openai/gpt-5.4-mini\",\n  \"openai/gpt-5.5\",\n  \"openai/gpt-5.6-terra\",\n  \"openai/gpt-5.6-luna\",\n  \"openai/gpt-5.6-sol\",\n  \"openai/computer-use-preview\",\n  \"openai/computer-use-preview-2025-03-11\",\n  \"anthropic/claude-opus-4-5-20251101\",\n  \"anthropic/claude-opus-4-6\",\n  \"anthropic/claude-opus-4-8\",\n  \"anthropic/claude-sonnet-4-6\",\n  \"anthropic/claude-haiku-4-5\",\n  \"anthropic/claude-haiku-4-5-20251001\",\n  \"anthropic/claude-sonnet-4-20250514\",\n  \"anthropic/claude-sonnet-4-5-20250929\",\n  \"anthropic/claude-fable-5\",\n  \"google/gemini-2.5-computer-use-preview-10-2025\",\n  \"google/gemini-3-flash-preview\",\n  \"google/gemini-3.5-flash\",\n  \"google/gemini-3-pro-preview\",\n  \"microsoft/fara-7b\",\n] as const;\nexport type AvailableCuaModel = (typeof AVAILABLE_CUA_MODELS)[number];\n\nexport interface AgentExecutionOptions<\n  TOptions extends AgentExecuteOptions = AgentExecuteOptions,\n> {\n  options: TOptions;\n  logger: (message: LogLine) => void;\n  retries?: number;\n}\n\nexport interface AgentHandlerOptions {\n  modelName: string;\n  clientOptions?: ClientOptions;\n  userProvidedInstructions?: string;\n  experimental?: boolean;\n}\n\nexport interface ActionExecutionResult {\n  success: boolean;\n  error?: string;\n  data?: unknown;\n}\n\n/**\n * Represents a safety check that requires user confirmation before proceeding.\n * These are issued by CUA providers (OpenAI, Google) when the agent attempts\n * potentially risky actions.\n */\nexport interface SafetyCheck {\n  /** Unique identifier for this safety check */\n  id: string;\n  /** Code identifying the type of safety concern */\n  code: string;\n  /** Human-readable description of the safety concern */\n  message: string;\n}\n\n/**\n * Response from the user for a safety confirmation request.\n */\nexport interface SafetyConfirmationResponse {\n  /** Whether the user acknowledged/approved the safety checks */\n  acknowledged: boolean;\n}\n\n/**\n * Callback for handling safety confirmation requests.\n * Called when the CUA provider issues safety checks that require user confirmation.\n * The callback should return a promise that resolves when the user has made a decision.\n *\n * @param safetyChecks - Array of safety checks requiring confirmation\n * @returns Promise resolving to the user's response\n *\n * @example\n * ```typescript\n * const agent = stagehand.agent({\n *   mode: \"cua\",\n * });\n * await agent.execute({\n *   instruction: \"...\",\n *   callbacks: {\n *     onSafetyConfirmation: async (checks) => {\n *       console.log(\"Safety checks:\", checks);\n *       const userApproved = await showConfirmationDialog(checks);\n *       return { acknowledged: userApproved };\n *     },\n *   },\n * });\n * ```\n */\nexport type SafetyConfirmationHandler = (\n  safetyChecks: SafetyCheck[],\n) => Promise<SafetyConfirmationResponse>;\n\n// Anthropic types:\n\nexport interface ToolUseItem extends ResponseItem {\n  type: \"tool_use\";\n  id: string; // This is the correct property name from Anthropic's API\n  name: string; // Name of the tool being used\n  input: Record<string, unknown>;\n}\n\nexport interface AnthropicMessage {\n  role: string;\n  content: string | Array<AnthropicContentBlock>;\n}\n\nexport interface AnthropicContentBlock {\n  type: string;\n  [key: string]: unknown;\n}\n\nexport interface AnthropicTextBlock extends AnthropicContentBlock {\n  type: \"text\";\n  text: string;\n}\n\nexport interface AnthropicToolResult {\n  type: \"tool_result\";\n  tool_use_id: string;\n  content: string | Array<AnthropicContentBlock>;\n}\n\n// OpenAI types:\n\nexport interface ResponseItem {\n  type: string;\n  id: string;\n  [key: string]: unknown;\n}\n\nexport interface ComputerCallItem extends ResponseItem {\n  type: \"computer_call\";\n  call_id: string;\n  action?: {\n    type: string;\n    [key: string]: unknown;\n  };\n  actions?: Array<{\n    type: string;\n    [key: string]: unknown;\n  }>;\n  pending_safety_checks?: Array<{\n    id: string;\n    code: string;\n    message: string;\n  }>;\n}\n\nexport interface FunctionCallItem extends ResponseItem {\n  type: \"function_call\";\n  call_id: string;\n  name: string;\n  arguments: string;\n}\n\nexport type ResponseInputItem =\n  | { role: string; content: string }\n  | {\n      type: \"computer_call_output\";\n      call_id: string;\n      output:\n        | {\n            type: \"input_image\" | \"computer_screenshot\";\n            image_url: string;\n            detail?: \"original\" | \"high\" | \"low\";\n            current_url?: string;\n            error?: string;\n            [key: string]: unknown;\n          }\n        | string;\n      acknowledged_safety_checks?: Array<{\n        id: string;\n        code: string;\n        message: string;\n      }>;\n    }\n  | {\n      type: \"function_call_output\";\n      call_id: string;\n      output: string;\n    };\n\nexport interface AgentInstance {\n  execute: (\n    instructionOrOptions: string | AgentExecuteOptions,\n  ) => Promise<AgentResult>;\n}\n\nexport type AgentProviderType = AgentType;\n\nexport type AgentModelConfig<TModelName extends string = string> = {\n  modelName: TModelName;\n} & Record<string, unknown>;\n\n/**\n * Agent tool mode determines which set of tools are available to the agent.\n * - 'dom': Uses DOM-based tools (act, fillForm) - better for structured page interactions\n * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, etc.) - better for visual/screenshot-based interactions\n * - 'cua': Uses Computer Use Agent (CUA) providers like Anthropic Claude or Google Gemini for screenshot-based automation\n */\nexport type AgentToolMode = \"dom\" | \"hybrid\" | \"cua\";\n\nexport type AgentConfig = {\n  /**\n   * Custom system prompt to provide to the agent. Overrides the default system prompt.\n   */\n  systemPrompt?: string;\n  /**\n   * MCP integrations - Array of Client objects\n   */\n  integrations?: (Client | string)[];\n  /**\n   * Tools passed to the agent client\n   */\n  tools?: ToolSet;\n  /**\n   * @deprecated Use `mode: \"cua\"` instead. This option will be removed in a future version.\n   * Enables Computer Use Agent (CUA) mode.\n   */\n  cua?: boolean;\n  /**\n   * The model to use for agent functionality\n   */\n  model?: string | AgentModelConfig<string>;\n  /**\n   * The model to use for tool execution (observe/act calls within agent tools).\n   * If not specified, inherits from the main model configuration.\n   * Format: \"provider/model\" (e.g., \"openai/gpt-4o-mini\", \"google/gemini-2.0-flash-exp\")\n   */\n  executionModel?: string | AgentModelConfig<string>;\n  /**\n   * Enable streaming mode for the agent.\n   * When true, execute() returns AgentStreamResult with textStream for incremental output.\n   * When false (default), execute() returns AgentResult after completion.\n   */\n  stream?: boolean;\n  /**\n   * Tool mode for the agent. Determines which set of tools are available.\n   * - 'dom' (default): Uses DOM-based tools (act, fillForm) for structured interactions\n   * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)\n   *             for visual/screenshot-based interactions\n   * - 'cua': Uses Computer Use Agent (CUA) providers for screenshot-based automation\n   */\n  mode?: AgentToolMode;\n};\n\n/**\n * Agent instance returned when stream: true is set in AgentConfig.\n * execute() returns a streaming result that can be consumed incrementally.\n * Accepts AgentStreamExecuteOptions with streaming-specific callbacks.\n */\nexport interface StreamingAgentInstance {\n  execute: (\n    instructionOrOptions: string | AgentStreamExecuteOptions,\n  ) => Promise<AgentStreamResult>;\n}\n\n/**\n * Agent instance returned when stream is false or not set in AgentConfig.\n * execute() returns a result after the agent completes.\n * Accepts AgentExecuteOptions with non-streaming callbacks only.\n */\nexport interface NonStreamingAgentInstance {\n  execute: (\n    instructionOrOptions: string | AgentExecuteOptions,\n  ) => Promise<AgentResult>;\n}\n\n// =============================================================================\n// Vision Action Tool Result Types\n// =============================================================================\n\n/**\n * Content item type for toModelOutput return values.\n * Used in tool definitions to return text and/or media to the model.\n */\nexport type ModelOutputContentItem =\n  | { type: \"text\"; text: string }\n  | { type: \"media\"; mediaType: string; data: string };\n\nexport interface ClickToolResult {\n  success: boolean;\n  describe?: string;\n  coordinates?: number[];\n  error?: string;\n  screenshotBase64?: string;\n}\n\nexport interface TypeToolResult {\n  success: boolean;\n  describe?: string;\n  text?: string;\n  error?: string;\n  screenshotBase64?: string;\n}\n\nexport interface DragAndDropToolResult {\n  success: boolean;\n  describe?: string;\n  error?: string;\n  screenshotBase64?: string;\n}\n\nexport interface FillFormField {\n  action: string;\n  value: string;\n  coordinates: { x: number; y: number };\n}\n\nexport interface FillFormVisionToolResult {\n  success: boolean;\n  playwrightArguments?: FillFormField[];\n  error?: string;\n  screenshotBase64?: string;\n}\n\nexport interface ScrollToolResult {\n  success: boolean;\n  message: string;\n  scrolledPixels: number;\n  error?: string;\n}\n\nexport interface ScrollVisionToolResult extends ScrollToolResult {\n  screenshotBase64?: string;\n}\n\nexport interface WaitToolResult {\n  success: boolean;\n  waited: number;\n  screenshotBase64?: string;\n  error?: string;\n}\n"]}

package/dist/cjs/lib/v3/types/public/api.d.ts

@@ -151,6 +151,10 @@ export declare const GenericModelConfigObjectSchema: z.ZodObject<{
     apiKey: z.ZodOptional<z.ZodString>;
     baseURL: z.ZodOptional<z.ZodString>;
     headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    openaiEndpointFormat: z.ZodOptional<z.ZodEnum<{
+        responses: "responses";
+        chat: "chat";
+    }>>;
     provider: z.ZodOptional<z.ZodEnum<{
         openai: "openai";
         anthropic: "anthropic";
@@ -333,6 +337,10 @@ export declare const ModelConfigObjectSchema: z.ZodUnion<readonly [z.ZodObject<{
     apiKey: z.ZodOptional<z.ZodString>;
     baseURL: z.ZodOptional<z.ZodString>;
     headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    openaiEndpointFormat: z.ZodOptional<z.ZodEnum<{
+        responses: "responses";
+        chat: "chat";
+    }>>;
     provider: z.ZodOptional<z.ZodEnum<{
         openai: "openai";
         anthropic: "anthropic";
@@ -413,6 +421,10 @@ export declare const ModelConfigSchema: z.ZodUnion<readonly [z.ZodObject<{
     apiKey: z.ZodOptional<z.ZodString>;
     baseURL: z.ZodOptional<z.ZodString>;
     headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    openaiEndpointFormat: z.ZodOptional<z.ZodEnum<{
+        responses: "responses";
+        chat: "chat";
+    }>>;
     provider: z.ZodOptional<z.ZodEnum<{
         openai: "openai";
         anthropic: "anthropic";
@@ -852,6 +864,7 @@ export declare const SessionStartRequestSchema: z.ZodObject<{
     experimental: z.ZodOptional<z.ZodBoolean>;
     waitForCaptchaSolves: z.ZodOptional<z.ZodBoolean>;
     actTimeoutMs: z.ZodOptional<z.ZodNumber>;
+    useTouch: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$strip>;
 export declare const SessionStartResultSchema: z.ZodObject<{
     sessionId: z.ZodString;
@@ -945,6 +958,10 @@ export declare const ActOptionsSchema: z.ZodOptional<z.ZodObject<{
         apiKey: z.ZodOptional<z.ZodString>;
         baseURL: z.ZodOptional<z.ZodString>;
         headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        openaiEndpointFormat: z.ZodOptional<z.ZodEnum<{
+            responses: "responses";
+            chat: "chat";
+        }>>;
         provider: z.ZodOptional<z.ZodEnum<{
             openai: "openai";
             anthropic: "anthropic";
@@ -1036,6 +1053,10 @@ export declare const ActRequestSchema: z.ZodObject<{
             apiKey: z.ZodOptional<z.ZodString>;
             baseURL: z.ZodOptional<z.ZodString>;
             headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+            openaiEndpointFormat: z.ZodOptional<z.ZodEnum<{
+                responses: "responses";
+                chat: "chat";
+            }>>;
             provider: z.ZodOptional<z.ZodEnum<{
                 openai: "openai";
                 anthropic: "anthropic";
@@ -1168,6 +1189,10 @@ export declare const ExtractOptionsSchema: z.ZodOptional<z.ZodObject<{
         apiKey: z.ZodOptional<z.ZodString>;
         baseURL: z.ZodOptional<z.ZodString>;
         headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        openaiEndpointFormat: z.ZodOptional<z.ZodEnum<{
+            responses: "responses";
+            chat: "chat";
+        }>>;
         provider: z.ZodOptional<z.ZodEnum<{
             openai: "openai";
             anthropic: "anthropic";
@@ -1256,6 +1281,10 @@ export declare const ExtractRequestSchema: z.ZodObject<{
             apiKey: z.ZodOptional<z.ZodString>;
             baseURL: z.ZodOptional<z.ZodString>;
             headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+            openaiEndpointFormat: z.ZodOptional<z.ZodEnum<{
+                responses: "responses";
+                chat: "chat";
+            }>>;
             provider: z.ZodOptional<z.ZodEnum<{
                 openai: "openai";
                 anthropic: "anthropic";
@@ -1355,6 +1384,10 @@ export declare const ObserveOptionsSchema: z.ZodOptional<z.ZodObject<{
         apiKey: z.ZodOptional<z.ZodString>;
         baseURL: z.ZodOptional<z.ZodString>;
         headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        openaiEndpointFormat: z.ZodOptional<z.ZodEnum<{
+            responses: "responses";
+            chat: "chat";
+        }>>;
         provider: z.ZodOptional<z.ZodEnum<{
             openai: "openai";
             anthropic: "anthropic";
@@ -1442,6 +1475,10 @@ export declare const ObserveRequestSchema: z.ZodObject<{
             apiKey: z.ZodOptional<z.ZodString>;
             baseURL: z.ZodOptional<z.ZodString>;
             headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+            openaiEndpointFormat: z.ZodOptional<z.ZodEnum<{
+                responses: "responses";
+                chat: "chat";
+            }>>;
             provider: z.ZodOptional<z.ZodEnum<{
                 openai: "openai";
                 anthropic: "anthropic";
@@ -1560,6 +1597,10 @@ export declare const AgentConfigSchema: z.ZodObject<{
         apiKey: z.ZodOptional<z.ZodString>;
         baseURL: z.ZodOptional<z.ZodString>;
         headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        openaiEndpointFormat: z.ZodOptional<z.ZodEnum<{
+            responses: "responses";
+            chat: "chat";
+        }>>;
         provider: z.ZodOptional<z.ZodEnum<{
             openai: "openai";
             anthropic: "anthropic";
@@ -1646,6 +1687,10 @@ export declare const AgentConfigSchema: z.ZodObject<{
         apiKey: z.ZodOptional<z.ZodString>;
         baseURL: z.ZodOptional<z.ZodString>;
         headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        openaiEndpointFormat: z.ZodOptional<z.ZodEnum<{
+            responses: "responses";
+            chat: "chat";
+        }>>;
         provider: z.ZodOptional<z.ZodEnum<{
             openai: "openai";
             anthropic: "anthropic";
@@ -1790,6 +1835,10 @@ export declare const AgentExecuteRequestSchema: z.ZodObject<{
             apiKey: z.ZodOptional<z.ZodString>;
             baseURL: z.ZodOptional<z.ZodString>;
             headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+            openaiEndpointFormat: z.ZodOptional<z.ZodEnum<{
+                responses: "responses";
+                chat: "chat";
+            }>>;
             provider: z.ZodOptional<z.ZodEnum<{
                 openai: "openai";
                 anthropic: "anthropic";
@@ -1876,6 +1925,10 @@ export declare const AgentExecuteRequestSchema: z.ZodObject<{
             apiKey: z.ZodOptional<z.ZodString>;
             baseURL: z.ZodOptional<z.ZodString>;
             headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+            openaiEndpointFormat: z.ZodOptional<z.ZodEnum<{
+                responses: "responses";
+                chat: "chat";
+            }>>;
             provider: z.ZodOptional<z.ZodEnum<{
                 openai: "openai";
                 anthropic: "anthropic";

package/dist/cjs/lib/v3/types/public/api.js

@@ -191,6 +191,10 @@ const ModelConfigBaseSchema = v4_1.z
 })
     .strict();
 exports.GenericModelConfigObjectSchema = ModelConfigBaseSchema.extend({
+    openaiEndpointFormat: v4_1.z.enum(["responses", "chat"]).optional().meta({
+        description: "Wire format used by an OpenAI-compatible endpoint. Defaults to the Responses API; use chat for Chat Completions-only endpoints.",
+        example: "chat",
+    }),
     provider: v4_1.z
         .enum(["openai", "anthropic", "google", "microsoft", "bedrock"])
         .optional()
@@ -498,6 +502,10 @@ exports.SessionStartRequestSchema = v4_1.z
     actTimeoutMs: v4_1.z.number().optional().meta({
         description: "Timeout in ms for act operations (deprecated, v2 only)",
     }),
+    useTouch: v4_1.z.boolean().optional().meta({
+        description: "Actuate coordinate pointer actions as trusted touch taps instead of mouse input (for sessions rendering a mobile layout)",
+        example: false,
+    }),
 })
     .meta({ id: "SessionStartRequest" });
 exports.SessionStartResultSchema = v4_1.z