npm - @browserbasehq/orca - Versions diffs - 3.4.0-preview.0 → 3.5.0-preview.0 - Mend

@browserbasehq/orca 3.4.0-preview.0 → 3.5.0-preview.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (829) hide show

package/dist/esm/lib/v3/types/private/snapshot.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"snapshot.js","sourceRoot":"","sources":["../../../../../../lib/v3/types/private/snapshot.ts"],"names":[],"mappings":"","sourcesContent":["/*\n Options that control how hybrid snapshots and targeted scopes are captured.\n /\nexport type SnapshotOptions = {\n /\n Filter the snapshot to a specific element/subtree using a selector that can cross iframes.\n * Supports XPath (prefixed with `xpath=` or starting with `/`) and CSS with iframe hops via `>>`.\n /\n focusSelector?: string;\n /\n Exclude matching elements and their descendants from the captured snapshot.\n * Each selector may be XPath (prefixed with `xpath=` or starting with `/`) or CSS.\n /\n ignoreSelectors?: string[];\n /\n Pierce shadow DOM when calling DOM.getDocument. Defaults to true to retain the\n * existing behaviour.\n /\n pierceShadow?: boolean;\n /\n Toggle whether iframe subtrees are included in the merged snapshot. Defaults to true.\n /\n includeIframes?: boolean;\n /\n Optional feature flag that surfaces experimental traversal tweaks in the Accessibility layer.\n /\n experimental?: boolean;\n};\n\n/\n Hybrid snapshot payload consumed by act/extract/observe handlers.\n /\nexport type HybridSnapshot = {\n /* Merged outline across every frame. /\n combinedTree: string;\n /* EncodedId (frameOrdinal-backendNodeId) -> absolute XPath. /\n combinedXpathMap: Record<string, string>;\n /* EncodedId -> URL extracted from AX properties. /\n combinedUrlMap: Record<string, string>;\n /* Per-frame payloads expose the original relative data for debugging. /\n perFrame?: PerFrameSnapshot[];\n};\n\nexport type PerFrameSnapshot = {\n frameId: string;\n outline: string;\n xpathMap: Record<string, string>;\n urlMap: Record<string, string>;\n};\n\n/\n Compact encoding of DOM data for an entire session. Shared between capture\n * and focus helpers so DOM traversal can be unit tested in isolation.\n /\nexport type SessionDomIndex = {\n rootBackend: number;\n absByBe: Map<number, string>;\n tagByBe: Map<number, string>;\n scrollByBe: Map<number, boolean>;\n docRootOf: Map<number, number>;\n contentDocRootByIframe: Map<number, number>;\n enterByBe: Map<number, number>;\n exitByBe: Map<number, number>;\n};\n\nexport type FrameDomMaps = {\n tagNameMap: Record<string, string>;\n xpathMap: Record<string, string>;\n scrollableMap: Record<string, boolean>;\n urlMap: Record<string, string>;\n};\n\nexport type ResolvedLocation = {\n frameId: string;\n backendNodeId: number;\n absoluteXPath: string;\n};\n\nexport type ResolvedFocusFrame = {\n targetFrameId: string;\n tailXPath: string;\n absPrefix: string;\n};\n\nexport type ResolvedCssFocus = {\n targetFrameId: string;\n tailSelector: string;\n absPrefix: string;\n};\n\nexport type Axis = \"child\" \| \"desc\";\n\nexport type Step = {\n axis: Axis;\n raw: string;\n name: string;\n};\n\nexport type A11yNode = {\n role: string;\n name?: string;\n description?: string;\n value?: string \| number \| boolean;\n nodeId: string;\n backendDOMNodeId?: number;\n parentId?: string;\n childIds?: string[];\n children?: A11yNode[];\n encodedId?: string;\n};\n\nexport type A11yOptions = {\n focusSelector?: string;\n isIgnoredBackendNode?: (backendNodeId: number) => boolean;\n experimental: boolean;\n tagNameMap: Record<string, string>;\n scrollableMap: Record<string, boolean>;\n encode: (backendNodeId: number) => string;\n};\n\nexport type AccessibilityTreeResult = {\n outline: string;\n urlMap: Record<string, string>;\n scopeApplied: boolean;\n};\n\nexport type FrameParentIndex = Map<string, string \| null>;\n\n/\n Shared frame metadata that every snapshot step needs.\n * - `rootId`: stable identifier for the main frame so we can detect root prefixes.\n * - `parentByFrame`: lookup table for iframe parentage (used by focus scoping and prefixing).\n * - `frames`: DFS-ordered frame ids so merging walks parents before children.\n */\nexport type FrameContext = {\n rootId: string;\n parentByFrame: FrameParentIndex;\n frames: string[];\n};\n"]}
1	+ {"version":3,"file":"snapshot.js","sourceRoot":"","sources":["../../../../../../lib/v3/types/private/snapshot.ts"],"names":[],"mappings":"","sourcesContent":["/*\n Options that control how hybrid snapshots and targeted scopes are captured.\n /\nexport type SnapshotOptions = {\n /\n Filter the snapshot to a specific element/subtree using a selector that can cross iframes.\n * Supports XPath (prefixed with `xpath=` or starting with `/`) and CSS with iframe hops via `>>`.\n /\n focusSelector?: string;\n /\n Exclude matching elements and their descendants from the captured snapshot.\n * Each selector may be XPath (prefixed with `xpath=` or starting with `/`) or CSS.\n /\n ignoreSelectors?: string[];\n /\n Pierce shadow DOM when calling DOM.getDocument. Defaults to true to retain the\n * existing behaviour.\n /\n pierceShadow?: boolean;\n /\n Toggle whether iframe subtrees are included in the merged snapshot. Defaults to true.\n /\n includeIframes?: boolean;\n /\n Optional feature flag that surfaces experimental traversal tweaks in the Accessibility layer.\n /\n experimental?: boolean;\n};\n\n/\n Hybrid snapshot payload consumed by act/extract/observe handlers.\n /\nexport type HybridSnapshot = {\n /* Merged outline across every frame. /\n combinedTree: string;\n /* EncodedId (frameOrdinal-backendNodeId) -> absolute XPath. /\n combinedXpathMap: Record<string, string>;\n /* EncodedId -> URL extracted from AX properties. /\n combinedUrlMap: Record<string, string>;\n /* Per-frame payloads expose the original relative data for debugging. /\n perFrame?: PerFrameSnapshot[];\n};\n\nexport type PerFrameSnapshot = {\n frameId: string;\n outline: string;\n xpathMap: Record<string, string>;\n urlMap: Record<string, string>;\n};\n\n/\n Compact encoding of DOM data for an entire session. Shared between capture\n * and focus helpers so DOM traversal can be unit tested in isolation.\n /\nexport type SessionDomIndex = {\n rootBackend: number;\n absByBe: Map<number, string>;\n tagByBe: Map<number, string>;\n scrollByBe: Map<number, boolean>;\n docRootOf: Map<number, number>;\n contentDocRootByIframe: Map<number, number>;\n enterByBe: Map<number, number>;\n exitByBe: Map<number, number>;\n};\n\nexport type FrameDomMaps = {\n tagNameMap: Record<string, string>;\n xpathMap: Record<string, string>;\n scrollableMap: Record<string, boolean>;\n urlMap: Record<string, string>;\n};\n\nexport type ResolvedLocation = {\n frameId: string;\n backendNodeId: number;\n absoluteXPath: string;\n};\n\nexport type ResolvedFocusFrame = {\n targetFrameId: string;\n tailXPath: string;\n absPrefix: string;\n};\n\nexport type ResolvedCssFocus = {\n targetFrameId: string;\n tailSelector: string;\n absPrefix: string;\n};\n\nexport type Axis = \"child\" \| \"desc\";\n\nexport type Step = {\n axis: Axis;\n raw: string;\n name: string;\n};\n\nexport type A11yNode = {\n role: string;\n name?: string;\n description?: string;\n value?: string \| number \| boolean;\n selected?: boolean;\n checked?: boolean;\n nodeId: string;\n backendDOMNodeId?: number;\n parentId?: string;\n childIds?: string[];\n children?: A11yNode[];\n encodedId?: string;\n};\n\nexport type A11yOptions = {\n focusSelector?: string;\n isIgnoredBackendNode?: (backendNodeId: number) => boolean;\n experimental: boolean;\n tagNameMap: Record<string, string>;\n scrollableMap: Record<string, boolean>;\n encode: (backendNodeId: number) => string;\n};\n\nexport type AccessibilityTreeResult = {\n outline: string;\n urlMap: Record<string, string>;\n scopeApplied: boolean;\n};\n\nexport type FrameParentIndex = Map<string, string \| null>;\n\n/\n Shared frame metadata that every snapshot step needs.\n * - `rootId`: stable identifier for the main frame so we can detect root prefixes.\n * - `parentByFrame`: lookup table for iframe parentage (used by focus scoping and prefixing).\n * - `frames`: DFS-ordered frame ids so merging walks parents before children.\n */\nexport type FrameContext = {\n rootId: string;\n parentByFrame: FrameParentIndex;\n frames: string[];\n};\n"]}

package/dist/esm/lib/v3/types/public/agent.d.ts CHANGED Viewed

@@ -395,7 +395,7 @@ export interface AgentStreamExecuteOptions extends AgentExecuteOptionsBase {
      */
     callbacks?: AgentStreamCallbacks;
 }
-export type AgentType = "openai" | "anthropic" | "google" | "microsoft" | "bedrock";
+export type AgentType = "openai" | "anthropic" | "google" | "microsoft" | "bedrock" | "vertex";
 export declare const AVAILABLE_CUA_MODELS: readonly ["openai/gpt-5.4", "openai/gpt-5.4-mini", "openai/gpt-5.5", "openai/computer-use-preview", "openai/computer-use-preview-2025-03-11", "anthropic/claude-opus-4-5-20251101", "anthropic/claude-opus-4-6", "anthropic/claude-sonnet-4-6", "anthropic/claude-haiku-4-5", "anthropic/claude-haiku-4-5-20251001", "anthropic/claude-sonnet-4-20250514", "anthropic/claude-sonnet-4-5-20250929", "google/gemini-2.5-computer-use-preview-10-2025", "google/gemini-3-flash-preview", "google/gemini-3-pro-preview", "microsoft/fara-7b"];
 export type AvailableCuaModel = (typeof AVAILABLE_CUA_MODELS)[number];
 export interface AgentExecutionOptions<TOptions extends AgentExecuteOptions = AgentExecuteOptions> {

package/dist/esm/lib/v3/types/public/agent.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"agent.js","sourceRoot":"","sources":["../../../../../../lib/v3/types/public/agent.ts"],"names":[],"mappings":"AAicA,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,gBAAgB;IAChB,qBAAqB;IACrB,gBAAgB;IAChB,6BAA6B;IAC7B,wCAAwC;IACxC,oCAAoC;IACpC,2BAA2B;IAC3B,6BAA6B;IAC7B,4BAA4B;IAC5B,qCAAqC;IACrC,oCAAoC;IACpC,sCAAsC;IACtC,gDAAgD;IAChD,+BAA+B;IAC/B,6BAA6B;IAC7B,mBAAmB;CACX,CAAC","sourcesContent":["import type { Client } from \"@modelcontextprotocol/sdk/client/index.js\";\nimport {\n ToolSet,\n ModelMessage,\n wrapLanguageModel,\n StreamTextResult,\n StepResult,\n PrepareStepFunction,\n GenerateTextOnStepFinishCallback,\n StreamTextOnStepFinishCallback,\n StreamTextOnErrorCallback,\n StreamTextOnChunkCallback,\n StreamTextOnFinishCallback,\n} from \"ai\";\nimport { LogLine } from \"./logs.js\";\nimport { ClientOptions } from \"./model.js\";\nimport { StagehandZodObject } from \"../../zodCompat.js\";\n\n// Re-export ModelMessage for consumers who want to use it for conversation continuation\nexport type { ModelMessage } from \"ai\";\n\n// Re-export Tool type for consumers who want to define custom tools\nexport type { Tool } from \"ai\";\nimport { Page as PlaywrightPage } from \"playwright-core\";\nimport { Page as PuppeteerPage } from \"puppeteer-core\";\nimport { Page as PatchrightPage } from \"patchright-core\";\nimport { Page } from \"../../understudy/page.js\";\n\n// =============================================================================\n// Variable Types\n// =============================================================================\n\n/*\n A variable value can be a simple primitive or a rich object with an optional description.\n * This unified type is shared across `act`, `agent.execute`, and other methods.\n \n @example Simple (backward-compatible):\n * ```typescript\n * variables: { username: \"john@example.com\" }\n * ```\n \n @example Rich with description (useful for agents):\n * ```typescript\n * variables: {\n * username: { value: \"john@example.com\", description: \"The login email\" }\n * }\n * ```\n /\nexport type VariableValue =\n \| string\n \| number\n \| boolean\n \| { value: string \| number \| boolean; description?: string };\n\n/\n A collection of named variables for use in act, agent, and other methods.\n /\nexport type Variables = Record<string, VariableValue>;\n\nexport interface AgentContext {\n options: AgentExecuteOptionsBase;\n maxSteps: number;\n systemPrompt: string;\n allTools: ToolSet;\n messages: ModelMessage[];\n wrappedModel: ReturnType<typeof wrapLanguageModel>;\n initialPageUrl: string;\n}\n\nexport interface AgentState {\n collectedReasoning: string[];\n actions: AgentAction[];\n finalMessage: string;\n completed: boolean;\n currentPageUrl: string;\n}\n\nexport interface AgentAction {\n type: string;\n reasoning?: string;\n taskCompleted?: boolean;\n action?: string;\n // Tool-specific fields\n timeMs?: number; // wait tool\n pageText?: string; // ariaTree tool\n pageUrl?: string; // ariaTree tool\n instruction?: string; // various tools\n [key: string]: unknown;\n}\n\nexport interface AgentResult {\n success: boolean;\n message: string;\n actions: AgentAction[];\n completed: boolean;\n metadata?: Record<string, unknown>;\n usage?: {\n input_tokens: number;\n output_tokens: number;\n reasoning_tokens?: number;\n cached_input_tokens?: number;\n inference_time_ms: number;\n };\n /\n The conversation messages from this execution.\n * Pass these to a subsequent execute() call via the `messages` option to continue the conversation.\n * @experimental\n /\n messages?: ModelMessage[];\n /\n Custom output data extracted based on the `output` schema provided in execute options.\n * Only populated if an `output` schema was provided.\n * @experimental\n /\n output?: Record<string, unknown>;\n}\n\nexport type AgentStreamResult = StreamTextResult<ToolSet, never> & {\n result: Promise<AgentResult>;\n};\n\n/\n Base callbacks shared between execute (non-streaming) and streaming modes.\n /\nexport interface AgentCallbacks {\n /\n Optional function called before each step to modify settings.\n * You can change the model, tool choices, active tools, system prompt,\n * and input messages for each step.\n /\n prepareStep?: PrepareStepFunction<ToolSet>;\n /\n Callback called when each step (LLM call) is finished.\n * This is called for intermediate steps as well as the final step.\n /\n onStepFinish?:\n \| GenerateTextOnStepFinishCallback<ToolSet>\n \| StreamTextOnStepFinishCallback<ToolSet>;\n}\n\n/\n Error message type for streaming-only callbacks used in non-streaming mode.\n * This provides a clear error message when users try to use streaming callbacks without stream: true.\n /\ntype StreamingCallbackNotAvailable =\n \"This callback requires 'stream: true' in AgentConfig. Set stream: true to use streaming callbacks like onChunk, onFinish, onError, and onAbort.\";\n\n/\n Error message for safety confirmation callback misuse.\n * Safety confirmations are only available for non-streaming CUA agent executions.\n /\ntype SafetyConfirmationCallbackNotAvailable =\n \"Safety confirmation callbacks are only available via non-streaming AgentExecuteOptions.callbacks when using mode: 'cua'.\";\n\n/\n Callbacks specific to the non-streaming execute method.\n /\nexport interface AgentExecuteCallbacks extends AgentCallbacks {\n /\n Callback called when each step (LLM call) is finished.\n /\n onStepFinish?: GenerateTextOnStepFinishCallback<ToolSet>;\n /\n Callback for handling safety confirmation requests from CUA providers.\n * Only available when running an agent configured with mode: \"cua\".\n /\n onSafetyConfirmation?: SafetyConfirmationHandler;\n\n /\n NOT AVAILABLE in non-streaming mode.\n * This callback requires `stream: true` in AgentConfig.\n \n @example\n * ```typescript\n * // Enable streaming to use onChunk:\n * const agent = stagehand.agent({ stream: true });\n * await agent.execute({\n * instruction: \"...\",\n * callbacks: { onChunk: async (chunk) => console.log(chunk) }\n * });\n * ```\n /\n onChunk?: StreamingCallbackNotAvailable;\n\n /\n NOT AVAILABLE in non-streaming mode.\n * This callback requires `stream: true` in AgentConfig.\n \n @example\n * ```typescript\n * // Enable streaming to use onFinish:\n * const agent = stagehand.agent({ stream: true });\n * await agent.execute({\n * instruction: \"...\",\n * callbacks: { onFinish: (event) => console.log(\"Done!\", event) }\n * });\n * ```\n /\n onFinish?: StreamingCallbackNotAvailable;\n\n /\n NOT AVAILABLE in non-streaming mode.\n * This callback requires `stream: true` in AgentConfig.\n \n @example\n * ```typescript\n * // Enable streaming to use onError:\n * const agent = stagehand.agent({ stream: true });\n * await agent.execute({\n * instruction: \"...\",\n * callbacks: { onError: ({ error }) => console.error(error) }\n * });\n * ```\n /\n onError?: StreamingCallbackNotAvailable;\n\n /\n NOT AVAILABLE in non-streaming mode.\n * This callback requires `stream: true` in AgentConfig.\n \n @example\n * ```typescript\n * // Enable streaming to use onAbort:\n * const agent = stagehand.agent({ stream: true });\n * await agent.execute({\n * instruction: \"...\",\n * callbacks: { onAbort: (event) => console.log(\"Aborted\", event.steps) }\n * });\n * ```\n /\n onAbort?: StreamingCallbackNotAvailable;\n}\n\n/\n Callbacks specific to the streaming mode.\n /\nexport interface AgentStreamCallbacks extends AgentCallbacks {\n /\n Callback called when each step (LLM call) is finished during streaming.\n /\n onStepFinish?: StreamTextOnStepFinishCallback<ToolSet>;\n /\n Callback called when an error occurs during streaming.\n * Use this to log errors or handle error states.\n /\n onError?: StreamTextOnErrorCallback;\n /\n Callback called for each chunk of the stream.\n * Stream processing will pause until the callback promise resolves.\n /\n onChunk?: StreamTextOnChunkCallback<ToolSet>;\n /\n Callback called when the stream finishes.\n /\n onFinish?: StreamTextOnFinishCallback<ToolSet>;\n /\n Callback called when the stream is aborted.\n /\n onAbort?: (event: {\n steps: Array<StepResult<ToolSet>>;\n }) => PromiseLike<void> \| void;\n /\n NOT AVAILABLE in streaming mode.\n * Safety confirmations currently require non-streaming execute() on CUA agents.\n /\n onSafetyConfirmation?: SafetyConfirmationCallbackNotAvailable;\n}\n\n/\n Base options for agent execution (without callbacks).\n /\nexport interface AgentExecuteOptionsBase {\n instruction: string;\n maxSteps?: number;\n page?: PlaywrightPage \| PuppeteerPage \| PatchrightPage \| Page;\n highlightCursor?: boolean;\n /\n Previous conversation messages to continue from.\n * Pass the `messages` from a previous AgentResult to continue that conversation.\n * @experimental\n /\n messages?: ModelMessage[];\n /\n An AbortSignal that can be used to cancel the agent execution.\n * When aborted, the agent will stop and return a partial result.\n * @experimental\n \n @example\n * ```typescript\n * const controller = new AbortController();\n * setTimeout(() => controller.abort(), 30000); // 30 second timeout\n \n const result = await agent.execute({\n * instruction: \"...\",\n * signal: controller.signal\n * });\n * ```\n /\n signal?: AbortSignal;\n /\n Tools to exclude from this execution.\n * Pass an array of tool names to prevent the agent from using those tools.\n \n Note: Not supported in CUA mode (`mode: \"cua\"`).\n \n Available tools by mode:\n \n DOM mode (default):\n * - `act` - Perform semantic actions (click, type, etc.)\n * - `fillForm` - Fill form fields using DOM selectors\n * - `ariaTree` - Get accessibility tree of the page\n * - `extract` - Extract structured data from page\n * - `goto` - Navigate to a URL\n * - `scroll` - Scroll using semantic directions (up/down/left/right)\n * - `keys` - Press keyboard keys\n * - `navback` - Navigate back in history\n * - `screenshot` - Take a screenshot\n * - `think` - Agent reasoning/planning step\n * - `wait` - Wait for time or condition\n * - `done` - Mark task as complete\n * - `search` - Web search (requires useSearch: true and BROWSERBASE_API_KEY)\n \n Hybrid mode:\n * - `click` - Click at specific coordinates\n * - `type` - Type text at coordinates\n * - `dragAndDrop` - Drag from one point to another\n * - `clickAndHold` - Click and hold at coordinates\n * - `fillFormVision` - Fill forms using vision/coordinates\n * - `act` - Perform semantic actions\n * - `ariaTree` - Get accessibility tree\n * - `extract` - Extract data from page\n * - `goto` - Navigate to URL\n * - `scroll` - Scroll using coordinates\n * - `keys` - Press keyboard keys\n * - `navback` - Navigate back\n * - `screenshot` - Take screenshot\n * - `think` - Agent reasoning step\n * - `wait` - Wait for time/condition\n * - `done` - Mark task complete\n * - `search` - Web search (requires useSearch: true and BROWSERBASE_API_KEY)\n \n @experimental\n * @example\n * ```typescript\n * // Exclude screenshot and extract tools\n * const result = await agent.execute({\n * instruction: \"Click the submit button\",\n * excludeTools: [\"screenshot\", \"extract\"]\n * });\n * ```\n /\n excludeTools?: string[];\n /\n A Zod schema defining custom output data to return when the task completes.\n * The agent will populate this data in the final done tool call.\n \n @experimental\n * @example\n * ```typescript\n * const result = await agent.execute({\n * instruction: \"Find the cheapest flight from NYC to LA\",\n * output: z.object({\n * price: z.string().describe(\"The price of the flight\"),\n * airline: z.string().describe(\"The airline name\"),\n * departureTime: z.string().describe(\"Departure time\"),\n * }),\n * });\n \n console.log(result.output); // { price: \"$199\", airline: \"Delta\", departureTime: \"8:00 AM\" }\n * ```\n /\n output?: StagehandZodObject;\n /\n Variables that the agent can use when filling forms or typing text.\n * The agent will see variable names and descriptions in the system prompt,\n * and can use them via `%variableName%` syntax in act/type/fillForm tool calls.\n \n Accepts both simple values and rich objects with descriptions (same type as `act`).\n \n Note: Not supported in CUA mode (`mode: \"cua\"`).\n \n @example\n * ```typescript\n * // Simple values\n * variables: { username: \"john@example.com\", password: \"secret123\" }\n \n // Rich values with descriptions (helps the agent understand context)\n * variables: {\n * username: { value: \"john@example.com\", description: \"The login email\" },\n * password: { value: \"secret123\", description: \"The login password\" },\n * }\n * ```\n /\n variables?: Variables;\n /\n Timeout in milliseconds for each agent tool call.\n * If a tool call exceeds this duration, it will be aborted and\n * reported back to the LLM as a timeout error so it can retry or adjust.\n * For tools that call v3 methods (act, extract, fillForm, ariaTree), the\n * timeout is also forwarded to the underlying v3 call for true cancellation.\n * @default 45000 (45 seconds)\n /\n toolTimeout?: number;\n /\n Enable the web search tool powered by Browserbase Search API.\n * Requires a valid Browserbase API key (BROWSERBASE_API_KEY).\n * When set to true, the agent gains access to a `search` tool for web searches.\n \n @example\n * ```typescript\n * const result = await agent.execute({\n * instruction: \"Find the latest news about AI\",\n * useSearch: true,\n * });\n * ```\n /\n useSearch?: boolean;\n}\n\n/\n Options for non-streaming agent execution.\n * Only accepts AgentExecuteCallbacks (no streaming-specific callbacks like onChunk, onFinish).\n /\nexport interface AgentExecuteOptions extends AgentExecuteOptionsBase {\n /\n Callbacks for non-streaming agent execution.\n * For streaming callbacks (onChunk, onFinish, onError, onAbort), use stream: true in AgentConfig.\n /\n callbacks?: AgentExecuteCallbacks;\n}\n\n/\n Options for streaming agent execution.\n * Accepts AgentStreamCallbacks including onChunk, onFinish, onError, and onAbort.\n /\nexport interface AgentStreamExecuteOptions extends AgentExecuteOptionsBase {\n /\n Callbacks for streaming agent execution.\n * Includes streaming-specific callbacks: onChunk, onFinish, onError, onAbort.\n /\n callbacks?: AgentStreamCallbacks;\n}\nexport type AgentType =\n \| \"openai\"\n \| \"anthropic\"\n \| \"google\"\n \| \"microsoft\"\n \| \"bedrock\";\n\nexport const AVAILABLE_CUA_MODELS = [\n \"openai/gpt-5.4\",\n \"openai/gpt-5.4-mini\",\n \"openai/gpt-5.5\",\n \"openai/computer-use-preview\",\n \"openai/computer-use-preview-2025-03-11\",\n \"anthropic/claude-opus-4-5-20251101\",\n \"anthropic/claude-opus-4-6\",\n \"anthropic/claude-sonnet-4-6\",\n \"anthropic/claude-haiku-4-5\",\n \"anthropic/claude-haiku-4-5-20251001\",\n \"anthropic/claude-sonnet-4-20250514\",\n \"anthropic/claude-sonnet-4-5-20250929\",\n \"google/gemini-2.5-computer-use-preview-10-2025\",\n \"google/gemini-3-flash-preview\",\n \"google/gemini-3-pro-preview\",\n \"microsoft/fara-7b\",\n] as const;\nexport type AvailableCuaModel = (typeof AVAILABLE_CUA_MODELS)[number];\n\nexport interface AgentExecutionOptions<\n TOptions extends AgentExecuteOptions = AgentExecuteOptions,\n> {\n options: TOptions;\n logger: (message: LogLine) => void;\n retries?: number;\n}\n\nexport interface AgentHandlerOptions {\n modelName: string;\n clientOptions?: ClientOptions;\n userProvidedInstructions?: string;\n experimental?: boolean;\n}\n\nexport interface ActionExecutionResult {\n success: boolean;\n error?: string;\n data?: unknown;\n}\n\n/\n Represents a safety check that requires user confirmation before proceeding.\n * These are issued by CUA providers (OpenAI, Google) when the agent attempts\n * potentially risky actions.\n /\nexport interface SafetyCheck {\n /* Unique identifier for this safety check /\n id: string;\n /* Code identifying the type of safety concern /\n code: string;\n /* Human-readable description of the safety concern /\n message: string;\n}\n\n/\n Response from the user for a safety confirmation request.\n /\nexport interface SafetyConfirmationResponse {\n /* Whether the user acknowledged/approved the safety checks /\n acknowledged: boolean;\n}\n\n/\n Callback for handling safety confirmation requests.\n * Called when the CUA provider issues safety checks that require user confirmation.\n * The callback should return a promise that resolves when the user has made a decision.\n \n @param safetyChecks - Array of safety checks requiring confirmation\n * @returns Promise resolving to the user's response\n \n @example\n * ```typescript\n * const agent = stagehand.agent({\n * mode: \"cua\",\n * });\n * await agent.execute({\n * instruction: \"...\",\n * callbacks: {\n * onSafetyConfirmation: async (checks) => {\n * console.log(\"Safety checks:\", checks);\n * const userApproved = await showConfirmationDialog(checks);\n * return { acknowledged: userApproved };\n * },\n * },\n * });\n * ```\n /\nexport type SafetyConfirmationHandler = (\n safetyChecks: SafetyCheck[],\n) => Promise<SafetyConfirmationResponse>;\n\n// Anthropic types:\n\nexport interface ToolUseItem extends ResponseItem {\n type: \"tool_use\";\n id: string; // This is the correct property name from Anthropic's API\n name: string; // Name of the tool being used\n input: Record<string, unknown>;\n}\n\nexport interface AnthropicMessage {\n role: string;\n content: string \| Array<AnthropicContentBlock>;\n}\n\nexport interface AnthropicContentBlock {\n type: string;\n [key: string]: unknown;\n}\n\nexport interface AnthropicTextBlock extends AnthropicContentBlock {\n type: \"text\";\n text: string;\n}\n\nexport interface AnthropicToolResult {\n type: \"tool_result\";\n tool_use_id: string;\n content: string \| Array<AnthropicContentBlock>;\n}\n\n// OpenAI types:\n\nexport interface ResponseItem {\n type: string;\n id: string;\n [key: string]: unknown;\n}\n\nexport interface ComputerCallItem extends ResponseItem {\n type: \"computer_call\";\n call_id: string;\n action?: {\n type: string;\n [key: string]: unknown;\n };\n actions?: Array<{\n type: string;\n [key: string]: unknown;\n }>;\n pending_safety_checks?: Array<{\n id: string;\n code: string;\n message: string;\n }>;\n}\n\nexport interface FunctionCallItem extends ResponseItem {\n type: \"function_call\";\n call_id: string;\n name: string;\n arguments: string;\n}\n\nexport type ResponseInputItem =\n \| { role: string; content: string }\n \| {\n type: \"computer_call_output\";\n call_id: string;\n output:\n \| {\n type: \"input_image\" \| \"computer_screenshot\";\n image_url: string;\n detail?: \"original\" \| \"high\" \| \"low\";\n current_url?: string;\n error?: string;\n [key: string]: unknown;\n }\n \| string;\n acknowledged_safety_checks?: Array<{\n id: string;\n code: string;\n message: string;\n }>;\n }\n \| {\n type: \"function_call_output\";\n call_id: string;\n output: string;\n };\n\nexport interface AgentInstance {\n execute: (\n instructionOrOptions: string \| AgentExecuteOptions,\n ) => Promise<AgentResult>;\n}\n\nexport type AgentProviderType = AgentType;\n\nexport type AgentModelConfig<TModelName extends string = string> = {\n modelName: TModelName;\n} & Record<string, unknown>;\n\n/\n Agent tool mode determines which set of tools are available to the agent.\n * - 'dom': Uses DOM-based tools (act, fillForm) - better for structured page interactions\n * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, etc.) - better for visual/screenshot-based interactions\n * - 'cua': Uses Computer Use Agent (CUA) providers like Anthropic Claude or Google Gemini for screenshot-based automation\n /\nexport type AgentToolMode = \"dom\" \| \"hybrid\" \| \"cua\";\n\nexport type AgentConfig = {\n /\n Custom system prompt to provide to the agent. Overrides the default system prompt.\n /\n systemPrompt?: string;\n /\n MCP integrations - Array of Client objects\n /\n integrations?: (Client \| string)[];\n /\n Tools passed to the agent client\n /\n tools?: ToolSet;\n /\n @deprecated Use `mode: \"cua\"` instead. This option will be removed in a future version.\n * Enables Computer Use Agent (CUA) mode.\n /\n cua?: boolean;\n /\n The model to use for agent functionality\n /\n model?: string \| AgentModelConfig<string>;\n /\n The model to use for tool execution (observe/act calls within agent tools).\n * If not specified, inherits from the main model configuration.\n * Format: \"provider/model\" (e.g., \"openai/gpt-4o-mini\", \"google/gemini-2.0-flash-exp\")\n /\n executionModel?: string \| AgentModelConfig<string>;\n /\n Enable streaming mode for the agent.\n * When true, execute() returns AgentStreamResult with textStream for incremental output.\n * When false (default), execute() returns AgentResult after completion.\n /\n stream?: boolean;\n /\n Tool mode for the agent. Determines which set of tools are available.\n * - 'dom' (default): Uses DOM-based tools (act, fillForm) for structured interactions\n * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)\n * for visual/screenshot-based interactions\n * - 'cua': Uses Computer Use Agent (CUA) providers for screenshot-based automation\n /\n mode?: AgentToolMode;\n};\n\n/\n Agent instance returned when stream: true is set in AgentConfig.\n * execute() returns a streaming result that can be consumed incrementally.\n * Accepts AgentStreamExecuteOptions with streaming-specific callbacks.\n /\nexport interface StreamingAgentInstance {\n execute: (\n instructionOrOptions: string \| AgentStreamExecuteOptions,\n ) => Promise<AgentStreamResult>;\n}\n\n/\n Agent instance returned when stream is false or not set in AgentConfig.\n * execute() returns a result after the agent completes.\n * Accepts AgentExecuteOptions with non-streaming callbacks only.\n /\nexport interface NonStreamingAgentInstance {\n execute: (\n instructionOrOptions: string \| AgentExecuteOptions,\n ) => Promise<AgentResult>;\n}\n\n// =============================================================================\n// Vision Action Tool Result Types\n// =============================================================================\n\n/\n Content item type for toModelOutput return values.\n * Used in tool definitions to return text and/or media to the model.\n */\nexport type ModelOutputContentItem =\n \| { type: \"text\"; text: string }\n \| { type: \"media\"; mediaType: string; data: string };\n\nexport interface ClickToolResult {\n success: boolean;\n describe?: string;\n coordinates?: number[];\n error?: string;\n screenshotBase64?: string;\n}\n\nexport interface TypeToolResult {\n success: boolean;\n describe?: string;\n text?: string;\n error?: string;\n screenshotBase64?: string;\n}\n\nexport interface DragAndDropToolResult {\n success: boolean;\n describe?: string;\n error?: string;\n screenshotBase64?: string;\n}\n\nexport interface FillFormField {\n action: string;\n value: string;\n coordinates: { x: number; y: number };\n}\n\nexport interface FillFormVisionToolResult {\n success: boolean;\n playwrightArguments?: FillFormField[];\n error?: string;\n screenshotBase64?: string;\n}\n\nexport interface ScrollToolResult {\n success: boolean;\n message: string;\n scrolledPixels: number;\n error?: string;\n}\n\nexport interface ScrollVisionToolResult extends ScrollToolResult {\n screenshotBase64?: string;\n}\n\nexport interface WaitToolResult {\n success: boolean;\n waited: number;\n screenshotBase64?: string;\n error?: string;\n}\n"]}
1	+ {"version":3,"file":"agent.js","sourceRoot":"","sources":["../../../../../../lib/v3/types/public/agent.ts"],"names":[],"mappings":"AAkcA,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,gBAAgB;IAChB,qBAAqB;IACrB,gBAAgB;IAChB,6BAA6B;IAC7B,wCAAwC;IACxC,oCAAoC;IACpC,2BAA2B;IAC3B,6BAA6B;IAC7B,4BAA4B;IAC5B,qCAAqC;IACrC,oCAAoC;IACpC,sCAAsC;IACtC,gDAAgD;IAChD,+BAA+B;IAC/B,6BAA6B;IAC7B,mBAAmB;CACX,CAAC","sourcesContent":["import type { Client } from \"@modelcontextprotocol/sdk/client/index.js\";\nimport {\n ToolSet,\n ModelMessage,\n wrapLanguageModel,\n StreamTextResult,\n StepResult,\n PrepareStepFunction,\n GenerateTextOnStepFinishCallback,\n StreamTextOnStepFinishCallback,\n StreamTextOnErrorCallback,\n StreamTextOnChunkCallback,\n StreamTextOnFinishCallback,\n} from \"ai\";\nimport { LogLine } from \"./logs.js\";\nimport { ClientOptions } from \"./model.js\";\nimport { StagehandZodObject } from \"../../zodCompat.js\";\n\n// Re-export ModelMessage for consumers who want to use it for conversation continuation\nexport type { ModelMessage } from \"ai\";\n\n// Re-export Tool type for consumers who want to define custom tools\nexport type { Tool } from \"ai\";\nimport { Page as PlaywrightPage } from \"playwright-core\";\nimport { Page as PuppeteerPage } from \"puppeteer-core\";\nimport { Page as PatchrightPage } from \"patchright-core\";\nimport { Page } from \"../../understudy/page.js\";\n\n// =============================================================================\n// Variable Types\n// =============================================================================\n\n/*\n A variable value can be a simple primitive or a rich object with an optional description.\n * This unified type is shared across `act`, `agent.execute`, and other methods.\n \n @example Simple (backward-compatible):\n * ```typescript\n * variables: { username: \"john@example.com\" }\n * ```\n \n @example Rich with description (useful for agents):\n * ```typescript\n * variables: {\n * username: { value: \"john@example.com\", description: \"The login email\" }\n * }\n * ```\n /\nexport type VariableValue =\n \| string\n \| number\n \| boolean\n \| { value: string \| number \| boolean; description?: string };\n\n/\n A collection of named variables for use in act, agent, and other methods.\n /\nexport type Variables = Record<string, VariableValue>;\n\nexport interface AgentContext {\n options: AgentExecuteOptionsBase;\n maxSteps: number;\n systemPrompt: string;\n allTools: ToolSet;\n messages: ModelMessage[];\n wrappedModel: ReturnType<typeof wrapLanguageModel>;\n initialPageUrl: string;\n}\n\nexport interface AgentState {\n collectedReasoning: string[];\n actions: AgentAction[];\n finalMessage: string;\n completed: boolean;\n currentPageUrl: string;\n}\n\nexport interface AgentAction {\n type: string;\n reasoning?: string;\n taskCompleted?: boolean;\n action?: string;\n // Tool-specific fields\n timeMs?: number; // wait tool\n pageText?: string; // ariaTree tool\n pageUrl?: string; // ariaTree tool\n instruction?: string; // various tools\n [key: string]: unknown;\n}\n\nexport interface AgentResult {\n success: boolean;\n message: string;\n actions: AgentAction[];\n completed: boolean;\n metadata?: Record<string, unknown>;\n usage?: {\n input_tokens: number;\n output_tokens: number;\n reasoning_tokens?: number;\n cached_input_tokens?: number;\n inference_time_ms: number;\n };\n /\n The conversation messages from this execution.\n * Pass these to a subsequent execute() call via the `messages` option to continue the conversation.\n * @experimental\n /\n messages?: ModelMessage[];\n /\n Custom output data extracted based on the `output` schema provided in execute options.\n * Only populated if an `output` schema was provided.\n * @experimental\n /\n output?: Record<string, unknown>;\n}\n\nexport type AgentStreamResult = StreamTextResult<ToolSet, never> & {\n result: Promise<AgentResult>;\n};\n\n/\n Base callbacks shared between execute (non-streaming) and streaming modes.\n /\nexport interface AgentCallbacks {\n /\n Optional function called before each step to modify settings.\n * You can change the model, tool choices, active tools, system prompt,\n * and input messages for each step.\n /\n prepareStep?: PrepareStepFunction<ToolSet>;\n /\n Callback called when each step (LLM call) is finished.\n * This is called for intermediate steps as well as the final step.\n /\n onStepFinish?:\n \| GenerateTextOnStepFinishCallback<ToolSet>\n \| StreamTextOnStepFinishCallback<ToolSet>;\n}\n\n/\n Error message type for streaming-only callbacks used in non-streaming mode.\n * This provides a clear error message when users try to use streaming callbacks without stream: true.\n /\ntype StreamingCallbackNotAvailable =\n \"This callback requires 'stream: true' in AgentConfig. Set stream: true to use streaming callbacks like onChunk, onFinish, onError, and onAbort.\";\n\n/\n Error message for safety confirmation callback misuse.\n * Safety confirmations are only available for non-streaming CUA agent executions.\n /\ntype SafetyConfirmationCallbackNotAvailable =\n \"Safety confirmation callbacks are only available via non-streaming AgentExecuteOptions.callbacks when using mode: 'cua'.\";\n\n/\n Callbacks specific to the non-streaming execute method.\n /\nexport interface AgentExecuteCallbacks extends AgentCallbacks {\n /\n Callback called when each step (LLM call) is finished.\n /\n onStepFinish?: GenerateTextOnStepFinishCallback<ToolSet>;\n /\n Callback for handling safety confirmation requests from CUA providers.\n * Only available when running an agent configured with mode: \"cua\".\n /\n onSafetyConfirmation?: SafetyConfirmationHandler;\n\n /\n NOT AVAILABLE in non-streaming mode.\n * This callback requires `stream: true` in AgentConfig.\n \n @example\n * ```typescript\n * // Enable streaming to use onChunk:\n * const agent = stagehand.agent({ stream: true });\n * await agent.execute({\n * instruction: \"...\",\n * callbacks: { onChunk: async (chunk) => console.log(chunk) }\n * });\n * ```\n /\n onChunk?: StreamingCallbackNotAvailable;\n\n /\n NOT AVAILABLE in non-streaming mode.\n * This callback requires `stream: true` in AgentConfig.\n \n @example\n * ```typescript\n * // Enable streaming to use onFinish:\n * const agent = stagehand.agent({ stream: true });\n * await agent.execute({\n * instruction: \"...\",\n * callbacks: { onFinish: (event) => console.log(\"Done!\", event) }\n * });\n * ```\n /\n onFinish?: StreamingCallbackNotAvailable;\n\n /\n NOT AVAILABLE in non-streaming mode.\n * This callback requires `stream: true` in AgentConfig.\n \n @example\n * ```typescript\n * // Enable streaming to use onError:\n * const agent = stagehand.agent({ stream: true });\n * await agent.execute({\n * instruction: \"...\",\n * callbacks: { onError: ({ error }) => console.error(error) }\n * });\n * ```\n /\n onError?: StreamingCallbackNotAvailable;\n\n /\n NOT AVAILABLE in non-streaming mode.\n * This callback requires `stream: true` in AgentConfig.\n \n @example\n * ```typescript\n * // Enable streaming to use onAbort:\n * const agent = stagehand.agent({ stream: true });\n * await agent.execute({\n * instruction: \"...\",\n * callbacks: { onAbort: (event) => console.log(\"Aborted\", event.steps) }\n * });\n * ```\n /\n onAbort?: StreamingCallbackNotAvailable;\n}\n\n/\n Callbacks specific to the streaming mode.\n /\nexport interface AgentStreamCallbacks extends AgentCallbacks {\n /\n Callback called when each step (LLM call) is finished during streaming.\n /\n onStepFinish?: StreamTextOnStepFinishCallback<ToolSet>;\n /\n Callback called when an error occurs during streaming.\n * Use this to log errors or handle error states.\n /\n onError?: StreamTextOnErrorCallback;\n /\n Callback called for each chunk of the stream.\n * Stream processing will pause until the callback promise resolves.\n /\n onChunk?: StreamTextOnChunkCallback<ToolSet>;\n /\n Callback called when the stream finishes.\n /\n onFinish?: StreamTextOnFinishCallback<ToolSet>;\n /\n Callback called when the stream is aborted.\n /\n onAbort?: (event: {\n steps: Array<StepResult<ToolSet>>;\n }) => PromiseLike<void> \| void;\n /\n NOT AVAILABLE in streaming mode.\n * Safety confirmations currently require non-streaming execute() on CUA agents.\n /\n onSafetyConfirmation?: SafetyConfirmationCallbackNotAvailable;\n}\n\n/\n Base options for agent execution (without callbacks).\n /\nexport interface AgentExecuteOptionsBase {\n instruction: string;\n maxSteps?: number;\n page?: PlaywrightPage \| PuppeteerPage \| PatchrightPage \| Page;\n highlightCursor?: boolean;\n /\n Previous conversation messages to continue from.\n * Pass the `messages` from a previous AgentResult to continue that conversation.\n * @experimental\n /\n messages?: ModelMessage[];\n /\n An AbortSignal that can be used to cancel the agent execution.\n * When aborted, the agent will stop and return a partial result.\n * @experimental\n \n @example\n * ```typescript\n * const controller = new AbortController();\n * setTimeout(() => controller.abort(), 30000); // 30 second timeout\n \n const result = await agent.execute({\n * instruction: \"...\",\n * signal: controller.signal\n * });\n * ```\n /\n signal?: AbortSignal;\n /\n Tools to exclude from this execution.\n * Pass an array of tool names to prevent the agent from using those tools.\n \n Note: Not supported in CUA mode (`mode: \"cua\"`).\n \n Available tools by mode:\n \n DOM mode (default):\n * - `act` - Perform semantic actions (click, type, etc.)\n * - `fillForm` - Fill form fields using DOM selectors\n * - `ariaTree` - Get accessibility tree of the page\n * - `extract` - Extract structured data from page\n * - `goto` - Navigate to a URL\n * - `scroll` - Scroll using semantic directions (up/down/left/right)\n * - `keys` - Press keyboard keys\n * - `navback` - Navigate back in history\n * - `screenshot` - Take a screenshot\n * - `think` - Agent reasoning/planning step\n * - `wait` - Wait for time or condition\n * - `done` - Mark task as complete\n * - `search` - Web search (requires useSearch: true and BROWSERBASE_API_KEY)\n \n Hybrid mode:\n * - `click` - Click at specific coordinates\n * - `type` - Type text at coordinates\n * - `dragAndDrop` - Drag from one point to another\n * - `clickAndHold` - Click and hold at coordinates\n * - `fillFormVision` - Fill forms using vision/coordinates\n * - `act` - Perform semantic actions\n * - `ariaTree` - Get accessibility tree\n * - `extract` - Extract data from page\n * - `goto` - Navigate to URL\n * - `scroll` - Scroll using coordinates\n * - `keys` - Press keyboard keys\n * - `navback` - Navigate back\n * - `screenshot` - Take screenshot\n * - `think` - Agent reasoning step\n * - `wait` - Wait for time/condition\n * - `done` - Mark task complete\n * - `search` - Web search (requires useSearch: true and BROWSERBASE_API_KEY)\n \n @experimental\n * @example\n * ```typescript\n * // Exclude screenshot and extract tools\n * const result = await agent.execute({\n * instruction: \"Click the submit button\",\n * excludeTools: [\"screenshot\", \"extract\"]\n * });\n * ```\n /\n excludeTools?: string[];\n /\n A Zod schema defining custom output data to return when the task completes.\n * The agent will populate this data in the final done tool call.\n \n @experimental\n * @example\n * ```typescript\n * const result = await agent.execute({\n * instruction: \"Find the cheapest flight from NYC to LA\",\n * output: z.object({\n * price: z.string().describe(\"The price of the flight\"),\n * airline: z.string().describe(\"The airline name\"),\n * departureTime: z.string().describe(\"Departure time\"),\n * }),\n * });\n \n console.log(result.output); // { price: \"$199\", airline: \"Delta\", departureTime: \"8:00 AM\" }\n * ```\n /\n output?: StagehandZodObject;\n /\n Variables that the agent can use when filling forms or typing text.\n * The agent will see variable names and descriptions in the system prompt,\n * and can use them via `%variableName%` syntax in act/type/fillForm tool calls.\n \n Accepts both simple values and rich objects with descriptions (same type as `act`).\n \n Note: Not supported in CUA mode (`mode: \"cua\"`).\n \n @example\n * ```typescript\n * // Simple values\n * variables: { username: \"john@example.com\", password: \"secret123\" }\n \n // Rich values with descriptions (helps the agent understand context)\n * variables: {\n * username: { value: \"john@example.com\", description: \"The login email\" },\n * password: { value: \"secret123\", description: \"The login password\" },\n * }\n * ```\n /\n variables?: Variables;\n /\n Timeout in milliseconds for each agent tool call.\n * If a tool call exceeds this duration, it will be aborted and\n * reported back to the LLM as a timeout error so it can retry or adjust.\n * For tools that call v3 methods (act, extract, fillForm, ariaTree), the\n * timeout is also forwarded to the underlying v3 call for true cancellation.\n * @default 45000 (45 seconds)\n /\n toolTimeout?: number;\n /\n Enable the web search tool powered by Browserbase Search API.\n * Requires a valid Browserbase API key (BROWSERBASE_API_KEY).\n * When set to true, the agent gains access to a `search` tool for web searches.\n \n @example\n * ```typescript\n * const result = await agent.execute({\n * instruction: \"Find the latest news about AI\",\n * useSearch: true,\n * });\n * ```\n /\n useSearch?: boolean;\n}\n\n/\n Options for non-streaming agent execution.\n * Only accepts AgentExecuteCallbacks (no streaming-specific callbacks like onChunk, onFinish).\n /\nexport interface AgentExecuteOptions extends AgentExecuteOptionsBase {\n /\n Callbacks for non-streaming agent execution.\n * For streaming callbacks (onChunk, onFinish, onError, onAbort), use stream: true in AgentConfig.\n /\n callbacks?: AgentExecuteCallbacks;\n}\n\n/\n Options for streaming agent execution.\n * Accepts AgentStreamCallbacks including onChunk, onFinish, onError, and onAbort.\n /\nexport interface AgentStreamExecuteOptions extends AgentExecuteOptionsBase {\n /\n Callbacks for streaming agent execution.\n * Includes streaming-specific callbacks: onChunk, onFinish, onError, onAbort.\n /\n callbacks?: AgentStreamCallbacks;\n}\nexport type AgentType =\n \| \"openai\"\n \| \"anthropic\"\n \| \"google\"\n \| \"microsoft\"\n \| \"bedrock\"\n \| \"vertex\";\n\nexport const AVAILABLE_CUA_MODELS = [\n \"openai/gpt-5.4\",\n \"openai/gpt-5.4-mini\",\n \"openai/gpt-5.5\",\n \"openai/computer-use-preview\",\n \"openai/computer-use-preview-2025-03-11\",\n \"anthropic/claude-opus-4-5-20251101\",\n \"anthropic/claude-opus-4-6\",\n \"anthropic/claude-sonnet-4-6\",\n \"anthropic/claude-haiku-4-5\",\n \"anthropic/claude-haiku-4-5-20251001\",\n \"anthropic/claude-sonnet-4-20250514\",\n \"anthropic/claude-sonnet-4-5-20250929\",\n \"google/gemini-2.5-computer-use-preview-10-2025\",\n \"google/gemini-3-flash-preview\",\n \"google/gemini-3-pro-preview\",\n \"microsoft/fara-7b\",\n] as const;\nexport type AvailableCuaModel = (typeof AVAILABLE_CUA_MODELS)[number];\n\nexport interface AgentExecutionOptions<\n TOptions extends AgentExecuteOptions = AgentExecuteOptions,\n> {\n options: TOptions;\n logger: (message: LogLine) => void;\n retries?: number;\n}\n\nexport interface AgentHandlerOptions {\n modelName: string;\n clientOptions?: ClientOptions;\n userProvidedInstructions?: string;\n experimental?: boolean;\n}\n\nexport interface ActionExecutionResult {\n success: boolean;\n error?: string;\n data?: unknown;\n}\n\n/\n Represents a safety check that requires user confirmation before proceeding.\n * These are issued by CUA providers (OpenAI, Google) when the agent attempts\n * potentially risky actions.\n /\nexport interface SafetyCheck {\n /* Unique identifier for this safety check /\n id: string;\n /* Code identifying the type of safety concern /\n code: string;\n /* Human-readable description of the safety concern /\n message: string;\n}\n\n/\n Response from the user for a safety confirmation request.\n /\nexport interface SafetyConfirmationResponse {\n /* Whether the user acknowledged/approved the safety checks /\n acknowledged: boolean;\n}\n\n/\n Callback for handling safety confirmation requests.\n * Called when the CUA provider issues safety checks that require user confirmation.\n * The callback should return a promise that resolves when the user has made a decision.\n \n @param safetyChecks - Array of safety checks requiring confirmation\n * @returns Promise resolving to the user's response\n \n @example\n * ```typescript\n * const agent = stagehand.agent({\n * mode: \"cua\",\n * });\n * await agent.execute({\n * instruction: \"...\",\n * callbacks: {\n * onSafetyConfirmation: async (checks) => {\n * console.log(\"Safety checks:\", checks);\n * const userApproved = await showConfirmationDialog(checks);\n * return { acknowledged: userApproved };\n * },\n * },\n * });\n * ```\n /\nexport type SafetyConfirmationHandler = (\n safetyChecks: SafetyCheck[],\n) => Promise<SafetyConfirmationResponse>;\n\n// Anthropic types:\n\nexport interface ToolUseItem extends ResponseItem {\n type: \"tool_use\";\n id: string; // This is the correct property name from Anthropic's API\n name: string; // Name of the tool being used\n input: Record<string, unknown>;\n}\n\nexport interface AnthropicMessage {\n role: string;\n content: string \| Array<AnthropicContentBlock>;\n}\n\nexport interface AnthropicContentBlock {\n type: string;\n [key: string]: unknown;\n}\n\nexport interface AnthropicTextBlock extends AnthropicContentBlock {\n type: \"text\";\n text: string;\n}\n\nexport interface AnthropicToolResult {\n type: \"tool_result\";\n tool_use_id: string;\n content: string \| Array<AnthropicContentBlock>;\n}\n\n// OpenAI types:\n\nexport interface ResponseItem {\n type: string;\n id: string;\n [key: string]: unknown;\n}\n\nexport interface ComputerCallItem extends ResponseItem {\n type: \"computer_call\";\n call_id: string;\n action?: {\n type: string;\n [key: string]: unknown;\n };\n actions?: Array<{\n type: string;\n [key: string]: unknown;\n }>;\n pending_safety_checks?: Array<{\n id: string;\n code: string;\n message: string;\n }>;\n}\n\nexport interface FunctionCallItem extends ResponseItem {\n type: \"function_call\";\n call_id: string;\n name: string;\n arguments: string;\n}\n\nexport type ResponseInputItem =\n \| { role: string; content: string }\n \| {\n type: \"computer_call_output\";\n call_id: string;\n output:\n \| {\n type: \"input_image\" \| \"computer_screenshot\";\n image_url: string;\n detail?: \"original\" \| \"high\" \| \"low\";\n current_url?: string;\n error?: string;\n [key: string]: unknown;\n }\n \| string;\n acknowledged_safety_checks?: Array<{\n id: string;\n code: string;\n message: string;\n }>;\n }\n \| {\n type: \"function_call_output\";\n call_id: string;\n output: string;\n };\n\nexport interface AgentInstance {\n execute: (\n instructionOrOptions: string \| AgentExecuteOptions,\n ) => Promise<AgentResult>;\n}\n\nexport type AgentProviderType = AgentType;\n\nexport type AgentModelConfig<TModelName extends string = string> = {\n modelName: TModelName;\n} & Record<string, unknown>;\n\n/\n Agent tool mode determines which set of tools are available to the agent.\n * - 'dom': Uses DOM-based tools (act, fillForm) - better for structured page interactions\n * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, etc.) - better for visual/screenshot-based interactions\n * - 'cua': Uses Computer Use Agent (CUA) providers like Anthropic Claude or Google Gemini for screenshot-based automation\n /\nexport type AgentToolMode = \"dom\" \| \"hybrid\" \| \"cua\";\n\nexport type AgentConfig = {\n /\n Custom system prompt to provide to the agent. Overrides the default system prompt.\n /\n systemPrompt?: string;\n /\n MCP integrations - Array of Client objects\n /\n integrations?: (Client \| string)[];\n /\n Tools passed to the agent client\n /\n tools?: ToolSet;\n /\n @deprecated Use `mode: \"cua\"` instead. This option will be removed in a future version.\n * Enables Computer Use Agent (CUA) mode.\n /\n cua?: boolean;\n /\n The model to use for agent functionality\n /\n model?: string \| AgentModelConfig<string>;\n /\n The model to use for tool execution (observe/act calls within agent tools).\n * If not specified, inherits from the main model configuration.\n * Format: \"provider/model\" (e.g., \"openai/gpt-4o-mini\", \"google/gemini-2.0-flash-exp\")\n /\n executionModel?: string \| AgentModelConfig<string>;\n /\n Enable streaming mode for the agent.\n * When true, execute() returns AgentStreamResult with textStream for incremental output.\n * When false (default), execute() returns AgentResult after completion.\n /\n stream?: boolean;\n /\n Tool mode for the agent. Determines which set of tools are available.\n * - 'dom' (default): Uses DOM-based tools (act, fillForm) for structured interactions\n * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)\n * for visual/screenshot-based interactions\n * - 'cua': Uses Computer Use Agent (CUA) providers for screenshot-based automation\n /\n mode?: AgentToolMode;\n};\n\n/\n Agent instance returned when stream: true is set in AgentConfig.\n * execute() returns a streaming result that can be consumed incrementally.\n * Accepts AgentStreamExecuteOptions with streaming-specific callbacks.\n /\nexport interface StreamingAgentInstance {\n execute: (\n instructionOrOptions: string \| AgentStreamExecuteOptions,\n ) => Promise<AgentStreamResult>;\n}\n\n/\n Agent instance returned when stream is false or not set in AgentConfig.\n * execute() returns a result after the agent completes.\n * Accepts AgentExecuteOptions with non-streaming callbacks only.\n /\nexport interface NonStreamingAgentInstance {\n execute: (\n instructionOrOptions: string \| AgentExecuteOptions,\n ) => Promise<AgentResult>;\n}\n\n// =============================================================================\n// Vision Action Tool Result Types\n// =============================================================================\n\n/\n Content item type for toModelOutput return values.\n * Used in tool definitions to return text and/or media to the model.\n */\nexport type ModelOutputContentItem =\n \| { type: \"text\"; text: string }\n \| { type: \"media\"; mediaType: string; data: string };\n\nexport interface ClickToolResult {\n success: boolean;\n describe?: string;\n coordinates?: number[];\n error?: string;\n screenshotBase64?: string;\n}\n\nexport interface TypeToolResult {\n success: boolean;\n describe?: string;\n text?: string;\n error?: string;\n screenshotBase64?: string;\n}\n\nexport interface DragAndDropToolResult {\n success: boolean;\n describe?: string;\n error?: string;\n screenshotBase64?: string;\n}\n\nexport interface FillFormField {\n action: string;\n value: string;\n coordinates: { x: number; y: number };\n}\n\nexport interface FillFormVisionToolResult {\n success: boolean;\n playwrightArguments?: FillFormField[];\n error?: string;\n screenshotBase64?: string;\n}\n\nexport interface ScrollToolResult {\n success: boolean;\n message: string;\n scrolledPixels: number;\n error?: string;\n}\n\nexport interface ScrollVisionToolResult extends ScrollToolResult {\n screenshotBase64?: string;\n}\n\nexport interface WaitToolResult {\n success: boolean;\n waited: number;\n screenshotBase64?: string;\n error?: string;\n}\n"]}

package/dist/esm/lib/v3/types/public/api.d.ts CHANGED Viewed

@@ -42,6 +42,37 @@ export declare const LocalBrowserLaunchOptionsSchema: z.ZodObject<{
     acceptDownloads: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$strict>;
 /** Detailed model configuration object */
+export declare const GoogleServiceAccountCredentialsSchema: z.ZodObject<{
+    type: z.ZodOptional<z.ZodLiteral<"service_account">>;
+    project_id: z.ZodOptional<z.ZodString>;
+    private_key_id: z.ZodOptional<z.ZodString>;
+    private_key: z.ZodString;
+    client_email: z.ZodString;
+    client_id: z.ZodOptional<z.ZodString>;
+    auth_uri: z.ZodOptional<z.ZodURL>;
+    token_uri: z.ZodOptional<z.ZodURL>;
+    auth_provider_x509_cert_url: z.ZodOptional<z.ZodURL>;
+    client_x509_cert_url: z.ZodOptional<z.ZodURL>;
+    universe_domain: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+export declare const GoogleAuthOptionsSchema: z.ZodObject<{
+    credentials: z.ZodOptional<z.ZodObject<{
+        type: z.ZodOptional<z.ZodLiteral<"service_account">>;
+        project_id: z.ZodOptional<z.ZodString>;
+        private_key_id: z.ZodOptional<z.ZodString>;
+        private_key: z.ZodString;
+        client_email: z.ZodString;
+        client_id: z.ZodOptional<z.ZodString>;
+        auth_uri: z.ZodOptional<z.ZodURL>;
+        token_uri: z.ZodOptional<z.ZodURL>;
+        auth_provider_x509_cert_url: z.ZodOptional<z.ZodURL>;
+        client_x509_cert_url: z.ZodOptional<z.ZodURL>;
+        universe_domain: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>>;
+    scopes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+    projectId: z.ZodOptional<z.ZodString>;
+    universeDomain: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
 export declare const ModelConfigObjectSchema: z.ZodObject<{
     provider: z.ZodOptional<z.ZodEnum<{
         openai: "openai";
@@ -49,11 +80,32 @@ export declare const ModelConfigObjectSchema: z.ZodObject<{
         google: "google";
         microsoft: "microsoft";
         bedrock: "bedrock";
+        vertex: "vertex";
     }>>;
     modelName: z.ZodString;
     apiKey: z.ZodOptional<z.ZodString>;
     baseURL: z.ZodOptional<z.ZodString>;
     headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    project: z.ZodOptional<z.ZodString>;
+    location: z.ZodOptional<z.ZodString>;
+    googleAuthOptions: z.ZodOptional<z.ZodObject<{
+        credentials: z.ZodOptional<z.ZodObject<{
+            type: z.ZodOptional<z.ZodLiteral<"service_account">>;
+            project_id: z.ZodOptional<z.ZodString>;
+            private_key_id: z.ZodOptional<z.ZodString>;
+            private_key: z.ZodString;
+            client_email: z.ZodString;
+            client_id: z.ZodOptional<z.ZodString>;
+            auth_uri: z.ZodOptional<z.ZodURL>;
+            token_uri: z.ZodOptional<z.ZodURL>;
+            auth_provider_x509_cert_url: z.ZodOptional<z.ZodURL>;
+            client_x509_cert_url: z.ZodOptional<z.ZodURL>;
+            universe_domain: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>>;
+        scopes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+        projectId: z.ZodOptional<z.ZodString>;
+        universeDomain: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>>;
 }, z.core.$strip>;
 /** Model configuration */
 export declare const ModelConfigSchema: z.ZodObject<{
@@ -63,11 +115,32 @@ export declare const ModelConfigSchema: z.ZodObject<{
         google: "google";
         microsoft: "microsoft";
         bedrock: "bedrock";
+        vertex: "vertex";
     }>>;
     modelName: z.ZodString;
     apiKey: z.ZodOptional<z.ZodString>;
     baseURL: z.ZodOptional<z.ZodString>;
     headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    project: z.ZodOptional<z.ZodString>;
+    location: z.ZodOptional<z.ZodString>;
+    googleAuthOptions: z.ZodOptional<z.ZodObject<{
+        credentials: z.ZodOptional<z.ZodObject<{
+            type: z.ZodOptional<z.ZodLiteral<"service_account">>;
+            project_id: z.ZodOptional<z.ZodString>;
+            private_key_id: z.ZodOptional<z.ZodString>;
+            private_key: z.ZodString;
+            client_email: z.ZodString;
+            client_id: z.ZodOptional<z.ZodString>;
+            auth_uri: z.ZodOptional<z.ZodURL>;
+            token_uri: z.ZodOptional<z.ZodURL>;
+            auth_provider_x509_cert_url: z.ZodOptional<z.ZodURL>;
+            client_x509_cert_url: z.ZodOptional<z.ZodURL>;
+            universe_domain: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>>;
+        scopes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+        projectId: z.ZodOptional<z.ZodString>;
+        universeDomain: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>>;
 }, z.core.$strip>;
 /** Action object returned by observe and used by act */
 export declare const ActionSchema: z.ZodObject<{
@@ -529,11 +602,32 @@ export declare const ActOptionsSchema: z.ZodOptional<z.ZodObject<{
             google: "google";
             microsoft: "microsoft";
             bedrock: "bedrock";
+            vertex: "vertex";
         }>>;
         modelName: z.ZodString;
         apiKey: z.ZodOptional<z.ZodString>;
         baseURL: z.ZodOptional<z.ZodString>;
         headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        project: z.ZodOptional<z.ZodString>;
+        location: z.ZodOptional<z.ZodString>;
+        googleAuthOptions: z.ZodOptional<z.ZodObject<{
+            credentials: z.ZodOptional<z.ZodObject<{
+                type: z.ZodOptional<z.ZodLiteral<"service_account">>;
+                project_id: z.ZodOptional<z.ZodString>;
+                private_key_id: z.ZodOptional<z.ZodString>;
+                private_key: z.ZodString;
+                client_email: z.ZodString;
+                client_id: z.ZodOptional<z.ZodString>;
+                auth_uri: z.ZodOptional<z.ZodURL>;
+                token_uri: z.ZodOptional<z.ZodURL>;
+                auth_provider_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                client_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                universe_domain: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>;
+            scopes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+            projectId: z.ZodOptional<z.ZodString>;
+            universeDomain: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>>;
     }, z.core.$strip>, z.ZodString]>>;
     variables: z.ZodOptional<z.ZodType<import("./agent.js").Variables, unknown, z.core.$ZodTypeInternals<import("./agent.js").Variables, unknown>>>;
     timeout: z.ZodOptional<z.ZodNumber>;
@@ -554,11 +648,32 @@ export declare const ActRequestSchema: z.ZodObject<{
                 google: "google";
                 microsoft: "microsoft";
                 bedrock: "bedrock";
+                vertex: "vertex";
             }>>;
             modelName: z.ZodString;
             apiKey: z.ZodOptional<z.ZodString>;
             baseURL: z.ZodOptional<z.ZodString>;
             headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+            project: z.ZodOptional<z.ZodString>;
+            location: z.ZodOptional<z.ZodString>;
+            googleAuthOptions: z.ZodOptional<z.ZodObject<{
+                credentials: z.ZodOptional<z.ZodObject<{
+                    type: z.ZodOptional<z.ZodLiteral<"service_account">>;
+                    project_id: z.ZodOptional<z.ZodString>;
+                    private_key_id: z.ZodOptional<z.ZodString>;
+                    private_key: z.ZodString;
+                    client_email: z.ZodString;
+                    client_id: z.ZodOptional<z.ZodString>;
+                    auth_uri: z.ZodOptional<z.ZodURL>;
+                    token_uri: z.ZodOptional<z.ZodURL>;
+                    auth_provider_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                    client_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                    universe_domain: z.ZodOptional<z.ZodString>;
+                }, z.core.$strict>>;
+                scopes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+                projectId: z.ZodOptional<z.ZodString>;
+                universeDomain: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>;
         }, z.core.$strip>, z.ZodString]>>;
         variables: z.ZodOptional<z.ZodType<import("./agent.js").Variables, unknown, z.core.$ZodTypeInternals<import("./agent.js").Variables, unknown>>>;
         timeout: z.ZodOptional<z.ZodNumber>;
@@ -620,15 +735,37 @@ export declare const ExtractOptionsSchema: z.ZodOptional<z.ZodObject<{
             google: "google";
             microsoft: "microsoft";
             bedrock: "bedrock";
+            vertex: "vertex";
         }>>;
         modelName: z.ZodString;
         apiKey: z.ZodOptional<z.ZodString>;
         baseURL: z.ZodOptional<z.ZodString>;
         headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        project: z.ZodOptional<z.ZodString>;
+        location: z.ZodOptional<z.ZodString>;
+        googleAuthOptions: z.ZodOptional<z.ZodObject<{
+            credentials: z.ZodOptional<z.ZodObject<{
+                type: z.ZodOptional<z.ZodLiteral<"service_account">>;
+                project_id: z.ZodOptional<z.ZodString>;
+                private_key_id: z.ZodOptional<z.ZodString>;
+                private_key: z.ZodString;
+                client_email: z.ZodString;
+                client_id: z.ZodOptional<z.ZodString>;
+                auth_uri: z.ZodOptional<z.ZodURL>;
+                token_uri: z.ZodOptional<z.ZodURL>;
+                auth_provider_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                client_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                universe_domain: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>;
+            scopes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+            projectId: z.ZodOptional<z.ZodString>;
+            universeDomain: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>>;
     }, z.core.$strip>, z.ZodString]>>;
     timeout: z.ZodOptional<z.ZodNumber>;
     selector: z.ZodOptional<z.ZodString>;
     ignoreSelectors: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    screenshot: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$strip>>;
 export declare const ExtractRequestSchema: z.ZodObject<{
     instruction: z.ZodOptional<z.ZodString>;
@@ -641,15 +778,37 @@ export declare const ExtractRequestSchema: z.ZodObject<{
                 google: "google";
                 microsoft: "microsoft";
                 bedrock: "bedrock";
+                vertex: "vertex";
             }>>;
             modelName: z.ZodString;
             apiKey: z.ZodOptional<z.ZodString>;
             baseURL: z.ZodOptional<z.ZodString>;
             headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+            project: z.ZodOptional<z.ZodString>;
+            location: z.ZodOptional<z.ZodString>;
+            googleAuthOptions: z.ZodOptional<z.ZodObject<{
+                credentials: z.ZodOptional<z.ZodObject<{
+                    type: z.ZodOptional<z.ZodLiteral<"service_account">>;
+                    project_id: z.ZodOptional<z.ZodString>;
+                    private_key_id: z.ZodOptional<z.ZodString>;
+                    private_key: z.ZodString;
+                    client_email: z.ZodString;
+                    client_id: z.ZodOptional<z.ZodString>;
+                    auth_uri: z.ZodOptional<z.ZodURL>;
+                    token_uri: z.ZodOptional<z.ZodURL>;
+                    auth_provider_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                    client_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                    universe_domain: z.ZodOptional<z.ZodString>;
+                }, z.core.$strict>>;
+                scopes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+                projectId: z.ZodOptional<z.ZodString>;
+                universeDomain: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>;
         }, z.core.$strip>, z.ZodString]>>;
         timeout: z.ZodOptional<z.ZodNumber>;
         selector: z.ZodOptional<z.ZodString>;
         ignoreSelectors: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        screenshot: z.ZodOptional<z.ZodBoolean>;
     }, z.core.$strip>>;
     frameId: z.ZodOptional<z.ZodNullable<z.ZodString>>;
     streamResponse: z.ZodOptional<z.ZodBoolean>;
@@ -673,15 +832,37 @@ export declare const ObserveOptionsSchema: z.ZodOptional<z.ZodObject<{
             google: "google";
             microsoft: "microsoft";
             bedrock: "bedrock";
+            vertex: "vertex";
         }>>;
         modelName: z.ZodString;
         apiKey: z.ZodOptional<z.ZodString>;
         baseURL: z.ZodOptional<z.ZodString>;
         headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        project: z.ZodOptional<z.ZodString>;
+        location: z.ZodOptional<z.ZodString>;
+        googleAuthOptions: z.ZodOptional<z.ZodObject<{
+            credentials: z.ZodOptional<z.ZodObject<{
+                type: z.ZodOptional<z.ZodLiteral<"service_account">>;
+                project_id: z.ZodOptional<z.ZodString>;
+                private_key_id: z.ZodOptional<z.ZodString>;
+                private_key: z.ZodString;
+                client_email: z.ZodString;
+                client_id: z.ZodOptional<z.ZodString>;
+                auth_uri: z.ZodOptional<z.ZodURL>;
+                token_uri: z.ZodOptional<z.ZodURL>;
+                auth_provider_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                client_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                universe_domain: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>;
+            scopes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+            projectId: z.ZodOptional<z.ZodString>;
+            universeDomain: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>>;
     }, z.core.$strip>, z.ZodString]>>;
     variables: z.ZodOptional<z.ZodType<import("./agent.js").Variables, unknown, z.core.$ZodTypeInternals<import("./agent.js").Variables, unknown>>>;
     timeout: z.ZodOptional<z.ZodNumber>;
     selector: z.ZodOptional<z.ZodString>;
+    ignoreSelectors: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strip>>;
 export declare const ObserveRequestSchema: z.ZodObject<{
     instruction: z.ZodOptional<z.ZodString>;
@@ -693,15 +874,37 @@ export declare const ObserveRequestSchema: z.ZodObject<{
                 google: "google";
                 microsoft: "microsoft";
                 bedrock: "bedrock";
+                vertex: "vertex";
             }>>;
             modelName: z.ZodString;
             apiKey: z.ZodOptional<z.ZodString>;
             baseURL: z.ZodOptional<z.ZodString>;
             headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+            project: z.ZodOptional<z.ZodString>;
+            location: z.ZodOptional<z.ZodString>;
+            googleAuthOptions: z.ZodOptional<z.ZodObject<{
+                credentials: z.ZodOptional<z.ZodObject<{
+                    type: z.ZodOptional<z.ZodLiteral<"service_account">>;
+                    project_id: z.ZodOptional<z.ZodString>;
+                    private_key_id: z.ZodOptional<z.ZodString>;
+                    private_key: z.ZodString;
+                    client_email: z.ZodString;
+                    client_id: z.ZodOptional<z.ZodString>;
+                    auth_uri: z.ZodOptional<z.ZodURL>;
+                    token_uri: z.ZodOptional<z.ZodURL>;
+                    auth_provider_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                    client_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                    universe_domain: z.ZodOptional<z.ZodString>;
+                }, z.core.$strict>>;
+                scopes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+                projectId: z.ZodOptional<z.ZodString>;
+                universeDomain: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>;
         }, z.core.$strip>, z.ZodString]>>;
         variables: z.ZodOptional<z.ZodType<import("./agent.js").Variables, unknown, z.core.$ZodTypeInternals<import("./agent.js").Variables, unknown>>>;
         timeout: z.ZodOptional<z.ZodNumber>;
         selector: z.ZodOptional<z.ZodString>;
+        ignoreSelectors: z.ZodOptional<z.ZodArray<z.ZodString>>;
     }, z.core.$strip>>;
     frameId: z.ZodOptional<z.ZodNullable<z.ZodString>>;
     streamResponse: z.ZodOptional<z.ZodBoolean>;
@@ -744,11 +947,32 @@ export declare const AgentConfigSchema: z.ZodObject<{
             google: "google";
             microsoft: "microsoft";
             bedrock: "bedrock";
+            vertex: "vertex";
         }>>;
         modelName: z.ZodString;
         apiKey: z.ZodOptional<z.ZodString>;
         baseURL: z.ZodOptional<z.ZodString>;
         headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        project: z.ZodOptional<z.ZodString>;
+        location: z.ZodOptional<z.ZodString>;
+        googleAuthOptions: z.ZodOptional<z.ZodObject<{
+            credentials: z.ZodOptional<z.ZodObject<{
+                type: z.ZodOptional<z.ZodLiteral<"service_account">>;
+                project_id: z.ZodOptional<z.ZodString>;
+                private_key_id: z.ZodOptional<z.ZodString>;
+                private_key: z.ZodString;
+                client_email: z.ZodString;
+                client_id: z.ZodOptional<z.ZodString>;
+                auth_uri: z.ZodOptional<z.ZodURL>;
+                token_uri: z.ZodOptional<z.ZodURL>;
+                auth_provider_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                client_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                universe_domain: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>;
+            scopes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+            projectId: z.ZodOptional<z.ZodString>;
+            universeDomain: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>>;
     }, z.core.$strip>, z.ZodString]>>;
     systemPrompt: z.ZodOptional<z.ZodString>;
     cua: z.ZodOptional<z.ZodBoolean>;
@@ -764,11 +988,32 @@ export declare const AgentConfigSchema: z.ZodObject<{
             google: "google";
             microsoft: "microsoft";
             bedrock: "bedrock";
+            vertex: "vertex";
         }>>;
         modelName: z.ZodString;
         apiKey: z.ZodOptional<z.ZodString>;
         baseURL: z.ZodOptional<z.ZodString>;
         headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        project: z.ZodOptional<z.ZodString>;
+        location: z.ZodOptional<z.ZodString>;
+        googleAuthOptions: z.ZodOptional<z.ZodObject<{
+            credentials: z.ZodOptional<z.ZodObject<{
+                type: z.ZodOptional<z.ZodLiteral<"service_account">>;
+                project_id: z.ZodOptional<z.ZodString>;
+                private_key_id: z.ZodOptional<z.ZodString>;
+                private_key: z.ZodString;
+                client_email: z.ZodString;
+                client_id: z.ZodOptional<z.ZodString>;
+                auth_uri: z.ZodOptional<z.ZodURL>;
+                token_uri: z.ZodOptional<z.ZodURL>;
+                auth_provider_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                client_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                universe_domain: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>;
+            scopes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+            projectId: z.ZodOptional<z.ZodString>;
+            universeDomain: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>>;
     }, z.core.$strip>, z.ZodString]>>;
 }, z.core.$strip>;
 /** Action taken by the agent during execution */
@@ -842,11 +1087,32 @@ export declare const AgentExecuteRequestSchema: z.ZodObject<{
                 google: "google";
                 microsoft: "microsoft";
                 bedrock: "bedrock";
+                vertex: "vertex";
             }>>;
             modelName: z.ZodString;
             apiKey: z.ZodOptional<z.ZodString>;
             baseURL: z.ZodOptional<z.ZodString>;
             headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+            project: z.ZodOptional<z.ZodString>;
+            location: z.ZodOptional<z.ZodString>;
+            googleAuthOptions: z.ZodOptional<z.ZodObject<{
+                credentials: z.ZodOptional<z.ZodObject<{
+                    type: z.ZodOptional<z.ZodLiteral<"service_account">>;
+                    project_id: z.ZodOptional<z.ZodString>;
+                    private_key_id: z.ZodOptional<z.ZodString>;
+                    private_key: z.ZodString;
+                    client_email: z.ZodString;
+                    client_id: z.ZodOptional<z.ZodString>;
+                    auth_uri: z.ZodOptional<z.ZodURL>;
+                    token_uri: z.ZodOptional<z.ZodURL>;
+                    auth_provider_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                    client_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                    universe_domain: z.ZodOptional<z.ZodString>;
+                }, z.core.$strict>>;
+                scopes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+                projectId: z.ZodOptional<z.ZodString>;
+                universeDomain: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>;
         }, z.core.$strip>, z.ZodString]>>;
         systemPrompt: z.ZodOptional<z.ZodString>;
         cua: z.ZodOptional<z.ZodBoolean>;
@@ -862,11 +1128,32 @@ export declare const AgentExecuteRequestSchema: z.ZodObject<{
                 google: "google";
                 microsoft: "microsoft";
                 bedrock: "bedrock";
+                vertex: "vertex";
             }>>;
             modelName: z.ZodString;
             apiKey: z.ZodOptional<z.ZodString>;
             baseURL: z.ZodOptional<z.ZodString>;
             headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+            project: z.ZodOptional<z.ZodString>;
+            location: z.ZodOptional<z.ZodString>;
+            googleAuthOptions: z.ZodOptional<z.ZodObject<{
+                credentials: z.ZodOptional<z.ZodObject<{
+                    type: z.ZodOptional<z.ZodLiteral<"service_account">>;
+                    project_id: z.ZodOptional<z.ZodString>;
+                    private_key_id: z.ZodOptional<z.ZodString>;
+                    private_key: z.ZodString;
+                    client_email: z.ZodString;
+                    client_id: z.ZodOptional<z.ZodString>;
+                    auth_uri: z.ZodOptional<z.ZodURL>;
+                    token_uri: z.ZodOptional<z.ZodURL>;
+                    auth_provider_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                    client_x509_cert_url: z.ZodOptional<z.ZodURL>;
+                    universe_domain: z.ZodOptional<z.ZodString>;
+                }, z.core.$strict>>;
+                scopes: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+                projectId: z.ZodOptional<z.ZodString>;
+                universeDomain: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>;
         }, z.core.$strip>, z.ZodString]>>;
     }, z.core.$strip>;
     executeOptions: z.ZodObject<{