@browserbasehq/orca 3.7.0-preview.1 → 3.7.2-preview.0

package/dist/cjs/lib/modelUtils.d.ts

@@ -1,4 +1,14 @@
 import { ClientOptions, ModelConfiguration } from "./v3/types/public/model.js";
+/**
+ * Sentinel model name that delegates model selection to the Stagehand API.
+ * Only valid when running through the API (env: "BROWSERBASE" with
+ * disableAPI: false); no local LLM client or provider API key is involved.
+ */
+export declare const AUTO_MODEL_NAME = "auto";
+export declare function isAutoModel(model?: string | {
+    modelName: string;
+    [key: string]: unknown;
+}): boolean;
 export declare function extractModelName(model?: string | {
     modelName: string;
     [key: string]: unknown;

package/dist/cjs/lib/modelUtils.js

@@ -1,9 +1,20 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.AUTO_MODEL_NAME = void 0;
+exports.isAutoModel = isAutoModel;
 exports.extractModelName = extractModelName;
 exports.splitModelName = splitModelName;
 exports.resolveModel = resolveModel;
 const agent_js_1 = require("./v3/types/public/agent.js");
+/**
+ * Sentinel model name that delegates model selection to the Stagehand API.
+ * Only valid when running through the API (env: "BROWSERBASE" with
+ * disableAPI: false); no local LLM client or provider API key is involved.
+ */
+exports.AUTO_MODEL_NAME = "auto";
+function isAutoModel(model) {
+    return extractModelName(model) === exports.AUTO_MODEL_NAME;
+}
 //useful when resolving a model from string or object formats we accept
 function extractModelName(model) {
     if (!model)

package/dist/cjs/lib/modelUtils.js.map

@@ -1 +1 @@
-{"version":3,"file":"modelUtils.js","sourceRoot":"","sources":["../../../lib/modelUtils.ts"],"names":[],"mappings":";;AAOA,4CAKC;AAED,wCAQC;AAED,oCA4CC;AAnED,yDAGoC;AAEpC,uEAAuE;AACvE,SAAgB,gBAAgB,CAC9B,KAA8D;IAE9D,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC;AAC7D,CAAC;AAED,SAAgB,cAAc,CAAC,KAAa;IAI1C,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;IACrD,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC;IACvD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC;AAED,SAAgB,YAAY,CAAC,KAAkC;IAM7D,MAAM,WAAW,GAAG,gBAAgB,CAAC,KAAK,CAAE,CAAC;IAC7C,MAAM,aAAa,GACjB,OAAO,KAAK,KAAK,QAAQ;QACvB,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,CAAC,GAAG,EAAE;YACJ,6DAA6D;YAC7D,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC;YACxC,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,EAAE,CAAC;IAEX,uDAAuD;IACvD,MAAM,mBAAmB,GAAG,aAAa,CAAC,QAAQ,KAAK,SAAS,CAAC;IAEjE,oFAAoF;IACpF,IAAI,QAAgB,CAAC;IACrB,IAAI,eAAuB,CAAC;IAE5B,IAAI,mBAAmB,EAAE,CAAC;QACxB,QAAQ,GAAG,aAAa,CAAC,QAAkB,CAAC;QAC5C,eAAe,GAAG,WAAW,CAAC,CAAC,2BAA2B;IAC5D,CAAC;SAAM,CAAC;QACN,kCAAkC;QAClC,MAAM,KAAK,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;QAC1C,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;QAC1B,eAAe,GAAG,KAAK,CAAC,SAAS,CAAC;IACpC,CAAC;IAED,4BAA4B;IAC5B,MAAM,KAAK,GACT,mBAAmB;QACnB,+BAAoB,CAAC,QAAQ,CAAC,WAAgC,CAAC,CAAC;IAElE,OAAO;QACL,QAAQ;QACR,SAAS,EAAE,eAAe;QAC1B,aAAa;QACb,KAAK;KACN,CAAC;AACJ,CAAC","sourcesContent":["import { ClientOptions, ModelConfiguration } from \"./v3/types/public/model.js\";\nimport {\n  AVAILABLE_CUA_MODELS,\n  AvailableCuaModel,\n} from \"./v3/types/public/agent.js\";\n\n//useful when resolving a model from string or object formats we accept\nexport function extractModelName(\n  model?: string | { modelName: string; [key: string]: unknown },\n): string | undefined {\n  if (!model) return undefined;\n  return typeof model === \"string\" ? model : model.modelName;\n}\n\nexport function splitModelName(model: string): {\n  provider: string;\n  modelName: string;\n} {\n  const firstSlashIndex = model.indexOf(\"/\");\n  const provider = model.substring(0, firstSlashIndex);\n  const modelName = model.substring(firstSlashIndex + 1);\n  return { provider, modelName };\n}\n\nexport function resolveModel(model: string | ModelConfiguration): {\n  provider: string;\n  modelName: string;\n  clientOptions: ClientOptions;\n  isCua: boolean;\n} {\n  const modelString = extractModelName(model)!;\n  const clientOptions =\n    typeof model === \"string\"\n      ? {}\n      : (() => {\n          // eslint-disable-next-line @typescript-eslint/no-unused-vars\n          const { modelName: _, ...rest } = model;\n          return rest;\n        })();\n\n  // Check if provider is explicitly set in clientOptions\n  const hasExplicitProvider = clientOptions.provider !== undefined;\n\n  // If provider is explicitly set, don't split the model name - pass it through as-is\n  let provider: string;\n  let parsedModelName: string;\n\n  if (hasExplicitProvider) {\n    provider = clientOptions.provider as string;\n    parsedModelName = modelString; // Keep the full model name\n  } else {\n    // Parse the model string normally\n    const split = splitModelName(modelString);\n    provider = split.provider;\n    parsedModelName = split.modelName;\n  }\n\n  // Check if it's a CUA model\n  const isCua =\n    hasExplicitProvider ||\n    AVAILABLE_CUA_MODELS.includes(modelString as AvailableCuaModel);\n\n  return {\n    provider,\n    modelName: parsedModelName,\n    clientOptions,\n    isCua,\n  };\n}\n"]}
+{"version":3,"file":"modelUtils.js","sourceRoot":"","sources":["../../../lib/modelUtils.ts"],"names":[],"mappings":";;;AAaA,kCAIC;AAGD,4CAKC;AAED,wCAQC;AAED,oCA4CC;AAhFD,yDAGoC;AAEpC;;;;GAIG;AACU,QAAA,eAAe,GAAG,MAAM,CAAC;AAEtC,SAAgB,WAAW,CACzB,KAA8D;IAE9D,OAAO,gBAAgB,CAAC,KAAK,CAAC,KAAK,uBAAe,CAAC;AACrD,CAAC;AAED,uEAAuE;AACvE,SAAgB,gBAAgB,CAC9B,KAA8D;IAE9D,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC;AAC7D,CAAC;AAED,SAAgB,cAAc,CAAC,KAAa;IAI1C,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;IACrD,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC;IACvD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC;AAED,SAAgB,YAAY,CAAC,KAAkC;IAM7D,MAAM,WAAW,GAAG,gBAAgB,CAAC,KAAK,CAAE,CAAC;IAC7C,MAAM,aAAa,GACjB,OAAO,KAAK,KAAK,QAAQ;QACvB,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,CAAC,GAAG,EAAE;YACJ,6DAA6D;YAC7D,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC;YACxC,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,EAAE,CAAC;IAEX,uDAAuD;IACvD,MAAM,mBAAmB,GAAG,aAAa,CAAC,QAAQ,KAAK,SAAS,CAAC;IAEjE,oFAAoF;IACpF,IAAI,QAAgB,CAAC;IACrB,IAAI,eAAuB,CAAC;IAE5B,IAAI,mBAAmB,EAAE,CAAC;QACxB,QAAQ,GAAG,aAAa,CAAC,QAAkB,CAAC;QAC5C,eAAe,GAAG,WAAW,CAAC,CAAC,2BAA2B;IAC5D,CAAC;SAAM,CAAC;QACN,kCAAkC;QAClC,MAAM,KAAK,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;QAC1C,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;QAC1B,eAAe,GAAG,KAAK,CAAC,SAAS,CAAC;IACpC,CAAC;IAED,4BAA4B;IAC5B,MAAM,KAAK,GACT,mBAAmB;QACnB,+BAAoB,CAAC,QAAQ,CAAC,WAAgC,CAAC,CAAC;IAElE,OAAO;QACL,QAAQ;QACR,SAAS,EAAE,eAAe;QAC1B,aAAa;QACb,KAAK;KACN,CAAC;AACJ,CAAC","sourcesContent":["import { ClientOptions, ModelConfiguration } from \"./v3/types/public/model.js\";\nimport {\n  AVAILABLE_CUA_MODELS,\n  AvailableCuaModel,\n} from \"./v3/types/public/agent.js\";\n\n/**\n * Sentinel model name that delegates model selection to the Stagehand API.\n * Only valid when running through the API (env: \"BROWSERBASE\" with\n * disableAPI: false); no local LLM client or provider API key is involved.\n */\nexport const AUTO_MODEL_NAME = \"auto\";\n\nexport function isAutoModel(\n  model?: string | { modelName: string; [key: string]: unknown },\n): boolean {\n  return extractModelName(model) === AUTO_MODEL_NAME;\n}\n\n//useful when resolving a model from string or object formats we accept\nexport function extractModelName(\n  model?: string | { modelName: string; [key: string]: unknown },\n): string | undefined {\n  if (!model) return undefined;\n  return typeof model === \"string\" ? model : model.modelName;\n}\n\nexport function splitModelName(model: string): {\n  provider: string;\n  modelName: string;\n} {\n  const firstSlashIndex = model.indexOf(\"/\");\n  const provider = model.substring(0, firstSlashIndex);\n  const modelName = model.substring(firstSlashIndex + 1);\n  return { provider, modelName };\n}\n\nexport function resolveModel(model: string | ModelConfiguration): {\n  provider: string;\n  modelName: string;\n  clientOptions: ClientOptions;\n  isCua: boolean;\n} {\n  const modelString = extractModelName(model)!;\n  const clientOptions =\n    typeof model === \"string\"\n      ? {}\n      : (() => {\n          // eslint-disable-next-line @typescript-eslint/no-unused-vars\n          const { modelName: _, ...rest } = model;\n          return rest;\n        })();\n\n  // Check if provider is explicitly set in clientOptions\n  const hasExplicitProvider = clientOptions.provider !== undefined;\n\n  // If provider is explicitly set, don't split the model name - pass it through as-is\n  let provider: string;\n  let parsedModelName: string;\n\n  if (hasExplicitProvider) {\n    provider = clientOptions.provider as string;\n    parsedModelName = modelString; // Keep the full model name\n  } else {\n    // Parse the model string normally\n    const split = splitModelName(modelString);\n    provider = split.provider;\n    parsedModelName = split.modelName;\n  }\n\n  // Check if it's a CUA model\n  const isCua =\n    hasExplicitProvider ||\n    AVAILABLE_CUA_MODELS.includes(modelString as AvailableCuaModel);\n\n  return {\n    provider,\n    modelName: parsedModelName,\n    clientOptions,\n    isCua,\n  };\n}\n"]}

package/dist/cjs/lib/v3/agent/AgentProvider.js

@@ -12,6 +12,9 @@ exports.modelToAgentProviderMap = {
     "gpt-5.4": "openai",
     "gpt-5.4-mini": "openai",
     "gpt-5.5": "openai",
+    "gpt-5.6-terra": "openai",
+    "gpt-5.6-luna": "openai",
+    "gpt-5.6-sol": "openai",
     "computer-use-preview": "openai",
     "computer-use-preview-2025-03-11": "openai",
     "claude-sonnet-4-20250514": "anthropic",

package/dist/cjs/lib/v3/agent/AgentProvider.js.map

@@ -1 +1 @@
-{"version":3,"file":"AgentProvider.js","sourceRoot":"","sources":["../../../../../lib/v3/agent/AgentProvider.ts"],"names":[],"mappings":";;;AACA,6CAAoD;AAIpD,+DAGsC;AAEtC,mEAA6D;AAC7D,6DAAuD;AACvD,6DAAuD;AACvD,mEAA6D;AAE7D,0CAA0C;AAC7B,QAAA,uBAAuB,GAAsC;IACxE,SAAS,EAAE,QAAQ;IACnB,cAAc,EAAE,QAAQ;IACxB,SAAS,EAAE,QAAQ;IACnB,sBAAsB,EAAE,QAAQ;IAChC,iCAAiC,EAAE,QAAQ;IAC3C,0BAA0B,EAAE,WAAW;IACvC,4BAA4B,EAAE,WAAW;IACzC,0BAA0B,EAAE,WAAW;IACvC,iBAAiB,EAAE,WAAW;IAC9B,iBAAiB,EAAE,WAAW;IAC9B,mBAAmB,EAAE,WAAW;IAChC,kBAAkB,EAAE,WAAW;IAC/B,2BAA2B,EAAE,WAAW;IACxC,gBAAgB,EAAE,WAAW;IAC7B,yCAAyC,EAAE,QAAQ;IACnD,wBAAwB,EAAE,QAAQ;IAClC,kBAAkB,EAAE,QAAQ;IAC5B,sBAAsB,EAAE,QAAQ;IAChC,SAAS,EAAE,WAAW;CACvB,CAAC;AAEF;;;;GAIG;AACH,MAAa,aAAa;IAChB,MAAM,CAA6B;IAE3C;;OAEG;IACH,YAAY,MAAkC;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,SAAS,CACP,SAAiB,EACjB,aAA6B,EAC7B,wBAAiC,EACjC,KAAe;QAEf,uDAAuD;QACvD,MAAM,gBAAgB,GAAG,aAAa,EAAE,QAE3B,CAAC;QACd,MAAM,IAAI,GAAG,gBAAgB,IAAI,aAAa,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAE3E,IAAI,CAAC,MAAM,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,kCAAkC,IAAI,YAAY,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,EAAE;YACvH,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,QAAQ,IAAI,EAAE,CAAC;gBACb,KAAK,QAAQ;oBACX,OAAO,IAAI,oCAAe,CACxB,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,WAAW;oBACd,OAAO,IAAI,0CAAkB,CAC3B,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,QAAQ;oBACX,OAAO,IAAI,oCAAe,CACxB,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,WAAW;oBACd,OAAO,IAAI,0CAAkB,CAC3B,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,CACd,CAAC;gBACJ;oBACE,MAAM,IAAI,4CAA6B,CACrC,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,CAAC,EAC9C,oBAAoB,CACrB,CAAC;YACN,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,IAAI,CAAC,MAAM,CAAC;gBACV,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,gCAAgC,YAAY,EAAE;gBACvD,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,MAAM,CAAC,gBAAgB,CAAC,SAAiB;QACvC,MAAM,UAAU,GAAG,IAAA,6BAAkB,EAAC,SAAS,CAAC,CAAC;QAEjD,IAAI,UAAU,IAAI,+BAAuB,EAAE,CAAC;YAC1C,OAAO,+BAAuB,CAAC,UAAU,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,IAAI,oCAAqB,CAC7B,MAAM,CAAC,IAAI,CAAC,+BAAuB,CAAC,EACpC,oBAAoB,CACrB,CAAC;IACJ,CAAC;CACF;AA3FD,sCA2FC","sourcesContent":["import { ToolSet } from \"ai\";\nimport { stripModelProvider } from \"../../utils.js\";\nimport { AgentProviderType } from \"../types/public/agent.js\";\nimport { LogLine } from \"../types/public/logs.js\";\nimport { ClientOptions } from \"../types/public/model.js\";\nimport {\n  UnsupportedModelError,\n  UnsupportedModelProviderError,\n} from \"../types/public/sdkErrors.js\";\nimport { AgentClient } from \"./AgentClient.js\";\nimport { AnthropicCUAClient } from \"./AnthropicCUAClient.js\";\nimport { OpenAICUAClient } from \"./OpenAICUAClient.js\";\nimport { GoogleCUAClient } from \"./GoogleCUAClient.js\";\nimport { MicrosoftCUAClient } from \"./MicrosoftCUAClient.js\";\n\n// Map model names to their provider types\nexport const modelToAgentProviderMap: Record<string, AgentProviderType> = {\n  \"gpt-5.4\": \"openai\",\n  \"gpt-5.4-mini\": \"openai\",\n  \"gpt-5.5\": \"openai\",\n  \"computer-use-preview\": \"openai\",\n  \"computer-use-preview-2025-03-11\": \"openai\",\n  \"claude-sonnet-4-20250514\": \"anthropic\",\n  \"claude-sonnet-4-5-20250929\": \"anthropic\",\n  \"claude-opus-4-5-20251101\": \"anthropic\",\n  \"claude-opus-4-6\": \"anthropic\",\n  \"claude-opus-4-8\": \"anthropic\",\n  \"claude-sonnet-4-6\": \"anthropic\",\n  \"claude-haiku-4-5\": \"anthropic\",\n  \"claude-haiku-4-5-20251001\": \"anthropic\",\n  \"claude-fable-5\": \"anthropic\",\n  \"gemini-2.5-computer-use-preview-10-2025\": \"google\",\n  \"gemini-3-flash-preview\": \"google\",\n  \"gemini-3.5-flash\": \"google\",\n  \"gemini-3-pro-preview\": \"google\",\n  \"fara-7b\": \"microsoft\",\n};\n\n/**\n * Provider for agent clients\n * This class is responsible for creating the appropriate agent client\n * based on the provider type\n */\nexport class AgentProvider {\n  private logger: (message: LogLine) => void;\n\n  /**\n   * Create a new agent provider\n   */\n  constructor(logger: (message: LogLine) => void) {\n    this.logger = logger;\n  }\n\n  getClient(\n    modelName: string,\n    clientOptions?: ClientOptions,\n    userProvidedInstructions?: string,\n    tools?: ToolSet,\n  ): AgentClient {\n    // Check if provider is explicitly set in clientOptions\n    const explicitProvider = clientOptions?.provider as\n      | AgentProviderType\n      | undefined;\n    const type = explicitProvider || AgentProvider.getAgentProvider(modelName);\n\n    this.logger({\n      category: \"agent\",\n      message: `Getting agent client for type: ${type}, model: ${modelName}${explicitProvider ? \" (explicit provider)\" : \"\"}`,\n      level: 2,\n    });\n\n    try {\n      switch (type) {\n        case \"openai\":\n          return new OpenAICUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"anthropic\":\n          return new AnthropicCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"google\":\n          return new GoogleCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"microsoft\":\n          return new MicrosoftCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n          );\n        default:\n          throw new UnsupportedModelProviderError(\n            [\"openai\", \"anthropic\", \"google\", \"microsoft\"],\n            \"Computer Use Agent\",\n          );\n      }\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      this.logger({\n        category: \"agent\",\n        message: `Error creating agent client: ${errorMessage}`,\n        level: 0,\n      });\n      throw error;\n    }\n  }\n\n  static getAgentProvider(modelName: string): AgentProviderType {\n    const normalized = stripModelProvider(modelName);\n\n    if (normalized in modelToAgentProviderMap) {\n      return modelToAgentProviderMap[normalized];\n    }\n\n    throw new UnsupportedModelError(\n      Object.keys(modelToAgentProviderMap),\n      \"Computer Use Agent\",\n    );\n  }\n}\n"]}
+{"version":3,"file":"AgentProvider.js","sourceRoot":"","sources":["../../../../../lib/v3/agent/AgentProvider.ts"],"names":[],"mappings":";;;AACA,6CAAoD;AAIpD,+DAGsC;AAEtC,mEAA6D;AAC7D,6DAAuD;AACvD,6DAAuD;AACvD,mEAA6D;AAE7D,0CAA0C;AAC7B,QAAA,uBAAuB,GAAsC;IACxE,SAAS,EAAE,QAAQ;IACnB,cAAc,EAAE,QAAQ;IACxB,SAAS,EAAE,QAAQ;IACnB,eAAe,EAAE,QAAQ;IACzB,cAAc,EAAE,QAAQ;IACxB,aAAa,EAAE,QAAQ;IACvB,sBAAsB,EAAE,QAAQ;IAChC,iCAAiC,EAAE,QAAQ;IAC3C,0BAA0B,EAAE,WAAW;IACvC,4BAA4B,EAAE,WAAW;IACzC,0BAA0B,EAAE,WAAW;IACvC,iBAAiB,EAAE,WAAW;IAC9B,iBAAiB,EAAE,WAAW;IAC9B,mBAAmB,EAAE,WAAW;IAChC,kBAAkB,EAAE,WAAW;IAC/B,2BAA2B,EAAE,WAAW;IACxC,gBAAgB,EAAE,WAAW;IAC7B,yCAAyC,EAAE,QAAQ;IACnD,wBAAwB,EAAE,QAAQ;IAClC,kBAAkB,EAAE,QAAQ;IAC5B,sBAAsB,EAAE,QAAQ;IAChC,SAAS,EAAE,WAAW;CACvB,CAAC;AAEF;;;;GAIG;AACH,MAAa,aAAa;IAChB,MAAM,CAA6B;IAE3C;;OAEG;IACH,YAAY,MAAkC;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,SAAS,CACP,SAAiB,EACjB,aAA6B,EAC7B,wBAAiC,EACjC,KAAe;QAEf,uDAAuD;QACvD,MAAM,gBAAgB,GAAG,aAAa,EAAE,QAE3B,CAAC;QACd,MAAM,IAAI,GAAG,gBAAgB,IAAI,aAAa,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAE3E,IAAI,CAAC,MAAM,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,kCAAkC,IAAI,YAAY,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,EAAE;YACvH,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,QAAQ,IAAI,EAAE,CAAC;gBACb,KAAK,QAAQ;oBACX,OAAO,IAAI,oCAAe,CACxB,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,WAAW;oBACd,OAAO,IAAI,0CAAkB,CAC3B,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,QAAQ;oBACX,OAAO,IAAI,oCAAe,CACxB,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,WAAW;oBACd,OAAO,IAAI,0CAAkB,CAC3B,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,CACd,CAAC;gBACJ;oBACE,MAAM,IAAI,4CAA6B,CACrC,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,CAAC,EAC9C,oBAAoB,CACrB,CAAC;YACN,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,IAAI,CAAC,MAAM,CAAC;gBACV,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,gCAAgC,YAAY,EAAE;gBACvD,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,MAAM,CAAC,gBAAgB,CAAC,SAAiB;QACvC,MAAM,UAAU,GAAG,IAAA,6BAAkB,EAAC,SAAS,CAAC,CAAC;QAEjD,IAAI,UAAU,IAAI,+BAAuB,EAAE,CAAC;YAC1C,OAAO,+BAAuB,CAAC,UAAU,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,IAAI,oCAAqB,CAC7B,MAAM,CAAC,IAAI,CAAC,+BAAuB,CAAC,EACpC,oBAAoB,CACrB,CAAC;IACJ,CAAC;CACF;AA3FD,sCA2FC","sourcesContent":["import { ToolSet } from \"ai\";\nimport { stripModelProvider } from \"../../utils.js\";\nimport { AgentProviderType } from \"../types/public/agent.js\";\nimport { LogLine } from \"../types/public/logs.js\";\nimport { ClientOptions } from \"../types/public/model.js\";\nimport {\n  UnsupportedModelError,\n  UnsupportedModelProviderError,\n} from \"../types/public/sdkErrors.js\";\nimport { AgentClient } from \"./AgentClient.js\";\nimport { AnthropicCUAClient } from \"./AnthropicCUAClient.js\";\nimport { OpenAICUAClient } from \"./OpenAICUAClient.js\";\nimport { GoogleCUAClient } from \"./GoogleCUAClient.js\";\nimport { MicrosoftCUAClient } from \"./MicrosoftCUAClient.js\";\n\n// Map model names to their provider types\nexport const modelToAgentProviderMap: Record<string, AgentProviderType> = {\n  \"gpt-5.4\": \"openai\",\n  \"gpt-5.4-mini\": \"openai\",\n  \"gpt-5.5\": \"openai\",\n  \"gpt-5.6-terra\": \"openai\",\n  \"gpt-5.6-luna\": \"openai\",\n  \"gpt-5.6-sol\": \"openai\",\n  \"computer-use-preview\": \"openai\",\n  \"computer-use-preview-2025-03-11\": \"openai\",\n  \"claude-sonnet-4-20250514\": \"anthropic\",\n  \"claude-sonnet-4-5-20250929\": \"anthropic\",\n  \"claude-opus-4-5-20251101\": \"anthropic\",\n  \"claude-opus-4-6\": \"anthropic\",\n  \"claude-opus-4-8\": \"anthropic\",\n  \"claude-sonnet-4-6\": \"anthropic\",\n  \"claude-haiku-4-5\": \"anthropic\",\n  \"claude-haiku-4-5-20251001\": \"anthropic\",\n  \"claude-fable-5\": \"anthropic\",\n  \"gemini-2.5-computer-use-preview-10-2025\": \"google\",\n  \"gemini-3-flash-preview\": \"google\",\n  \"gemini-3.5-flash\": \"google\",\n  \"gemini-3-pro-preview\": \"google\",\n  \"fara-7b\": \"microsoft\",\n};\n\n/**\n * Provider for agent clients\n * This class is responsible for creating the appropriate agent client\n * based on the provider type\n */\nexport class AgentProvider {\n  private logger: (message: LogLine) => void;\n\n  /**\n   * Create a new agent provider\n   */\n  constructor(logger: (message: LogLine) => void) {\n    this.logger = logger;\n  }\n\n  getClient(\n    modelName: string,\n    clientOptions?: ClientOptions,\n    userProvidedInstructions?: string,\n    tools?: ToolSet,\n  ): AgentClient {\n    // Check if provider is explicitly set in clientOptions\n    const explicitProvider = clientOptions?.provider as\n      | AgentProviderType\n      | undefined;\n    const type = explicitProvider || AgentProvider.getAgentProvider(modelName);\n\n    this.logger({\n      category: \"agent\",\n      message: `Getting agent client for type: ${type}, model: ${modelName}${explicitProvider ? \" (explicit provider)\" : \"\"}`,\n      level: 2,\n    });\n\n    try {\n      switch (type) {\n        case \"openai\":\n          return new OpenAICUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"anthropic\":\n          return new AnthropicCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"google\":\n          return new GoogleCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"microsoft\":\n          return new MicrosoftCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n          );\n        default:\n          throw new UnsupportedModelProviderError(\n            [\"openai\", \"anthropic\", \"google\", \"microsoft\"],\n            \"Computer Use Agent\",\n          );\n      }\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      this.logger({\n        category: \"agent\",\n        message: `Error creating agent client: ${errorMessage}`,\n        level: 0,\n      });\n      throw error;\n    }\n  }\n\n  static getAgentProvider(modelName: string): AgentProviderType {\n    const normalized = stripModelProvider(modelName);\n\n    if (normalized in modelToAgentProviderMap) {\n      return modelToAgentProviderMap[normalized];\n    }\n\n    throw new UnsupportedModelError(\n      Object.keys(modelToAgentProviderMap),\n      \"Computer Use Agent\",\n    );\n  }\n}\n"]}

package/dist/cjs/lib/v3/agent/tools/click.js

@@ -33,9 +33,18 @@ const clickTool = (v3, provider) => (0, ai_1.tool)({
             });
             // Only request XPath when caching is enabled to avoid unnecessary computation
             const shouldCollectXpath = v3.isAgentReplayActive();
-            const xpath = await page.click(processed.x, processed.y, {
-                returnXpath: shouldCollectXpath,
-            });
+            // On a touch session, actuate as a trusted tap rather than a mouse click:
+            // mobile layouts commonly gate their handlers on touch/pointer events, where
+            // a synthesized mouse click never registers (a size selector highlights but
+            // still reports "please choose a size"). Mirrors the CUA handler's routing.
+            const useTouch = v3.usesTouch;
+            const xpath = useTouch
+                ? await page.tap(processed.x, processed.y, {
+                    returnXpath: shouldCollectXpath,
+                })
+                : await page.click(processed.x, processed.y, {
+                    returnXpath: shouldCollectXpath,
+                });
             const screenshotBase64 = await (0, screenshotHandler_js_1.waitAndCaptureScreenshot)(page);
             // Record as an "act" step with proper Action for deterministic replay (only when caching)
             if (shouldCollectXpath) {
@@ -44,7 +53,9 @@ const clickTool = (v3, provider) => (0, ai_1.tool)({
                     const action = {
                         selector: normalizedXpath,
                         description: describe,
-                        method: "click",
+                        // Replay the same input kind that was recorded, so a cached mobile
+                        // run does not silently fall back to a mouse click.
+                        method: useTouch ? "tap" : "click",
                         arguments: [],
                     };
                     v3.recordAgentReplayStep({

package/dist/cjs/lib/v3/agent/tools/click.js.map

@@ -1 +1 @@
-{"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAOxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AAElE,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAA,SAAI,EAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,OAAC;aACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;gBACvD,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,OAAO;wBACf,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAmB,KAAK,CAAC,OAAO,EAAE;aAC1C,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC;AAvGQ,QAAA,SAAS,aAuGjB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  ClickToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: click`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const xpath = await page.click(processed.x, processed.y, {\n          returnXpath: shouldCollectXpath,\n        });\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"click\",\n              arguments: [],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          coordinates: [processed.x, processed.y],\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            coordinates: result.coordinates,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}
+{"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAOxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AAElE,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAA,SAAI,EAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,OAAC;aACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,0EAA0E;YAC1E,6EAA6E;YAC7E,4EAA4E;YAC5E,4EAA4E;YAC5E,MAAM,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;YAC9B,MAAM,KAAK,GAAG,QAAQ;gBACpB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;oBACvC,WAAW,EAAE,kBAAkB;iBAChC,CAAC;gBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;oBACzC,WAAW,EAAE,kBAAkB;iBAChC,CAAC,CAAC;YAEP,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,mEAAmE;wBACnE,oDAAoD;wBACpD,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO;wBAClC,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAmB,KAAK,CAAC,OAAO,EAAE;aAC1C,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC;AAlHQ,QAAA,SAAS,aAkHjB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  ClickToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: click`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        // On a touch session, actuate as a trusted tap rather than a mouse click:\n        // mobile layouts commonly gate their handlers on touch/pointer events, where\n        // a synthesized mouse click never registers (a size selector highlights but\n        // still reports \"please choose a size\"). Mirrors the CUA handler's routing.\n        const useTouch = v3.usesTouch;\n        const xpath = useTouch\n          ? await page.tap(processed.x, processed.y, {\n              returnXpath: shouldCollectXpath,\n            })\n          : await page.click(processed.x, processed.y, {\n              returnXpath: shouldCollectXpath,\n            });\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              // Replay the same input kind that was recorded, so a cached mobile\n              // run does not silently fall back to a mouse click.\n              method: useTouch ? \"tap\" : \"click\",\n              arguments: [],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          coordinates: [processed.x, processed.y],\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            coordinates: result.coordinates,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/cjs/lib/v3/agent/tools/fillFormVision.js

@@ -77,10 +77,18 @@ MANDATORY USE CASES (always use fillFormVision for these):
                 const shouldCollectXpath = v3.isAgentReplayActive();
                 const actions = [];
                 for (const field of safeFields) {
-                    // Click the field, only requesting XPath when caching is enabled
-                    const xpath = await page.click(field.coordinates.x, field.coordinates.y, {
-                        returnXpath: shouldCollectXpath,
-                    });
+                    // Focus the field, only requesting XPath when caching is enabled. On a
+                    // touch session this must be a trusted tap — a synthesized mouse click
+                    // never registers on touch-gated mobile layouts, so the field would stay
+                    // unfocused and the value go nowhere. The recorded step stays a "type",
+                    // whose replay focuses via the selector rather than a pointer event.
+                    const xpath = v3.usesTouch
+                        ? await page.tap(field.coordinates.x, field.coordinates.y, {
+                            returnXpath: shouldCollectXpath,
+                        })
+                        : await page.click(field.coordinates.x, field.coordinates.y, {
+                            returnXpath: shouldCollectXpath,
+                        });
                     // Substitute variables only at the moment of typing so the resolved
                     // value never leaves this scope.
                     await page.type((0, variables_js_1.substituteVariables)(field.value, variables));

package/dist/cjs/lib/v3/agent/tools/fillFormVision.js.map

@@ -1 +1 @@
-{"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAQxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AACzE,wDAA4D;AAErD,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,OAAC;iBACN,KAAK,CACJ,OAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,OAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,OAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAEhD,qEAAqE;gBACrE,qEAAqE;gBACrE,kEAAkE;gBAClE,mEAAmE;gBACnE,qBAAqB;gBACrB,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBACtC,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;oBACF,OAAO;wBACL,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;oBAC/B,iEAAiE;oBACjE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAC5B,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB;wBACE,WAAW,EAAE,kBAAkB;qBAChC,CACF,CAAC;oBACF,oEAAoE;oBACpE,iCAAiC;oBACjC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAA,kCAAmB,EAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;oBAE7D,wEAAwE;oBACxE,sEAAsE;oBACtE,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;6BACzB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,UAAU;oBAC/B,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAuB,KAAK,CAAC,OAAO,EAAE;iBAC9C,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gCACvB,KAAK,EAAE,MAAM,CAAC,KAAK;6BACpB,CAAC;yBACH;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AAjLW,QAAA,kBAAkB,sBAiL7B","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  FillFormVisionToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n  v3: V3,\n  provider?: string,\n  variables?: Variables,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const valueDescription = hasVariables\n    ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"Text to type into the target field\";\n\n  return tool({\n    description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                \"Description of the typing action, e.g. 'type foo into the bar field'\",\n              ),\n            value: z.string().describe(valueDescription),\n            coordinates: z\n              .object({\n                x: z.number(),\n                y: z.number(),\n              })\n              .describe(\"Coordinates of the target field\"),\n          }),\n        )\n        .min(2, \"Provide at least two fields to fill\"),\n    }),\n    execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n\n        // Process coordinates per field. Keep the original `value` (with any\n        // `%variableName%` tokens) so substituted secrets never leak through\n        // the tool result, replay cache, returned AgentResult.actions, or\n        // anything that logs the action. Variables are substituted only at\n        // typing time below.\n        const safeFields = fields.map((field) => {\n          const processed = processCoordinates(\n            field.coordinates.x,\n            field.coordinates.y,\n            provider,\n            v3,\n          );\n          return {\n            action: field.action,\n            value: field.value,\n            coordinates: { x: processed.x, y: processed.y },\n          };\n        });\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillFormVision`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ fields }), // Don't log substituted values\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const actions: Action[] = [];\n\n        for (const field of safeFields) {\n          // Click the field, only requesting XPath when caching is enabled\n          const xpath = await page.click(\n            field.coordinates.x,\n            field.coordinates.y,\n            {\n              returnXpath: shouldCollectXpath,\n            },\n          );\n          // Substitute variables only at the moment of typing so the resolved\n          // value never leaves this scope.\n          await page.type(substituteVariables(field.value, variables));\n\n          // Build Action with XPath for deterministic replay (only when caching).\n          // Store the placeholder value so cache entries don't contain secrets.\n          if (shouldCollectXpath) {\n            const normalizedXpath = ensureXPath(xpath);\n            if (normalizedXpath) {\n              actions.push({\n                selector: normalizedXpath,\n                description: field.action,\n                method: \"type\",\n                arguments: [field.value],\n              });\n            }\n          }\n\n          // Small delay between fields\n          await new Promise((resolve) => setTimeout(resolve, 100));\n        }\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n        // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n        if (shouldCollectXpath && actions.length > 0) {\n          v3.recordAgentReplayStep({\n            type: \"act\",\n            instruction: `Fill ${fields.length} form fields`,\n            actions,\n            actionDescription: `Fill ${fields.length} form fields`,\n          });\n        }\n\n        return {\n          success: true,\n          playwrightArguments: safeFields,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error filling form: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [\n            {\n              type: \"text\",\n              text: JSON.stringify({\n                success: result.success,\n                error: result.error,\n              }),\n            },\n          ],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            fieldsCount: result.playwrightArguments?.length ?? 0,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}
+{"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAQxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AACzE,wDAA4D;AAErD,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,OAAC;iBACN,KAAK,CACJ,OAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,OAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,OAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAEhD,qEAAqE;gBACrE,qEAAqE;gBACrE,kEAAkE;gBAClE,mEAAmE;gBACnE,qBAAqB;gBACrB,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBACtC,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;oBACF,OAAO;wBACL,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;oBAC/B,uEAAuE;oBACvE,uEAAuE;oBACvE,yEAAyE;oBACzE,wEAAwE;oBACxE,qEAAqE;oBACrE,MAAM,KAAK,GAAG,EAAE,CAAC,SAAS;wBACxB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE;4BACvD,WAAW,EAAE,kBAAkB;yBAChC,CAAC;wBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE;4BACzD,WAAW,EAAE,kBAAkB;yBAChC,CAAC,CAAC;oBACP,oEAAoE;oBACpE,iCAAiC;oBACjC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAA,kCAAmB,EAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;oBAE7D,wEAAwE;oBACxE,sEAAsE;oBACtE,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;6BACzB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,UAAU;oBAC/B,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAuB,KAAK,CAAC,OAAO,EAAE;iBAC9C,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gCACvB,KAAK,EAAE,MAAM,CAAC,KAAK;6BACpB,CAAC;yBACH;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AArLW,QAAA,kBAAkB,sBAqL7B","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  FillFormVisionToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n  v3: V3,\n  provider?: string,\n  variables?: Variables,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const valueDescription = hasVariables\n    ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"Text to type into the target field\";\n\n  return tool({\n    description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                \"Description of the typing action, e.g. 'type foo into the bar field'\",\n              ),\n            value: z.string().describe(valueDescription),\n            coordinates: z\n              .object({\n                x: z.number(),\n                y: z.number(),\n              })\n              .describe(\"Coordinates of the target field\"),\n          }),\n        )\n        .min(2, \"Provide at least two fields to fill\"),\n    }),\n    execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n\n        // Process coordinates per field. Keep the original `value` (with any\n        // `%variableName%` tokens) so substituted secrets never leak through\n        // the tool result, replay cache, returned AgentResult.actions, or\n        // anything that logs the action. Variables are substituted only at\n        // typing time below.\n        const safeFields = fields.map((field) => {\n          const processed = processCoordinates(\n            field.coordinates.x,\n            field.coordinates.y,\n            provider,\n            v3,\n          );\n          return {\n            action: field.action,\n            value: field.value,\n            coordinates: { x: processed.x, y: processed.y },\n          };\n        });\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillFormVision`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ fields }), // Don't log substituted values\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const actions: Action[] = [];\n\n        for (const field of safeFields) {\n          // Focus the field, only requesting XPath when caching is enabled. On a\n          // touch session this must be a trusted tap — a synthesized mouse click\n          // never registers on touch-gated mobile layouts, so the field would stay\n          // unfocused and the value go nowhere. The recorded step stays a \"type\",\n          // whose replay focuses via the selector rather than a pointer event.\n          const xpath = v3.usesTouch\n            ? await page.tap(field.coordinates.x, field.coordinates.y, {\n                returnXpath: shouldCollectXpath,\n              })\n            : await page.click(field.coordinates.x, field.coordinates.y, {\n                returnXpath: shouldCollectXpath,\n              });\n          // Substitute variables only at the moment of typing so the resolved\n          // value never leaves this scope.\n          await page.type(substituteVariables(field.value, variables));\n\n          // Build Action with XPath for deterministic replay (only when caching).\n          // Store the placeholder value so cache entries don't contain secrets.\n          if (shouldCollectXpath) {\n            const normalizedXpath = ensureXPath(xpath);\n            if (normalizedXpath) {\n              actions.push({\n                selector: normalizedXpath,\n                description: field.action,\n                method: \"type\",\n                arguments: [field.value],\n              });\n            }\n          }\n\n          // Small delay between fields\n          await new Promise((resolve) => setTimeout(resolve, 100));\n        }\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n        // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n        if (shouldCollectXpath && actions.length > 0) {\n          v3.recordAgentReplayStep({\n            type: \"act\",\n            instruction: `Fill ${fields.length} form fields`,\n            actions,\n            actionDescription: `Fill ${fields.length} form fields`,\n          });\n        }\n\n        return {\n          success: true,\n          playwrightArguments: safeFields,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error filling form: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [\n            {\n              type: \"text\",\n              text: JSON.stringify({\n                success: result.success,\n                error: result.error,\n              }),\n            },\n          ],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            fieldsCount: result.playwrightArguments?.length ?? 0,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}

package/dist/cjs/lib/v3/agent/tools/type.js

@@ -42,9 +42,18 @@ const typeTool = (v3, provider, variables) => {
                 });
                 // Only request XPath when caching is enabled to avoid unnecessary computation
                 const shouldCollectXpath = v3.isAgentReplayActive();
-                const xpath = await page.click(processed.x, processed.y, {
-                    returnXpath: shouldCollectXpath,
-                });
+                // Focus the field the way the session's input actually works: on a touch
+                // session a synthesized mouse click never registers on touch-gated mobile
+                // layouts, so the field would stay unfocused and the text go nowhere.
+                // Only the actuation changes — the recorded step is a "type", whose replay
+                // focuses via the selector rather than a pointer event.
+                const xpath = v3.usesTouch
+                    ? await page.tap(processed.x, processed.y, {
+                        returnXpath: shouldCollectXpath,
+                    })
+                    : await page.click(processed.x, processed.y, {
+                        returnXpath: shouldCollectXpath,
+                    });
                 await page.type(actualText);
                 const screenshotBase64 = await (0, screenshotHandler_js_1.waitAndCaptureScreenshot)(page);
                 // Record as an "act" step with proper Action for deterministic replay (only when caching)

package/dist/cjs/lib/v3/agent/tools/type.js.map

@@ -1 +1 @@
-{"version":3,"file":"type.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/type.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAQxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AACzE,wDAA4D;AAErD,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,SAAqB,EAAE,EAAE;IAC3E,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,YAAY;QAClC,CAAC,CAAC,oGAAoG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACzI,CAAC,CAAC,mCAAmC,CAAC;IAExC,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EACT,2QAA2Q;QAC7Q,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,QAAQ,EAAE,OAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,CACP,4HAA4H,CAC7H;YACH,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;YAC1C,WAAW,EAAE,OAAC;iBACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,QAAQ,CAAC,iDAAiD,CAAC;SAC/D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,WAAW,EACX,IAAI,GACL,EAA2B,EAAE;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;gBAEF,mDAAmD;gBACnD,MAAM,UAAU,GAAG,IAAA,kCAAmB,EAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAExD,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,0BAA0B;oBACnC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;4BACzC,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;oBACvD,WAAW,EAAE,kBAAkB;iBAChC,CAAC,CAAC;gBAEH,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE5B,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;gBAE9D,0FAA0F;gBAC1F,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;oBAC3C,IAAI,eAAe,EAAE,CAAC;wBACpB,MAAM,MAAM,GAAW;4BACrB,QAAQ,EAAE,eAAe;4BACzB,WAAW,EAAE,QAAQ;4BACrB,MAAM,EAAE,MAAM;4BACd,SAAS,EAAE,CAAC,IAAI,CAAC;yBAClB,CAAC;wBACF,EAAE,CAAC,qBAAqB,CAAC;4BACvB,IAAI,EAAE,KAAK;4BACX,WAAW,EAAE,QAAQ;4BACrB,OAAO,EAAE,CAAC,MAAM,CAAC;4BACjB,iBAAiB,EAAE,QAAQ;yBAC5B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,QAAQ;oBACR,IAAI,EAAE,8FAA8F;oBACpG,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,iBAAiB,KAAK,CAAC,OAAO,EAAE;iBACxC,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;iBACxD,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AAvHW,QAAA,QAAQ,YAuHnB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  TypeToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const typeTool = (v3: V3, provider?: string, variables?: Variables) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const textDescription = hasVariables\n    ? `The text to type into the element. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"The text to type into the element\";\n\n  return tool({\n    description:\n      \"Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      text: z.string().describe(textDescription),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to type into the element\"),\n    }),\n    execute: async ({\n      describe,\n      coordinates,\n      text,\n    }): Promise<TypeToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        // Substitute any %variableName% tokens in the text\n        const actualText = substituteVariables(text, variables);\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: type`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe, text }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const xpath = await page.click(processed.x, processed.y, {\n          returnXpath: shouldCollectXpath,\n        });\n\n        await page.type(actualText);\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"type\",\n              arguments: [text],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          text, // Return original text (with %variableName% tokens) to avoid exposing sensitive values to LLM\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error typing: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            text: result.text,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}
+{"version":3,"file":"type.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/type.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAQxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AACzE,wDAA4D;AAErD,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,SAAqB,EAAE,EAAE;IAC3E,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,YAAY;QAClC,CAAC,CAAC,oGAAoG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACzI,CAAC,CAAC,mCAAmC,CAAC;IAExC,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EACT,2QAA2Q;QAC7Q,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,QAAQ,EAAE,OAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,CACP,4HAA4H,CAC7H;YACH,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;YAC1C,WAAW,EAAE,OAAC;iBACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,QAAQ,CAAC,iDAAiD,CAAC;SAC/D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,WAAW,EACX,IAAI,GACL,EAA2B,EAAE;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;gBAEF,mDAAmD;gBACnD,MAAM,UAAU,GAAG,IAAA,kCAAmB,EAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAExD,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,0BAA0B;oBACnC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;4BACzC,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,yEAAyE;gBACzE,0EAA0E;gBAC1E,sEAAsE;gBACtE,2EAA2E;gBAC3E,wDAAwD;gBACxD,MAAM,KAAK,GAAG,EAAE,CAAC,SAAS;oBACxB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;wBACvC,WAAW,EAAE,kBAAkB;qBAChC,CAAC;oBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;wBACzC,WAAW,EAAE,kBAAkB;qBAChC,CAAC,CAAC;gBAEP,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE5B,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;gBAE9D,0FAA0F;gBAC1F,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;oBAC3C,IAAI,eAAe,EAAE,CAAC;wBACpB,MAAM,MAAM,GAAW;4BACrB,QAAQ,EAAE,eAAe;4BACzB,WAAW,EAAE,QAAQ;4BACrB,MAAM,EAAE,MAAM;4BACd,SAAS,EAAE,CAAC,IAAI,CAAC;yBAClB,CAAC;wBACF,EAAE,CAAC,qBAAqB,CAAC;4BACvB,IAAI,EAAE,KAAK;4BACX,WAAW,EAAE,QAAQ;4BACrB,OAAO,EAAE,CAAC,MAAM,CAAC;4BACjB,iBAAiB,EAAE,QAAQ;yBAC5B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,QAAQ;oBACR,IAAI,EAAE,8FAA8F;oBACpG,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,iBAAiB,KAAK,CAAC,OAAO,EAAE;iBACxC,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;iBACxD,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AAhIW,QAAA,QAAQ,YAgInB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  TypeToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const typeTool = (v3: V3, provider?: string, variables?: Variables) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const textDescription = hasVariables\n    ? `The text to type into the element. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"The text to type into the element\";\n\n  return tool({\n    description:\n      \"Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      text: z.string().describe(textDescription),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to type into the element\"),\n    }),\n    execute: async ({\n      describe,\n      coordinates,\n      text,\n    }): Promise<TypeToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        // Substitute any %variableName% tokens in the text\n        const actualText = substituteVariables(text, variables);\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: type`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe, text }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        // Focus the field the way the session's input actually works: on a touch\n        // session a synthesized mouse click never registers on touch-gated mobile\n        // layouts, so the field would stay unfocused and the text go nowhere.\n        // Only the actuation changes — the recorded step is a \"type\", whose replay\n        // focuses via the selector rather than a pointer event.\n        const xpath = v3.usesTouch\n          ? await page.tap(processed.x, processed.y, {\n              returnXpath: shouldCollectXpath,\n            })\n          : await page.click(processed.x, processed.y, {\n              returnXpath: shouldCollectXpath,\n            });\n\n        await page.type(actualText);\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"type\",\n              arguments: [text],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          text, // Return original text (with %variableName% tokens) to avoid exposing sensitive values to LLM\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error typing: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            text: result.text,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}

package/dist/cjs/lib/v3/agent/utils/handleDoneToolCall.js

@@ -7,6 +7,7 @@ const zod_1 = require("zod");
 const ai_2 = require("ai");
 const anthropicOptions_js_1 = require("../../llm/anthropicOptions.js");
 const utils_js_1 = require("../../../utils.js");
+const providerOptions_js_1 = require("../../llm/providerOptions.js");
 /**
  * Deep-remove `undefined` values from the run history before it is re-submitted
  * to the forced "done" call.
@@ -109,6 +110,7 @@ Call the "done" tool with:
             : "Provide your final assessment.",
     };
     const modelId = typeof model === "string" ? model : model.modelId;
+    const modelProvider = typeof model === "string" ? undefined : model.provider;
     const fallbacks = (0, anthropicOptions_js_1.anthropicFallbacksOptions)(modelId);
     // Models whose always-on thinking rejects forced tool use go straight to
     // "auto" — the prompt already instructs calling "done", and the
@@ -123,7 +125,7 @@ Call the "done" tool with:
             : { type: "tool", toolName: "done" },
         providerOptions: {
             google: { mediaResolution: "MEDIA_RESOLUTION_HIGH" },
-            openai: { store: false },
+            ...(0, providerOptions_js_1.openAIStoreProviderOptions)(modelProvider),
             ...(fallbacks ? { anthropic: fallbacks } : {}),
         },
     });

package/dist/cjs/lib/v3/agent/utils/handleDoneToolCall.js.map

@@ -1 +1 @@
-{"version":3,"file":"handleDoneToolCall.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/handleDoneToolCall.ts"],"names":[],"mappings":";;AAqDA,0EAIC;AAkBD,gDA4HC;AAvMD,2BAAwE;AACxE,6BAAwB;AACxB,2BAA0B;AAE1B,uEAGuC;AAEvC,gDAAgD;AAUhD;;;;;;;;;;;;;;;;GAgBG;AACH,SAAS,kBAAkB,CAAI,KAAQ;IACrC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAiB,CAAC;IACjE,CAAC;IACD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,KAAK,KAAK,MAAM,CAAC,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjD,MAAM,GAAG,GAA4B,EAAE,CAAC;YACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,EAAE,CAAC;gBACtE,IAAI,CAAC,KAAK,SAAS;oBAAE,GAAG,CAAC,CAAC,CAAC,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC;YACtD,CAAC;YACD,OAAO,GAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAgB,+BAA+B,CAC7C,QAAwB;IAExB,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAiB;IAC5C,OAAO,OAAO,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,OAAO;aACf,MAAM,EAAE;aACR,QAAQ,CAAC,0DAA0D,CAAC;QACvE,YAAY,EAAE,OAAO;aAClB,OAAO,EAAE;aACT,QAAQ,CAAC,uDAAuD,CAAC;KACrE,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACI,KAAK,UAAU,kBAAkB,CAAC,OAMxC;IACC,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE5E,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,0BAA0B;QACnC,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IACH,4EAA4E;IAC5E,MAAM,OAAO,GAAG,YAAY;QAC1B,CAAC,CAAC,IAAA,sBAAW,EAAC,YAAkC,CAAC;QACjD,CAAC,CAAC,OAAC,CAAC;IACN,MAAM,cAAc,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEpD,qEAAqE;IACrE,MAAM,cAAc,GAAG,YAAY;QACjC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,YAAY,CAAC,QAAQ,CAC3B,qDAAqD,CACtD;SACF,CAAC;QACJ,CAAC,CAAC,cAAc,CAAC;IAEnB,MAAM,kBAAkB,GAAG,YAAY;QACrC,CAAC,CAAC,4GAA4G,IAAI,CAAC,SAAS,CACxH,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,GAAG,CACpC,CAAC,CAAC,GAAG,EAAE,KAAK,CAA+B,EAAE,EAAE,CAAC;YAC9C,GAAG;YACH,KAAK,CAAC,WAAW,IAAI,gBAAgB;SACtC,CACF,CACF,EACD,IAAI,EACJ,CAAC,CACF,EAAE;QACL,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,YAAY,GAAG;;;GAGpB,WAAW;;mPAEqO,kBAAkB;;;;gDAIrN,YAAY,CAAC,CAAC,CAAC,wDAAwD,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAE7H,MAAM,QAAQ,GAAG,IAAA,SAAI,EAAC;QACpB,WAAW,EAAE,YAAY;YACvB,CAAC,CAAC,uEAAuE;YACzE,CAAC,CAAC,+CAA+C;QACnD,WAAW,EAAE,cAAc;QAC3B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;QACtC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,GAAiB;QAC/B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,YAAY;YACnB,CAAC,CAAC,8DAA8D;YAChE,CAAC,CAAC,gCAAgC;KACrC,CAAC;IAEF,MAAM,OAAO,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;IAClE,MAAM,SAAS,GAAG,IAAA,+CAAyB,EAAC,OAAO,CAAC,CAAC;IAErD,yEAAyE;IACzE,gEAAgE;IAChE,uDAAuD;IACvD,MAAM,MAAM,GAAG,MAAM,IAAA,iBAAY,EAAC;QAChC,KAAK;QACL,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE,CAAC,GAAG,+BAA+B,CAAC,aAAa,CAAC,EAAE,UAAU,CAAC;QACzE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAa;QACpC,UAAU,EAAE,IAAA,0CAAoB,EAAC,OAAO,CAAC;YACvC,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE;QACtC,eAAe,EAAE;YACf,MAAM,EAAE,EAAE,eAAe,EAAE,uBAAuB,EAAE;YACpD,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;YACxB,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/C;KACF,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IAC5E,MAAM,cAAc,GAAmB;QACrC,UAAU;QACV,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,IAAI,EAAE,CAAC;KACrC,CAAC;IAEF,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO;YACL,SAAS,EAAE,MAAM,CAAC,IAAI,IAAI,0BAA0B;YACpD,YAAY,EAAE,KAAK;YACnB,QAAQ,EAAE,cAAc;SACzB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,KAI1B,CAAC;IACF,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,gBAAgB;QACzB,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IAEH,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,QAAQ,EAAE,cAAc;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC;AACJ,CAAC","sourcesContent":["import { generateText, ModelMessage, LanguageModel, ToolSet } from \"ai\";\nimport { z } from \"zod\";\nimport { tool } from \"ai\";\nimport { LogLine } from \"../../types/public/logs.js\";\nimport {\n  anthropicFallbacksOptions,\n  rejectsForcedToolUse,\n} from \"../../llm/anthropicOptions.js\";\nimport { StagehandZodObject } from \"../../zodCompat.js\";\nimport { getZFactory } from \"../../../utils.js\";\nimport type { StagehandZodSchema } from \"../../zodCompat.js\";\n\ninterface DoneResult {\n  reasoning: string;\n  taskComplete: boolean;\n  messages: ModelMessage[];\n  output?: Record<string, unknown>;\n}\n\n/**\n * Deep-remove `undefined` values from the run history before it is re-submitted\n * to the forced \"done\" call.\n *\n * The AI SDK validates re-submitted messages against its `ModelMessage` schema,\n * whose JSON-value schema rejects `undefined` (only null/string/number/boolean/\n * object/array are allowed). A custom tool that returns an object with an\n * optional field left `undefined` (e.g. `{ matchedExpected: undefined }`) lands\n * that `undefined` inside a tool-result `output.value`, and reasoning models can\n * leave `undefined` in part fields too. Either makes the forced \"done\" call\n * throw `Invalid prompt: messages must be a ModelMessage[]` (STG-2335) — a red\n * error that fires after the run has already completed. Stripping `undefined`\n * keeps the history valid without dropping any real content.\n *\n * Only plain objects and arrays are traversed; class instances (URL, typed\n * arrays for binary image data, Date, …) are passed through untouched.\n */\nfunction stripUndefinedDeep<T>(value: T): T {\n  if (Array.isArray(value)) {\n    return value.map((v) => stripUndefinedDeep(v)) as unknown as T;\n  }\n  if (value !== null && typeof value === \"object\") {\n    const proto = Object.getPrototypeOf(value);\n    if (proto === Object.prototype || proto === null) {\n      const out: Record<string, unknown> = {};\n      for (const [k, v] of Object.entries(value as Record<string, unknown>)) {\n        if (v !== undefined) out[k] = stripUndefinedDeep(v);\n      }\n      return out as T;\n    }\n  }\n  return value;\n}\n\nexport function sanitizeMessagesForResubmission(\n  messages: ModelMessage[],\n): ModelMessage[] {\n  return stripUndefinedDeep(messages);\n}\n\nfunction buildBaseDoneSchema(factory: typeof z) {\n  return factory.object({\n    reasoning: factory\n      .string()\n      .describe(\"Brief summary of what actions were taken and the outcome\"),\n    taskComplete: factory\n      .boolean()\n      .describe(\"true if the task was fully completed, false otherwise\"),\n  });\n}\n\n/**\n * Force a done tool call at the end of an agent run.\n * This ensures we always get a structured final response,\n * even if the main loop ended without calling done.\n */\nexport async function handleDoneToolCall(options: {\n  model: LanguageModel;\n  inputMessages: ModelMessage[];\n  instruction: string;\n  outputSchema?: StagehandZodObject;\n  logger: (message: LogLine) => void;\n}): Promise<DoneResult> {\n  const { model, inputMessages, instruction, outputSchema, logger } = options;\n\n  logger({\n    category: \"agent\",\n    message: \"Agent calling tool: done\",\n    level: 1,\n  });\n  // Use the same Zod version as the user's outputSchema to avoid v3/v4 mixing\n  const factory = outputSchema\n    ? getZFactory(outputSchema as StagehandZodSchema)\n    : z;\n  const baseDoneSchema = buildBaseDoneSchema(factory);\n\n  // Merge base done schema with user-provided output schema if present\n  const doneToolSchema = outputSchema\n    ? baseDoneSchema.extend({\n        output: outputSchema.describe(\n          \"The specific data the user requested from this task\",\n        ),\n      })\n    : baseDoneSchema;\n\n  const outputInstructions = outputSchema\n    ? `\\n\\nThe user also requested the following information from this task. Provide it in the \"output\" field:\\n${JSON.stringify(\n        Object.fromEntries(\n          Object.entries(outputSchema.shape).map(\n            ([key, value]: [string, StagehandZodSchema]) => [\n              key,\n              value.description || \"no description\",\n            ],\n          ),\n        ),\n        null,\n        2,\n      )}`\n    : \"\";\n\n  const systemPrompt = `You are a web automation assistant that was tasked with completing a task.\n\nThe task was:\n\"${instruction}\"\n\nReview what was accomplished and provide your final assessment in whether the task was completed successfully. you have been provided with the history of the actions taken so far, use this to determine if the task was completed successfully.${outputInstructions}\n\nCall the \"done\" tool with:\n1. A brief summary of what was done\n2. Whether the task was completed successfully${outputSchema ? \"\\n3. The requested output data based on what you found\" : \"\"}`;\n\n  const doneTool = tool({\n    description: outputSchema\n      ? \"Complete the task with your assessment and the requested output data.\"\n      : \"Complete the task with your final assessment.\",\n    inputSchema: doneToolSchema,\n    execute: async (params) => {\n      return { success: true, ...params };\n    },\n  });\n\n  const userPrompt: ModelMessage = {\n    role: \"user\",\n    content: outputSchema\n      ? \"Provide your final assessment and the requested output data.\"\n      : \"Provide your final assessment.\",\n  };\n\n  const modelId = typeof model === \"string\" ? model : model.modelId;\n  const fallbacks = anthropicFallbacksOptions(modelId);\n\n  // Models whose always-on thinking rejects forced tool use go straight to\n  // \"auto\" — the prompt already instructs calling \"done\", and the\n  // no-tool-call case below handles a plain-text answer.\n  const result = await generateText({\n    model,\n    system: systemPrompt,\n    messages: [...sanitizeMessagesForResubmission(inputMessages), userPrompt],\n    tools: { done: doneTool } as ToolSet,\n    toolChoice: rejectsForcedToolUse(modelId)\n      ? \"auto\"\n      : { type: \"tool\", toolName: \"done\" },\n    providerOptions: {\n      google: { mediaResolution: \"MEDIA_RESOLUTION_HIGH\" },\n      openai: { store: false },\n      ...(fallbacks ? { anthropic: fallbacks } : {}),\n    },\n  });\n\n  const doneToolCall = result.toolCalls?.find((tc) => tc.toolName === \"done\");\n  const outputMessages: ModelMessage[] = [\n    userPrompt,\n    ...(result.response?.messages || []),\n  ];\n\n  if (!doneToolCall) {\n    return {\n      reasoning: result.text || \"Task execution completed\",\n      taskComplete: false,\n      messages: outputMessages,\n    };\n  }\n\n  const input = doneToolCall.input as {\n    reasoning: string;\n    taskComplete: boolean;\n    output?: Record<string, unknown>;\n  };\n  logger({\n    category: \"agent\",\n    message: `Task completed`,\n    level: 1,\n  });\n\n  return {\n    reasoning: input.reasoning,\n    taskComplete: input.taskComplete,\n    messages: outputMessages,\n    output: input.output,\n  };\n}\n"]}
+{"version":3,"file":"handleDoneToolCall.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/handleDoneToolCall.ts"],"names":[],"mappings":";;AAsDA,0EAIC;AAkBD,gDA6HC;AAzMD,2BAAwE;AACxE,6BAAwB;AACxB,2BAA0B;AAE1B,uEAGuC;AAEvC,gDAAgD;AAEhD,qEAA0E;AAS1E;;;;;;;;;;;;;;;;GAgBG;AACH,SAAS,kBAAkB,CAAI,KAAQ;IACrC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAiB,CAAC;IACjE,CAAC;IACD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,KAAK,KAAK,MAAM,CAAC,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjD,MAAM,GAAG,GAA4B,EAAE,CAAC;YACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,EAAE,CAAC;gBACtE,IAAI,CAAC,KAAK,SAAS;oBAAE,GAAG,CAAC,CAAC,CAAC,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC;YACtD,CAAC;YACD,OAAO,GAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAgB,+BAA+B,CAC7C,QAAwB;IAExB,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAiB;IAC5C,OAAO,OAAO,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,OAAO;aACf,MAAM,EAAE;aACR,QAAQ,CAAC,0DAA0D,CAAC;QACvE,YAAY,EAAE,OAAO;aAClB,OAAO,EAAE;aACT,QAAQ,CAAC,uDAAuD,CAAC;KACrE,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACI,KAAK,UAAU,kBAAkB,CAAC,OAMxC;IACC,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE5E,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,0BAA0B;QACnC,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IACH,4EAA4E;IAC5E,MAAM,OAAO,GAAG,YAAY;QAC1B,CAAC,CAAC,IAAA,sBAAW,EAAC,YAAkC,CAAC;QACjD,CAAC,CAAC,OAAC,CAAC;IACN,MAAM,cAAc,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEpD,qEAAqE;IACrE,MAAM,cAAc,GAAG,YAAY;QACjC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,YAAY,CAAC,QAAQ,CAC3B,qDAAqD,CACtD;SACF,CAAC;QACJ,CAAC,CAAC,cAAc,CAAC;IAEnB,MAAM,kBAAkB,GAAG,YAAY;QACrC,CAAC,CAAC,4GAA4G,IAAI,CAAC,SAAS,CACxH,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,GAAG,CACpC,CAAC,CAAC,GAAG,EAAE,KAAK,CAA+B,EAAE,EAAE,CAAC;YAC9C,GAAG;YACH,KAAK,CAAC,WAAW,IAAI,gBAAgB;SACtC,CACF,CACF,EACD,IAAI,EACJ,CAAC,CACF,EAAE;QACL,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,YAAY,GAAG;;;GAGpB,WAAW;;mPAEqO,kBAAkB;;;;gDAIrN,YAAY,CAAC,CAAC,CAAC,wDAAwD,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAE7H,MAAM,QAAQ,GAAG,IAAA,SAAI,EAAC;QACpB,WAAW,EAAE,YAAY;YACvB,CAAC,CAAC,uEAAuE;YACzE,CAAC,CAAC,+CAA+C;QACnD,WAAW,EAAE,cAAc;QAC3B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;QACtC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,GAAiB;QAC/B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,YAAY;YACnB,CAAC,CAAC,8DAA8D;YAChE,CAAC,CAAC,gCAAgC;KACrC,CAAC;IAEF,MAAM,OAAO,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;IAClE,MAAM,aAAa,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;IAC7E,MAAM,SAAS,GAAG,IAAA,+CAAyB,EAAC,OAAO,CAAC,CAAC;IAErD,yEAAyE;IACzE,gEAAgE;IAChE,uDAAuD;IACvD,MAAM,MAAM,GAAG,MAAM,IAAA,iBAAY,EAAC;QAChC,KAAK;QACL,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE,CAAC,GAAG,+BAA+B,CAAC,aAAa,CAAC,EAAE,UAAU,CAAC;QACzE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAa;QACpC,UAAU,EAAE,IAAA,0CAAoB,EAAC,OAAO,CAAC;YACvC,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE;QACtC,eAAe,EAAE;YACf,MAAM,EAAE,EAAE,eAAe,EAAE,uBAAuB,EAAE;YACpD,GAAG,IAAA,+CAA0B,EAAC,aAAa,CAAC;YAC5C,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/C;KACF,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IAC5E,MAAM,cAAc,GAAmB;QACrC,UAAU;QACV,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,IAAI,EAAE,CAAC;KACrC,CAAC;IAEF,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO;YACL,SAAS,EAAE,MAAM,CAAC,IAAI,IAAI,0BAA0B;YACpD,YAAY,EAAE,KAAK;YACnB,QAAQ,EAAE,cAAc;SACzB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,KAI1B,CAAC;IACF,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,gBAAgB;QACzB,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IAEH,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,QAAQ,EAAE,cAAc;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC;AACJ,CAAC","sourcesContent":["import { generateText, ModelMessage, LanguageModel, ToolSet } from \"ai\";\nimport { z } from \"zod\";\nimport { tool } from \"ai\";\nimport { LogLine } from \"../../types/public/logs.js\";\nimport {\n  anthropicFallbacksOptions,\n  rejectsForcedToolUse,\n} from \"../../llm/anthropicOptions.js\";\nimport { StagehandZodObject } from \"../../zodCompat.js\";\nimport { getZFactory } from \"../../../utils.js\";\nimport type { StagehandZodSchema } from \"../../zodCompat.js\";\nimport { openAIStoreProviderOptions } from \"../../llm/providerOptions.js\";\n\ninterface DoneResult {\n  reasoning: string;\n  taskComplete: boolean;\n  messages: ModelMessage[];\n  output?: Record<string, unknown>;\n}\n\n/**\n * Deep-remove `undefined` values from the run history before it is re-submitted\n * to the forced \"done\" call.\n *\n * The AI SDK validates re-submitted messages against its `ModelMessage` schema,\n * whose JSON-value schema rejects `undefined` (only null/string/number/boolean/\n * object/array are allowed). A custom tool that returns an object with an\n * optional field left `undefined` (e.g. `{ matchedExpected: undefined }`) lands\n * that `undefined` inside a tool-result `output.value`, and reasoning models can\n * leave `undefined` in part fields too. Either makes the forced \"done\" call\n * throw `Invalid prompt: messages must be a ModelMessage[]` (STG-2335) — a red\n * error that fires after the run has already completed. Stripping `undefined`\n * keeps the history valid without dropping any real content.\n *\n * Only plain objects and arrays are traversed; class instances (URL, typed\n * arrays for binary image data, Date, …) are passed through untouched.\n */\nfunction stripUndefinedDeep<T>(value: T): T {\n  if (Array.isArray(value)) {\n    return value.map((v) => stripUndefinedDeep(v)) as unknown as T;\n  }\n  if (value !== null && typeof value === \"object\") {\n    const proto = Object.getPrototypeOf(value);\n    if (proto === Object.prototype || proto === null) {\n      const out: Record<string, unknown> = {};\n      for (const [k, v] of Object.entries(value as Record<string, unknown>)) {\n        if (v !== undefined) out[k] = stripUndefinedDeep(v);\n      }\n      return out as T;\n    }\n  }\n  return value;\n}\n\nexport function sanitizeMessagesForResubmission(\n  messages: ModelMessage[],\n): ModelMessage[] {\n  return stripUndefinedDeep(messages);\n}\n\nfunction buildBaseDoneSchema(factory: typeof z) {\n  return factory.object({\n    reasoning: factory\n      .string()\n      .describe(\"Brief summary of what actions were taken and the outcome\"),\n    taskComplete: factory\n      .boolean()\n      .describe(\"true if the task was fully completed, false otherwise\"),\n  });\n}\n\n/**\n * Force a done tool call at the end of an agent run.\n * This ensures we always get a structured final response,\n * even if the main loop ended without calling done.\n */\nexport async function handleDoneToolCall(options: {\n  model: LanguageModel;\n  inputMessages: ModelMessage[];\n  instruction: string;\n  outputSchema?: StagehandZodObject;\n  logger: (message: LogLine) => void;\n}): Promise<DoneResult> {\n  const { model, inputMessages, instruction, outputSchema, logger } = options;\n\n  logger({\n    category: \"agent\",\n    message: \"Agent calling tool: done\",\n    level: 1,\n  });\n  // Use the same Zod version as the user's outputSchema to avoid v3/v4 mixing\n  const factory = outputSchema\n    ? getZFactory(outputSchema as StagehandZodSchema)\n    : z;\n  const baseDoneSchema = buildBaseDoneSchema(factory);\n\n  // Merge base done schema with user-provided output schema if present\n  const doneToolSchema = outputSchema\n    ? baseDoneSchema.extend({\n        output: outputSchema.describe(\n          \"The specific data the user requested from this task\",\n        ),\n      })\n    : baseDoneSchema;\n\n  const outputInstructions = outputSchema\n    ? `\\n\\nThe user also requested the following information from this task. Provide it in the \"output\" field:\\n${JSON.stringify(\n        Object.fromEntries(\n          Object.entries(outputSchema.shape).map(\n            ([key, value]: [string, StagehandZodSchema]) => [\n              key,\n              value.description || \"no description\",\n            ],\n          ),\n        ),\n        null,\n        2,\n      )}`\n    : \"\";\n\n  const systemPrompt = `You are a web automation assistant that was tasked with completing a task.\n\nThe task was:\n\"${instruction}\"\n\nReview what was accomplished and provide your final assessment in whether the task was completed successfully. you have been provided with the history of the actions taken so far, use this to determine if the task was completed successfully.${outputInstructions}\n\nCall the \"done\" tool with:\n1. A brief summary of what was done\n2. Whether the task was completed successfully${outputSchema ? \"\\n3. The requested output data based on what you found\" : \"\"}`;\n\n  const doneTool = tool({\n    description: outputSchema\n      ? \"Complete the task with your assessment and the requested output data.\"\n      : \"Complete the task with your final assessment.\",\n    inputSchema: doneToolSchema,\n    execute: async (params) => {\n      return { success: true, ...params };\n    },\n  });\n\n  const userPrompt: ModelMessage = {\n    role: \"user\",\n    content: outputSchema\n      ? \"Provide your final assessment and the requested output data.\"\n      : \"Provide your final assessment.\",\n  };\n\n  const modelId = typeof model === \"string\" ? model : model.modelId;\n  const modelProvider = typeof model === \"string\" ? undefined : model.provider;\n  const fallbacks = anthropicFallbacksOptions(modelId);\n\n  // Models whose always-on thinking rejects forced tool use go straight to\n  // \"auto\" — the prompt already instructs calling \"done\", and the\n  // no-tool-call case below handles a plain-text answer.\n  const result = await generateText({\n    model,\n    system: systemPrompt,\n    messages: [...sanitizeMessagesForResubmission(inputMessages), userPrompt],\n    tools: { done: doneTool } as ToolSet,\n    toolChoice: rejectsForcedToolUse(modelId)\n      ? \"auto\"\n      : { type: \"tool\", toolName: \"done\" },\n    providerOptions: {\n      google: { mediaResolution: \"MEDIA_RESOLUTION_HIGH\" },\n      ...openAIStoreProviderOptions(modelProvider),\n      ...(fallbacks ? { anthropic: fallbacks } : {}),\n    },\n  });\n\n  const doneToolCall = result.toolCalls?.find((tc) => tc.toolName === \"done\");\n  const outputMessages: ModelMessage[] = [\n    userPrompt,\n    ...(result.response?.messages || []),\n  ];\n\n  if (!doneToolCall) {\n    return {\n      reasoning: result.text || \"Task execution completed\",\n      taskComplete: false,\n      messages: outputMessages,\n    };\n  }\n\n  const input = doneToolCall.input as {\n    reasoning: string;\n    taskComplete: boolean;\n    output?: Record<string, unknown>;\n  };\n  logger({\n    category: \"agent\",\n    message: `Task completed`,\n    level: 1,\n  });\n\n  return {\n    reasoning: input.reasoning,\n    taskComplete: input.taskComplete,\n    messages: outputMessages,\n    output: input.output,\n  };\n}\n"]}

package/dist/cjs/lib/v3/api.d.ts

@@ -91,7 +91,7 @@ export declare class StagehandAPIClient {
     private latestAgentCacheEntry;
     private warnedStagehandBaseUrl;
     constructor({ apiKey, projectId, logger, serverCache, }: StagehandAPIConstructorParams);
-    init({ modelName, modelApiKey, defaultModelConfig, domSettleTimeoutMs, verbose, systemPrompt, selfHeal, browserbaseSessionCreateParams, browserbaseSessionID, }: ClientSessionStartParams): Promise<Api.SessionStartResult>;
+    init({ modelName, modelApiKey, defaultModelConfig, domSettleTimeoutMs, verbose, systemPrompt, selfHeal, useTouch, browserbaseSessionCreateParams, browserbaseSessionID, }: ClientSessionStartParams): Promise<Api.SessionStartResult>;
     act({ input, options, frameId, }: ClientActParameters): Promise<ActResult>;
     extract<T extends StagehandZodSchema>({ instruction, schema: zodSchema, options, frameId, }: ClientExtractParameters): Promise<ExtractResult<T>>;
     observe({ instruction, options, frameId, }: ClientObserveParameters): Promise<ObserveResult>;

package/dist/cjs/lib/v3/api.js

@@ -7,6 +7,7 @@ exports.StagehandAPIClient = exports.REGION_API_URLS = void 0;
 exports.getApiUrlForRegion = getApiUrlForRegion;
 const fetch_cookie_1 = __importDefault(require("fetch-cookie"));
 const utils_js_1 = require("../utils.js");
+const modelUtils_js_1 = require("../modelUtils.js");
 const version_js_1 = require("../version.js");
 const index_js_1 = require("./types/public/index.js");
 const zodCompat_js_1 = require("./zodCompat.js");
@@ -58,7 +59,7 @@ class StagehandAPIClient {
         // Create a single cookie jar instance that will persist across all requests
         this.fetchWithCookies = (0, fetch_cookie_1.default)(fetch);
     }
-    async init({ modelName, modelApiKey, defaultModelConfig, domSettleTimeoutMs, verbose, systemPrompt, selfHeal, browserbaseSessionCreateParams, browserbaseSessionID,
+    async init({ modelName, modelApiKey, defaultModelConfig, domSettleTimeoutMs, verbose, systemPrompt, selfHeal, useTouch, browserbaseSessionCreateParams, browserbaseSessionID,
     // browser,  TODO for local browsers
      }) {
         this.modelApiKey = modelApiKey;
@@ -83,6 +84,7 @@ class StagehandAPIClient {
             verbose,
             systemPrompt,
             selfHeal,
+            useTouch,
             browserbaseSessionCreateParams,
             browserbaseSessionID,
             // browser, TODO: only send when connected to local fastify
@@ -419,6 +421,14 @@ class StagehandAPIClient {
      * model provider differs from the one used to init the session.
      */
     prepareModelConfig(model) {
+        // "auto" delegates model selection to the API, so never inherit the
+        // default model's provider config or API key — the server picks the
+        // provider. Explicit per-call options are still passed through.
+        if ((0, modelUtils_js_1.isAutoModel)(model)) {
+            return typeof model === "string"
+                ? { modelName: model }
+                : { ...model };
+        }
         if (typeof model === "string") {
             // Extract provider from model string (e.g., "openai/gpt-5-nano" -> "openai")
             const provider = this.getModelProvider(model);