npm - @browserbasehq/orca - Versions diffs - 3.2.0-preview.3 → 3.2.0-preview.4 - Mend

@browserbasehq/orca 3.2.0-preview.3 → 3.2.0-preview.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (171) hide show

package/dist/esm/lib/v3/agent/tools/braveSearch.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"braveSearch.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/braveSearch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAgCxB,KAAK,UAAU,kBAAkB,CAAC,KAAa;IAC7C,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAC1B,oDAAoD,YAAY,EAAE,EAClE;YACE,MAAM,EAAE,KAAK;YACb,OAAO,EAAE;gBACP,MAAM,EAAE,kBAAkB;gBAC1B,iBAAiB,EAAE,MAAM;gBACzB,sBAAsB,EAAE,OAAO,CAAC,GAAG,CAAC,aAAc;aACnD;SACF,CACF,CAAC;QAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO;gBACL,KAAK,EAAE,oBAAoB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE;gBACnE,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;aACtB,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAqB,CAAC;QACzD,MAAM,OAAO,GAAwB,EAAE,CAAC;QAExC,IAAI,IAAI,EAAE,GAAG,EAAE,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1D,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;gBAChD,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;oBAC3B,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,WAAW,EAAE,IAAI,CAAC,WAAW;qBAC9B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QACrC,OAAO;YACL,KAAK,EAAE,~~4BAA6B~~,~~KAAe~~,CAAC,OAAO,EAAE;~~YAC7D~~,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;SACtB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,EAAE,CACnC,IAAI,CAAC;IACH,WAAW,EACT,wSAAwS;IAC1S,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;QAC3B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC;oBAChC,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAE/C,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,KAAK;YAClB,mBAAmB,EAAE,EAAE,KAAK,EAAE;YAC9B,OAAO,EACL,MAAM,CAAC,KAAK,IAAI,SAAS,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,UAAU;SACtE,CAAC,CAAC;QAEH,OAAO;YACL,GAAG,MAAM;YACT,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport interface BraveSearchResult {\n title: string;\n url: string;\n description?: string;\n}\n\ninterface SearchResponse {\n data?: {\n results: BraveSearchResult[];\n };\n error?: string;\n}\n\ninterface BraveWebResult {\n title?: string;\n url?: string;\n description?: string;\n age?: string;\n meta_url?: {\n favicon?: string;\n };\n}\n\ninterface BraveApiResponse {\n web?: {\n results?: BraveWebResult[];\n };\n}\n\nasync function performBraveSearch(query: string): Promise<SearchResponse> {\n try {\n const encodedQuery = encodeURIComponent(query);\n const response = await fetch(\n `https://api.search.brave.com/res/v1/web/search?q=${encodedQuery}`,\n {\n method: \"GET\",\n headers: {\n Accept: \"application/json\",\n \"Accept-Encoding\": \"gzip\",\n \"X-Subscription-Token\": process.env.BRAVE_API_KEY!,\n },\n },\n );\n\n if (!response.ok) {\n return {\n error: `Brave API error: ${response.status} ${response.statusText}`,\n data: { results: [] },\n };\n }\n\n const data = (await response.json()) as BraveApiResponse;\n const results: BraveSearchResult[] = [];\n\n if (data?.web?.results && Array.isArray(data.web.results)) {\n for (const item of data.web.results.slice(0, 5)) {\n if (item.title && item.url) {\n results.push({\n title: item.title,\n url: item.url,\n description: item.description,\n });\n }\n }\n }\n\n return { data: { results } };\n } catch (error) {\n console.error(\"Search error\", error);\n return {\n error: `Error performing search: ${(error ~~as Error)~~.message}`,\n data: { results: [] },\n };\n }\n}\n\nexport const searchTool = (v3: V3) =>\n tool({\n description:\n \"Perform a web search and returns results. Use this tool when you need information from the web or when you are unsure of the exact URL you want to navigate to. This can be used to find the ideal entry point, resulting in a task that is easier to complete due to starting further in the process.\",\n inputSchema: z.object({\n query: z.string().describe(\"The search query to look for on the web\"),\n }),\n execute: async ({ query }) => {\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: search`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ query }),\n type: \"object\",\n },\n },\n });\n\n const result = await performBraveSearch(query);\n\n v3.recordAgentReplayStep({\n type: \"search\",\n instruction: query,\n playwrightArguments: { query },\n message:\n result.error ?? `Found ${result.data?.results.length ?? 0} results`,\n });\n\n return {\n ...result,\n timestamp: Date.now(),\n };\n },\n });\n"]}
1	+ {"version":3,"file":"braveSearch.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/braveSearch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAgCxB,KAAK,UAAU,kBAAkB,CAAC,KAAa;IAC7C,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAC1B,oDAAoD,YAAY,EAAE,EAClE;YACE,MAAM,EAAE,KAAK;YACb,OAAO,EAAE;gBACP,MAAM,EAAE,kBAAkB;gBAC1B,iBAAiB,EAAE,MAAM;gBACzB,sBAAsB,EAAE,OAAO,CAAC,GAAG,CAAC,aAAc;aACnD;SACF,CACF,CAAC;QAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO;gBACL,KAAK,EAAE,oBAAoB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE;gBACnE,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;aACtB,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAqB,CAAC;QACzD,MAAM,OAAO,GAAwB,EAAE,CAAC;QAExC,IAAI,IAAI,EAAE,GAAG,EAAE,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1D,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;gBAChD,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;oBAC3B,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,WAAW,EAAE,IAAI,CAAC,WAAW;qBAC9B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QACrC,OAAO;YACL,KAAK,EAAE,4BAA4B,KAAK,CAAC,OAAO,EAAE;YAClD,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;SACtB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,EAAE,CACnC,IAAI,CAAC;IACH,WAAW,EACT,wSAAwS;IAC1S,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;QAC3B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC;oBAChC,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAE/C,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,KAAK;YAClB,mBAAmB,EAAE,EAAE,KAAK,EAAE;YAC9B,OAAO,EACL,MAAM,CAAC,KAAK,IAAI,SAAS,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,UAAU;SACtE,CAAC,CAAC;QAEH,OAAO;YACL,GAAG,MAAM;YACT,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport interface BraveSearchResult {\n title: string;\n url: string;\n description?: string;\n}\n\ninterface SearchResponse {\n data?: {\n results: BraveSearchResult[];\n };\n error?: string;\n}\n\ninterface BraveWebResult {\n title?: string;\n url?: string;\n description?: string;\n age?: string;\n meta_url?: {\n favicon?: string;\n };\n}\n\ninterface BraveApiResponse {\n web?: {\n results?: BraveWebResult[];\n };\n}\n\nasync function performBraveSearch(query: string): Promise<SearchResponse> {\n try {\n const encodedQuery = encodeURIComponent(query);\n const response = await fetch(\n `https://api.search.brave.com/res/v1/web/search?q=${encodedQuery}`,\n {\n method: \"GET\",\n headers: {\n Accept: \"application/json\",\n \"Accept-Encoding\": \"gzip\",\n \"X-Subscription-Token\": process.env.BRAVE_API_KEY!,\n },\n },\n );\n\n if (!response.ok) {\n return {\n error: `Brave API error: ${response.status} ${response.statusText}`,\n data: { results: [] },\n };\n }\n\n const data = (await response.json()) as BraveApiResponse;\n const results: BraveSearchResult[] = [];\n\n if (data?.web?.results && Array.isArray(data.web.results)) {\n for (const item of data.web.results.slice(0, 5)) {\n if (item.title && item.url) {\n results.push({\n title: item.title,\n url: item.url,\n description: item.description,\n });\n }\n }\n }\n\n return { data: { results } };\n } catch (error) {\n console.error(\"Search error\", error);\n return {\n error: `Error performing search: ${error.message}`,\n data: { results: [] },\n };\n }\n}\n\nexport const searchTool = (v3: V3) =>\n tool({\n description:\n \"Perform a web search and returns results. Use this tool when you need information from the web or when you are unsure of the exact URL you want to navigate to. This can be used to find the ideal entry point, resulting in a task that is easier to complete due to starting further in the process.\",\n inputSchema: z.object({\n query: z.string().describe(\"The search query to look for on the web\"),\n }),\n execute: async ({ query }) => {\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: search`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ query }),\n type: \"object\",\n },\n },\n });\n\n const result = await performBraveSearch(query);\n\n v3.recordAgentReplayStep({\n type: \"search\",\n instruction: query,\n playwrightArguments: { query },\n message:\n result.error ?? `Found ${result.data?.results.length ?? 0} results`,\n });\n\n return {\n ...result,\n timestamp: Date.now(),\n };\n },\n });\n"]}

package/dist/esm/lib/v3/agent/tools/browserbaseSearch.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"browserbaseSearch.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/browserbaseSearch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAmBxB,KAAK,UAAU,wBAAwB,CACrC,EAAM,EACN,KAAa,EACb,MAAc,EACd,aAAqB,CAAC;IAEtB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE;YACpE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,cAAc,EAAE,MAAM;aACvB;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;SAC5C,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO;gBACL,OAAO,EAAE,EAAE;gBACX,KAAK,EAAE,iCAAiC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE;aACjF,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA2B,CAAC;QAC/D,MAAM,OAAO,GAAmB,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CACvD,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC;YAClC,KAAK,EAAE,KAAK;YACZ,GAAG,EAAE,GAAG;YACR,GAAG,CAAC,aAAa,IAAI,EAAE,aAAa,EAAE,CAAC;SACxC,CAAC,CACH,CAAC;QAEF,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,~~iBAAkB~~,~~KAAe~~,CAAC,OAAO,EAAE;~~YACpD~~,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,OAAO;YACL,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,~~4BAA6B~~,~~KAAe~~,CAAC,OAAO,EAAE;~~SAC9D~~,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,MAAc,EAAE,EAAE,CACnD,IAAI,CAAC;IACH,WAAW,EACT,wSAAwS;IAC1S,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;QAC3B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC;oBAChC,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QAEjE,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,KAAK;YAClB,mBAAmB,EAAE,EAAE,KAAK,EAAE;YAC9B,OAAO,EAAE,MAAM,CAAC,KAAK,IAAI,SAAS,MAAM,CAAC,OAAO,CAAC,MAAM,UAAU;SAClE,CAAC,CAAC;QAEH,OAAO,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC9C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport interface SearchResult {\n title: string;\n url: string;\n publishedDate?: string;\n}\n\ninterface BrowserbaseRawResult {\n title?: string;\n url?: string;\n publishedDate?: string;\n}\n\ninterface BrowserbaseApiResponse {\n results?: BrowserbaseRawResult[];\n}\n\nasync function performBrowserbaseSearch(\n v3: V3,\n query: string,\n apiKey: string,\n numResults: number = 5,\n): Promise<{ results: SearchResult[]; error?: string }> {\n try {\n const response = await fetch(\"https://api.browserbase.com/v1/search\", {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n \"x-bb-api-key\": apiKey,\n },\n body: JSON.stringify({ query, numResults }),\n });\n\n if (!response.ok) {\n return {\n results: [],\n error: `Browserbase Search API error: ${response.status} ${response.statusText}`,\n };\n }\n\n const data = (await response.json()) as BrowserbaseApiResponse;\n const results: SearchResult[] = (data?.results ?? []).map(\n ({ title, url, publishedDate }) => ({\n title: title,\n url: url,\n ...(publishedDate && { publishedDate }),\n }),\n );\n\n return { results };\n } catch (error) {\n v3.logger({\n category: \"agent\",\n message: `Search error: ${(error ~~as Error)~~.message}`,\n level: 0,\n });\n return {\n results: [],\n error: `Error performing search: ${(error ~~as Error)~~.message}`,\n };\n }\n}\n\nexport const searchTool = (v3: V3, apiKey: string) =>\n tool({\n description:\n \"Perform a web search and returns results. Use this tool when you need information from the web or when you are unsure of the exact URL you want to navigate to. This can be used to find the ideal entry point, resulting in a task that is easier to complete due to starting further in the process.\",\n inputSchema: z.object({\n query: z.string().describe(\"The search query to look for on the web\"),\n }),\n execute: async ({ query }) => {\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: search`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ query }),\n type: \"object\",\n },\n },\n });\n\n const result = await performBrowserbaseSearch(v3, query, apiKey);\n\n v3.recordAgentReplayStep({\n type: \"search\",\n instruction: query,\n playwrightArguments: { query },\n message: result.error ?? `Found ${result.results.length} results`,\n });\n\n return { ...result, timestamp: Date.now() };\n },\n });\n"]}
1	+ {"version":3,"file":"browserbaseSearch.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/browserbaseSearch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAmBxB,KAAK,UAAU,wBAAwB,CACrC,EAAM,EACN,KAAa,EACb,MAAc,EACd,aAAqB,CAAC;IAEtB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE;YACpE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,cAAc,EAAE,MAAM;aACvB;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;SAC5C,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO;gBACL,OAAO,EAAE,EAAE;gBACX,KAAK,EAAE,iCAAiC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE;aACjF,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA2B,CAAC;QAC/D,MAAM,OAAO,GAAmB,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CACvD,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC;YAClC,KAAK,EAAE,KAAK;YACZ,GAAG,EAAE,GAAG;YACR,GAAG,CAAC,aAAa,IAAI,EAAE,aAAa,EAAE,CAAC;SACxC,CAAC,CACH,CAAC;QAEF,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,iBAAiB,KAAK,CAAC,OAAO,EAAE;YACzC,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,OAAO;YACL,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,4BAA4B,KAAK,CAAC,OAAO,EAAE;SACnD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,MAAc,EAAE,EAAE,CACnD,IAAI,CAAC;IACH,WAAW,EACT,wSAAwS;IAC1S,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;QAC3B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC;oBAChC,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QAEjE,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,KAAK;YAClB,mBAAmB,EAAE,EAAE,KAAK,EAAE;YAC9B,OAAO,EAAE,MAAM,CAAC,KAAK,IAAI,SAAS,MAAM,CAAC,OAAO,CAAC,MAAM,UAAU;SAClE,CAAC,CAAC;QAEH,OAAO,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC9C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport interface SearchResult {\n title: string;\n url: string;\n publishedDate?: string;\n}\n\ninterface BrowserbaseRawResult {\n title?: string;\n url?: string;\n publishedDate?: string;\n}\n\ninterface BrowserbaseApiResponse {\n results?: BrowserbaseRawResult[];\n}\n\nasync function performBrowserbaseSearch(\n v3: V3,\n query: string,\n apiKey: string,\n numResults: number = 5,\n): Promise<{ results: SearchResult[]; error?: string }> {\n try {\n const response = await fetch(\"https://api.browserbase.com/v1/search\", {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n \"x-bb-api-key\": apiKey,\n },\n body: JSON.stringify({ query, numResults }),\n });\n\n if (!response.ok) {\n return {\n results: [],\n error: `Browserbase Search API error: ${response.status} ${response.statusText}`,\n };\n }\n\n const data = (await response.json()) as BrowserbaseApiResponse;\n const results: SearchResult[] = (data?.results ?? []).map(\n ({ title, url, publishedDate }) => ({\n title: title,\n url: url,\n ...(publishedDate && { publishedDate }),\n }),\n );\n\n return { results };\n } catch (error) {\n v3.logger({\n category: \"agent\",\n message: `Search error: ${error.message}`,\n level: 0,\n });\n return {\n results: [],\n error: `Error performing search: ${error.message}`,\n };\n }\n}\n\nexport const searchTool = (v3: V3, apiKey: string) =>\n tool({\n description:\n \"Perform a web search and returns results. Use this tool when you need information from the web or when you are unsure of the exact URL you want to navigate to. This can be used to find the ideal entry point, resulting in a task that is easier to complete due to starting further in the process.\",\n inputSchema: z.object({\n query: z.string().describe(\"The search query to look for on the web\"),\n }),\n execute: async ({ query }) => {\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: search`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ query }),\n type: \"object\",\n },\n },\n });\n\n const result = await performBrowserbaseSearch(v3, query, apiKey);\n\n v3.recordAgentReplayStep({\n type: \"search\",\n instruction: query,\n playwrightArguments: { query },\n message: result.error ?? `Found ${result.results.length} results`,\n });\n\n return { ...result, timestamp: Date.now() };\n },\n });\n"]}

package/dist/esm/lib/v3/agent/tools/click.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAI,CAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;gBACvD,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,OAAO;wBACf,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,~~mBAAoB~~,~~KAAe~~,CAAC,OAAO,EAAE;~~aACrD~~,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n ClickToolResult,\n ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n tool({\n description:\n \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n inputSchema: z.object({\n describe: z\n .string()\n .describe(\n \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n ),\n coordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to click on\"),\n }),\n execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const processed = processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: click`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ describe }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const xpath = await page.click(processed.x, processed.y, {\n returnXpath: shouldCollectXpath,\n });\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n const action: Action = {\n selector: normalizedXpath,\n description: describe,\n method: \"click\",\n arguments: [],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return {\n success: true,\n describe,\n coordinates: [processed.x, processed.y],\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error clicking: ${(error ~~as Error)~~.message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n describe: result.describe,\n coordinates: result.coordinates,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n"]}
1	+ {"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAI,CAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;gBACvD,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,OAAO;wBACf,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAmB,KAAK,CAAC,OAAO,EAAE;aAC1C,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n ClickToolResult,\n ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n tool({\n description:\n \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n inputSchema: z.object({\n describe: z\n .string()\n .describe(\n \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n ),\n coordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to click on\"),\n }),\n execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const processed = processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: click`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ describe }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const xpath = await page.click(processed.x, processed.y, {\n returnXpath: shouldCollectXpath,\n });\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n const action: Action = {\n selector: normalizedXpath,\n description: describe,\n method: \"click\",\n arguments: [],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return {\n success: true,\n describe,\n coordinates: [processed.x, processed.y],\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error clicking: ${error.message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n describe: result.describe,\n coordinates: result.coordinates,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n"]}

package/dist/esm/lib/v3/agent/tools/clickAndHold.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"clickAndHold.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/clickAndHold.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAI,CAAC;IACH,WAAW,EAAE,oDAAoD;IACjE,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CAAC,kDAAkD,CAAC;QAC/D,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,EAAE;QACrD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,kCAAkC;gBAC3C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;4BACR,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YAEpD,sFAAsF;YACtF,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACpC,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACrD,CAAC;YAEF,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,cAAc;wBACtB,SAAS,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;qBAC9B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,+~~BAAgC~~,~~KAAe~~,CAAC,OAAO,EAAE;~~aACjE~~,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\n\nexport const clickAndHoldTool = (v3: V3, provider?: string) =>\n tool({\n description: \"Click and hold on an element using its coordinates\",\n inputSchema: z.object({\n describe: z\n .string()\n .describe(\n \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n ),\n duration: z\n .number()\n .describe(\"The duration to hold the element in milliseconds\"),\n coordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to click on\"),\n }),\n execute: async ({ describe, coordinates, duration }) => {\n try {\n const page = await v3.context.awaitActivePage();\n const processed = processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: clickAndHold`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({\n describe,\n duration,\n }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n\n // Use dragAndDrop from same point to same point with delay to simulate click and hold\n const [xpath] = await page.dragAndDrop(\n processed.x,\n processed.y,\n processed.x,\n processed.y,\n { delay: duration, returnXpath: shouldCollectXpath },\n );\n\n // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n const action: Action = {\n selector: normalizedXpath,\n description: describe,\n method: \"clickAndHold\",\n arguments: [String(duration)],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return { success: true, describe };\n } catch (error) {\n return {\n success: false,\n error: `Error clicking and holding: ${(error ~~as Error)~~.message}`,\n };\n }\n },\n });\n"]}
1	+ {"version":3,"file":"clickAndHold.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/clickAndHold.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAI,CAAC;IACH,WAAW,EAAE,oDAAoD;IACjE,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CAAC,kDAAkD,CAAC;QAC/D,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,EAAE;QACrD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,kCAAkC;gBAC3C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;4BACR,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YAEpD,sFAAsF;YACtF,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACpC,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACrD,CAAC;YAEF,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,cAAc;wBACtB,SAAS,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;qBAC9B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,+BAA+B,KAAK,CAAC,OAAO,EAAE;aACtD,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\n\nexport const clickAndHoldTool = (v3: V3, provider?: string) =>\n tool({\n description: \"Click and hold on an element using its coordinates\",\n inputSchema: z.object({\n describe: z\n .string()\n .describe(\n \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n ),\n duration: z\n .number()\n .describe(\"The duration to hold the element in milliseconds\"),\n coordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to click on\"),\n }),\n execute: async ({ describe, coordinates, duration }) => {\n try {\n const page = await v3.context.awaitActivePage();\n const processed = processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: clickAndHold`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({\n describe,\n duration,\n }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n\n // Use dragAndDrop from same point to same point with delay to simulate click and hold\n const [xpath] = await page.dragAndDrop(\n processed.x,\n processed.y,\n processed.x,\n processed.y,\n { delay: duration, returnXpath: shouldCollectXpath },\n );\n\n // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n const action: Action = {\n selector: normalizedXpath,\n description: describe,\n method: \"clickAndHold\",\n arguments: [String(duration)],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return { success: true, describe };\n } catch (error) {\n return {\n success: false,\n error: `Error clicking and holding: ${error.message}`,\n };\n }\n },\n });\n"]}

package/dist/esm/lib/v3/agent/tools/dragAndDrop.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"dragAndDrop.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/dragAndDrop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC3D,IAAI,CAAC;IACH,WAAW,EACT,kNAAkN;IACpN,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACtE,gBAAgB,EAAE,CAAC;aAChB,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,wDAAwD,CAAC;QACrE,cAAc,EAAE,CAAC;aACd,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oDAAoD,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,gBAAgB,EAChB,cAAc,GACf,EAAkC,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,kBAAkB,CACvC,gBAAgB,CAAC,CAAC,CAAC,EACnB,gBAAgB,CAAC,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;YACF,MAAM,YAAY,GAAG,kBAAkB,CACrC,cAAc,CAAC,CAAC,CAAC,EACjB,cAAc,CAAC,CAAC,CAAC,EACjB,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,iCAAiC;gBAC1C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACjD,cAAc,CAAC,CAAC,EAChB,cAAc,CAAC,CAAC,EAChB,YAAY,CAAC,CAAC,EACd,YAAY,CAAC,CAAC,EACd,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,cAAc,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;gBAC1C,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,cAAc;wBACxB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,aAAa;wBACrB,SAAS,EAAE,CAAC,YAAY,CAAC;qBAC1B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,~~mBAAoB~~,~~KAAe~~,CAAC,OAAO,EAAE;~~aACrD~~,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBAC1B,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n DragAndDropToolResult,\n ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const dragAndDropTool = (v3: V3, provider?: string) =>\n tool({\n description:\n \"Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n inputSchema: z.object({\n describe: z.string().describe(\"Describe the element to drag and drop\"),\n startCoordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to start the drag and drop from\"),\n endCoordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to end the drag and drop at\"),\n }),\n execute: async ({\n describe,\n startCoordinates,\n endCoordinates,\n }): Promise<DragAndDropToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const processedStart = processCoordinates(\n startCoordinates[0],\n startCoordinates[1],\n provider,\n v3,\n );\n const processedEnd = processCoordinates(\n endCoordinates[0],\n endCoordinates[1],\n provider,\n v3,\n );\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: dragAndDrop`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({\n describe,\n }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const [fromXpath, toXpath] = await page.dragAndDrop(\n processedStart.x,\n processedStart.y,\n processedEnd.x,\n processedEnd.y,\n { returnXpath: shouldCollectXpath },\n );\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedFrom = ensureXPath(fromXpath);\n const normalizedTo = ensureXPath(toXpath);\n if (normalizedFrom && normalizedTo) {\n const action: Action = {\n selector: normalizedFrom,\n description: describe,\n method: \"dragAndDrop\",\n arguments: [normalizedTo],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return {\n success: true,\n describe,\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error dragging: ${(error ~~as Error)~~.message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n describe: result.describe,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n"]}
1	+ {"version":3,"file":"dragAndDrop.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/dragAndDrop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC3D,IAAI,CAAC;IACH,WAAW,EACT,kNAAkN;IACpN,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACtE,gBAAgB,EAAE,CAAC;aAChB,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,wDAAwD,CAAC;QACrE,cAAc,EAAE,CAAC;aACd,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oDAAoD,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,gBAAgB,EAChB,cAAc,GACf,EAAkC,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,kBAAkB,CACvC,gBAAgB,CAAC,CAAC,CAAC,EACnB,gBAAgB,CAAC,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;YACF,MAAM,YAAY,GAAG,kBAAkB,CACrC,cAAc,CAAC,CAAC,CAAC,EACjB,cAAc,CAAC,CAAC,CAAC,EACjB,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,iCAAiC;gBAC1C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACjD,cAAc,CAAC,CAAC,EAChB,cAAc,CAAC,CAAC,EAChB,YAAY,CAAC,CAAC,EACd,YAAY,CAAC,CAAC,EACd,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,cAAc,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;gBAC1C,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,cAAc;wBACxB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,aAAa;wBACrB,SAAS,EAAE,CAAC,YAAY,CAAC;qBAC1B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAmB,KAAK,CAAC,OAAO,EAAE;aAC1C,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBAC1B,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n DragAndDropToolResult,\n ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const dragAndDropTool = (v3: V3, provider?: string) =>\n tool({\n description:\n \"Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n inputSchema: z.object({\n describe: z.string().describe(\"Describe the element to drag and drop\"),\n startCoordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to start the drag and drop from\"),\n endCoordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to end the drag and drop at\"),\n }),\n execute: async ({\n describe,\n startCoordinates,\n endCoordinates,\n }): Promise<DragAndDropToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const processedStart = processCoordinates(\n startCoordinates[0],\n startCoordinates[1],\n provider,\n v3,\n );\n const processedEnd = processCoordinates(\n endCoordinates[0],\n endCoordinates[1],\n provider,\n v3,\n );\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: dragAndDrop`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({\n describe,\n }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const [fromXpath, toXpath] = await page.dragAndDrop(\n processedStart.x,\n processedStart.y,\n processedEnd.x,\n processedEnd.y,\n { returnXpath: shouldCollectXpath },\n );\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedFrom = ensureXPath(fromXpath);\n const normalizedTo = ensureXPath(toXpath);\n if (normalizedFrom && normalizedTo) {\n const action: Action = {\n selector: normalizedFrom,\n description: describe,\n method: \"dragAndDrop\",\n arguments: [normalizedTo],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return {\n success: true,\n describe,\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error dragging: ${error.message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n describe: result.describe,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n"]}

package/dist/esm/lib/v3/agent/tools/extract.js CHANGED Viewed

@@ -88,16 +88,7 @@ export const extractTool = (v3, executionModel, toolTimeout) => tool({
         }
         catch (error) {
             if (error instanceof TimeoutError) {
-                const timeoutMessage = `TimeoutError: extract() timed out — try using a smaller or simpler schema`;
-                v3.logger({
-                    category: "agent",
-                    message: timeoutMessage,
-                    level: 0,
-                });
-                return {
-                    success: false,
-                    error: timeoutMessage,
-                };
+                throw error;
             }
             return { success: false, error: error?.message ?? String(error) };
         }

package/dist/esm/lib/v3/agent/tools/extract.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAc,MAAM,KAAK,CAAC;AAGpC,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAU/D,SAAS,eAAe,CAAC,MAAkB;IACzC,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,KAAK,GAA+B,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC7D,KAAK,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;YACD,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,OAAO;YACV,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACzE,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;YACnB,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK;gBAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO;gBAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;gBAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACvC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAA6B,CAAC,CAAC;YACtD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;QACrB,KAAK,MAAM;YACT,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAClB;YACE,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,EAAM,EACN,cAA0C,EAC1C,WAAoB,EACpB,EAAE,CACF,IAAI,CAAC;IACH,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;;;0FAsByE;IACtF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;QACvB,MAAM,EAAE,CAAC;aACN,MAAM,CAAC;YACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC3B,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;YACxD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC7B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YACpC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;SACpD,CAAC;aACD,WAAW,EAAE;aACb,QAAQ,EAAE;aACV,QAAQ,CAAC,wDAAwD,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;QACzC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM;gBACzB,CAAC,CAAC,eAAe,CAAC,MAAoB,CAAC;gBACvC,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,YAAY,EAAE;gBACzD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,OAAO,EAAE,WAAW;aACrB,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBAClC,MAAM,~~cAAc,GAAG,2EAA2E,CAAC;gBACnG,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,cAAc;oBACvB,~~KAAK,~~EAAE,~~CAAC;~~iBACT~~,CAAC~~,CAAC~~;~~gBACH,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,cAAc;iBACtB,CAAC;YACJ,CAAC;~~YACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACpE,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z, ZodTypeAny } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { AgentModelConfig } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\ninterface JsonSchema {\n type?: string;\n properties?: Record<string, JsonSchema>;\n items?: JsonSchema;\n enum?: string[];\n format?: \"url\" \| \"email\" \| \"uuid\";\n}\n\nfunction jsonSchemaToZod(schema: JsonSchema): ZodTypeAny {\n switch (schema.type) {\n case \"object\": {\n const shape: Record<string, ZodTypeAny> = {};\n if (schema.properties) {\n for (const [key, value] of Object.entries(schema.properties)) {\n shape[key] = jsonSchemaToZod(value);\n }\n }\n return z.object(shape);\n }\n case \"array\":\n return z.array(schema.items ? jsonSchemaToZod(schema.items) : z.any());\n case \"string\": {\n let s = z.string();\n if (schema.format === \"url\") s = s.url();\n if (schema.format === \"email\") s = s.email();\n if (schema.format === \"uuid\") s = s.uuid();\n if (schema.enum && schema.enum.length > 0)\n return z.enum(schema.enum as [string, ...string[]]);\n return s;\n }\n case \"number\":\n case \"integer\":\n return z.number();\n case \"boolean\":\n return z.boolean();\n case \"null\":\n return z.null();\n default:\n return z.any();\n }\n}\n\nexport const extractTool = (\n v3: V3,\n executionModel?: string \| AgentModelConfig,\n toolTimeout?: number,\n) =>\n tool({\n description: `Extract structured data from the current page based on a provided schema.\n \n USAGE GUIDELINES:\n - Keep schemas MINIMAL - only include fields essential for the task\n - IMPORTANT: only use this if explicitly asked for structured output. In most scenarios, you should use the aria tree tool over this.\n - For URL fields, use format: \"url\"\n \n EXAMPLES:\n 1. Extract a single value:\n instruction: \"extract the product price\"\n schema: { type: \"object\", properties: { price: { type: \"number\" } } }\n \n 2. Extract multiple fields:\n instruction: \"extract product name and price\"\n schema: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } }\n \n 3. Extract arrays:\n instruction: \"extract all product names and prices\"\n schema: { type: \"object\", properties: { products: { type: \"array\", items: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } } } } }\n \n 4. Extract a URL:\n instruction: \"extract the link\"\n schema: { type: \"object\", properties: { url: { type: \"string\", format: \"url\" } } }`,\n inputSchema: z.object({\n instruction: z.string(),\n schema: z\n .object({\n type: z.string().optional(),\n properties: z.record(z.string(), z.unknown()).optional(),\n items: z.unknown().optional(),\n enum: z.array(z.string()).optional(),\n format: z.enum([\"url\", \"email\", \"uuid\"]).optional(),\n })\n .passthrough()\n .optional()\n .describe(\"JSON Schema object describing the structure to extract\"),\n }),\n execute: async ({ instruction, schema }) => {\n try {\n const parsedSchema = schema\n ? jsonSchemaToZod(schema as JsonSchema)\n : undefined;\n const result = await v3.extract(instruction, parsedSchema, {\n ...(executionModel ? { model: executionModel } : {}),\n timeout: toolTimeout,\n });\n return { success: true, result };\n } catch (error) {\n if (error instanceof TimeoutError) {\n ~~const~~ timeoutMessage = `TimeoutError: extract() timed out — try using a smaller or simpler schema`;\n v3.logger({\n category: \"agent\",\n message: timeoutMessage,\n level: 0,\n });\n return {\n success: false,\n error~~: timeoutMessage,\n }~~;\n }\n return { success: false, error: error?.message ?? String(error) };\n }\n },\n });\n"]}
1	+ {"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAc,MAAM,KAAK,CAAC;AAGpC,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAU/D,SAAS,eAAe,CAAC,MAAkB;IACzC,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,KAAK,GAA+B,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC7D,KAAK,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;YACD,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,OAAO;YACV,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACzE,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;YACnB,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK;gBAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO;gBAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;gBAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACvC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAA6B,CAAC,CAAC;YACtD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;QACrB,KAAK,MAAM;YACT,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAClB;YACE,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,EAAM,EACN,cAA0C,EAC1C,WAAoB,EACpB,EAAE,CACF,IAAI,CAAC;IACH,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;;;0FAsByE;IACtF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;QACvB,MAAM,EAAE,CAAC;aACN,MAAM,CAAC;YACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC3B,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;YACxD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC7B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YACpC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;SACpD,CAAC;aACD,WAAW,EAAE;aACb,QAAQ,EAAE;aACV,QAAQ,CAAC,wDAAwD,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;QACzC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM;gBACzB,CAAC,CAAC,eAAe,CAAC,MAAoB,CAAC;gBACvC,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,YAAY,EAAE;gBACzD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,OAAO,EAAE,WAAW;aACrB,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBAClC,MAAM,KAAK,CAAC;YACd,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACpE,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z, ZodTypeAny } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { AgentModelConfig } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\ninterface JsonSchema {\n type?: string;\n properties?: Record<string, JsonSchema>;\n items?: JsonSchema;\n enum?: string[];\n format?: \"url\" \| \"email\" \| \"uuid\";\n}\n\nfunction jsonSchemaToZod(schema: JsonSchema): ZodTypeAny {\n switch (schema.type) {\n case \"object\": {\n const shape: Record<string, ZodTypeAny> = {};\n if (schema.properties) {\n for (const [key, value] of Object.entries(schema.properties)) {\n shape[key] = jsonSchemaToZod(value);\n }\n }\n return z.object(shape);\n }\n case \"array\":\n return z.array(schema.items ? jsonSchemaToZod(schema.items) : z.any());\n case \"string\": {\n let s = z.string();\n if (schema.format === \"url\") s = s.url();\n if (schema.format === \"email\") s = s.email();\n if (schema.format === \"uuid\") s = s.uuid();\n if (schema.enum && schema.enum.length > 0)\n return z.enum(schema.enum as [string, ...string[]]);\n return s;\n }\n case \"number\":\n case \"integer\":\n return z.number();\n case \"boolean\":\n return z.boolean();\n case \"null\":\n return z.null();\n default:\n return z.any();\n }\n}\n\nexport const extractTool = (\n v3: V3,\n executionModel?: string \| AgentModelConfig,\n toolTimeout?: number,\n) =>\n tool({\n description: `Extract structured data from the current page based on a provided schema.\n \n USAGE GUIDELINES:\n - Keep schemas MINIMAL - only include fields essential for the task\n - IMPORTANT: only use this if explicitly asked for structured output. In most scenarios, you should use the aria tree tool over this.\n - For URL fields, use format: \"url\"\n \n EXAMPLES:\n 1. Extract a single value:\n instruction: \"extract the product price\"\n schema: { type: \"object\", properties: { price: { type: \"number\" } } }\n \n 2. Extract multiple fields:\n instruction: \"extract product name and price\"\n schema: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } }\n \n 3. Extract arrays:\n instruction: \"extract all product names and prices\"\n schema: { type: \"object\", properties: { products: { type: \"array\", items: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } } } } }\n \n 4. Extract a URL:\n instruction: \"extract the link\"\n schema: { type: \"object\", properties: { url: { type: \"string\", format: \"url\" } } }`,\n inputSchema: z.object({\n instruction: z.string(),\n schema: z\n .object({\n type: z.string().optional(),\n properties: z.record(z.string(), z.unknown()).optional(),\n items: z.unknown().optional(),\n enum: z.array(z.string()).optional(),\n format: z.enum([\"url\", \"email\", \"uuid\"]).optional(),\n })\n .passthrough()\n .optional()\n .describe(\"JSON Schema object describing the structure to extract\"),\n }),\n execute: async ({ instruction, schema }) => {\n try {\n const parsedSchema = schema\n ? jsonSchemaToZod(schema as JsonSchema)\n : undefined;\n const result = await v3.extract(instruction, parsedSchema, {\n ...(executionModel ? { model: executionModel } : {}),\n timeout: toolTimeout,\n });\n return { success: true, result };\n } catch (error) {\n if (error instanceof TimeoutError) {\n throw error;\n }\n return { success: false, error: error?.message ?? String(error) };\n }\n },\n });\n"]}

package/dist/esm/lib/v3/agent/tools/fillFormVision.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAI,CAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,CAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAEhD,8DAA8D;gBAC9D,0FAA0F;gBAC1F,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBAC3C,MAAM,SAAS,GAAG,kBAAkB,CAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;oBACF,OAAO;wBACL,GAAG,KAAK;wBACR,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,wCAAwC;wBACpE,KAAK,EAAE,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC;wBAClD,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;oBACpC,iEAAiE;oBACjE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAC5B,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB;wBACE,WAAW,EAAE,kBAAkB;qBAChC,CACF,CAAC;oBACF,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAE7B,uEAAuE;oBACvE,qFAAqF;oBACrF,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC;6BACjC,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,eAAe;oBACpC,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAwB,KAAe,CAAC,OAAO,EAAE;iBACzD,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gCACvB,KAAK,EAAE,MAAM,CAAC,KAAK;6BACpB,CAAC;yBACH;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n FillFormVisionToolResult,\n ModelOutputContentItem,\n Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n v3: V3,\n provider?: string,\n variables?: Variables,\n) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const valueDescription = hasVariables\n ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n : \"Text to type into the target field\";\n\n return tool({\n description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n inputSchema: z.object({\n fields: z\n .array(\n z.object({\n action: z\n .string()\n .describe(\n \"Description of the typing action, e.g. 'type foo into the bar field'\",\n ),\n value: z.string().describe(valueDescription),\n coordinates: z\n .object({\n x: z.number(),\n y: z.number(),\n })\n .describe(\"Coordinates of the target field\"),\n }),\n )\n .min(2, \"Provide at least two fields to fill\"),\n }),\n execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n\n // Process coordinates and substitute variables for each field\n // Keep original values (with %tokens%) for logging/caching, substituted values for typing\n const processedFields = fields.map((field) => {\n const processed = processCoordinates(\n field.coordinates.x,\n field.coordinates.y,\n provider,\n v3,\n );\n return {\n ...field,\n originalValue: field.value, // Keep original with %tokens% for cache\n value: substituteVariables(field.value, variables),\n coordinates: { x: processed.x, y: processed.y },\n };\n });\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: fillFormVision`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ fields }), // Don't log substituted values\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const actions: Action[] = [];\n\n for (const field of processedFields) {\n // Click the field, only requesting XPath when caching is enabled\n const xpath = await page.click(\n field.coordinates.x,\n field.coordinates.y,\n {\n returnXpath: shouldCollectXpath,\n },\n );\n await page.type(field.value);\n\n // Build Action with XPath for deterministic replay (only when caching)\n // Use originalValue (with %tokens%) so cache stores references, not sensitive values\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n actions.push({\n selector: normalizedXpath,\n description: field.action,\n method: \"type\",\n arguments: [field.originalValue],\n });\n }\n }\n\n // Small delay between fields\n await new Promise((resolve) => setTimeout(resolve, 100));\n }\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n if (shouldCollectXpath && actions.length > 0) {\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: `Fill ${fields.length} form fields`,\n actions,\n actionDescription: `Fill ${fields.length} form fields`,\n });\n }\n\n return {\n success: true,\n playwrightArguments: processedFields,\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error filling form: ${(error as Error).message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n error: result.error,\n }),\n },\n ],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n fieldsCount: result.playwrightArguments?.length ?? 0,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n};\n"]}
1	+ {"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAI,CAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,CAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAEhD,8DAA8D;gBAC9D,0FAA0F;gBAC1F,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBAC3C,MAAM,SAAS,GAAG,kBAAkB,CAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;oBACF,OAAO;wBACL,GAAG,KAAK;wBACR,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,wCAAwC;wBACpE,KAAK,EAAE,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC;wBAClD,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;oBACpC,iEAAiE;oBACjE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAC5B,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB;wBACE,WAAW,EAAE,kBAAkB;qBAChC,CACF,CAAC;oBACF,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAE7B,uEAAuE;oBACvE,qFAAqF;oBACrF,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC;6BACjC,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,eAAe;oBACpC,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAuB,KAAK,CAAC,OAAO,EAAE;iBAC9C,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gCACvB,KAAK,EAAE,MAAM,CAAC,KAAK;6BACpB,CAAC;yBACH;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n FillFormVisionToolResult,\n ModelOutputContentItem,\n Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n v3: V3,\n provider?: string,\n variables?: Variables,\n) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const valueDescription = hasVariables\n ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n : \"Text to type into the target field\";\n\n return tool({\n description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n inputSchema: z.object({\n fields: z\n .array(\n z.object({\n action: z\n .string()\n .describe(\n \"Description of the typing action, e.g. 'type foo into the bar field'\",\n ),\n value: z.string().describe(valueDescription),\n coordinates: z\n .object({\n x: z.number(),\n y: z.number(),\n })\n .describe(\"Coordinates of the target field\"),\n }),\n )\n .min(2, \"Provide at least two fields to fill\"),\n }),\n execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n\n // Process coordinates and substitute variables for each field\n // Keep original values (with %tokens%) for logging/caching, substituted values for typing\n const processedFields = fields.map((field) => {\n const processed = processCoordinates(\n field.coordinates.x,\n field.coordinates.y,\n provider,\n v3,\n );\n return {\n ...field,\n originalValue: field.value, // Keep original with %tokens% for cache\n value: substituteVariables(field.value, variables),\n coordinates: { x: processed.x, y: processed.y },\n };\n });\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: fillFormVision`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ fields }), // Don't log substituted values\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const actions: Action[] = [];\n\n for (const field of processedFields) {\n // Click the field, only requesting XPath when caching is enabled\n const xpath = await page.click(\n field.coordinates.x,\n field.coordinates.y,\n {\n returnXpath: shouldCollectXpath,\n },\n );\n await page.type(field.value);\n\n // Build Action with XPath for deterministic replay (only when caching)\n // Use originalValue (with %tokens%) so cache stores references, not sensitive values\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n actions.push({\n selector: normalizedXpath,\n description: field.action,\n method: \"type\",\n arguments: [field.originalValue],\n });\n }\n }\n\n // Small delay between fields\n await new Promise((resolve) => setTimeout(resolve, 100));\n }\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n if (shouldCollectXpath && actions.length > 0) {\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: `Fill ${fields.length} form fields`,\n actions,\n actionDescription: `Fill ${fields.length} form fields`,\n });\n }\n\n return {\n success: true,\n playwrightArguments: processedFields,\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error filling form: ${error.message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n error: result.error,\n }),\n },\n ],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n fieldsCount: result.playwrightArguments?.length ?? 0,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n};\n"]}

package/dist/esm/lib/v3/agent/tools/fillform.js CHANGED Viewed

@@ -64,16 +64,7 @@ export const fillFormTool = (v3, executionModel, variables, toolTimeout) => {
             }
             catch (error) {
                 if (error instanceof TimeoutError) {
-                    const timeoutMessage = `TimeoutError while waiting for fillForm() to complete (it may continue executing in the background)`;
-                    v3.logger({
-                        category: "agent",
-                        message: timeoutMessage,
-                        level: 0,
-                    });
-                    return {
-                        success: false,
-                        error: `${timeoutMessage} — try filling fewer fields at once or use a different tool`,
-                    };
+                    throw error;
                 }
                 return {
                     success: false,

package/dist/esm/lib/v3/agent/tools/fillform.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"fillform.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAE/D,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,EAAM,EACN,cAA0C,EAC1C,SAAqB,EACrB,WAAoB,EACpB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,+FAA+F,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACpI,CAAC,CAAC,8BAA8B,CAAC;IAEnC,OAAO,IAAI,CAAC;QACV,WAAW,EAAE,8GAA8G;QAC3H,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,oEAAoE,CACrE;gBACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;aAC7C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;SAChD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC5B,IAAI,CAAC;gBACH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,8BAA8B;oBACvC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;4BAC7B,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBACH,MAAM,WAAW,GAAG,yDAAyD,MAAM;qBAChF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;qBACpB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAEhB,MAAM,cAAc,GAAG,cAAc;oBACnC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE;oBACjD,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;gBAC7B,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;gBAErE,MAAM,SAAS,GAAG,EAAe,CAAC;gBAClC,MAAM,iBAAiB,GAAa,EAAE,CAAC;gBACvC,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;oBACjC,MAAM,UAAU,GAAG,SAAS;wBAC1B,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE;wBACrC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;oBAC7B,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;oBAChD,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;wBACrC,iBAAiB,CAAC,IAAI,CAAC,GAAI,SAAS,CAAC,OAAoB,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,UAAU;oBAChB,MAAM;oBACN,cAAc;oBACd,OAAO,EAAE,iBAAiB;iBAC3B,CAAC,CAAC;gBACH,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,SAAS;oBAClB,mBAAmB,EAAE,iBAAiB;iBACvC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;oBAClC,MAAM,~~cAAc,GAAG,qGAAqG,CAAC;oBAC7H,EAAE,CAAC,MAAM,CAAC;wBACR,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,cAAc;wBACvB,~~KAAK,~~EAAE,~~CAAC;~~qBACT~~,CAAC~~,CAAC~~;~~oBACH,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,KAAK,EAAE,GAAG,cAAc,6DAA6D;qBACtF,CAAC;gBACJ,CAAC;~~gBACD,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC;iBACvC,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type { AgentModelConfig, Variables } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\nexport const fillFormTool = (\n v3: V3,\n executionModel?: string \| AgentModelConfig,\n variables?: Variables,\n toolTimeout?: number,\n) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const valueDescription = hasVariables\n ? `Text to type into the target. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n : \"Text to type into the target\";\n\n return tool({\n description: `📝 FORM FILL - MULTI-FIELD INPUT TOOL\\nFor any form with 2+ inputs/textareas. Faster than individual typing.`,\n inputSchema: z.object({\n fields: z\n .array(\n z.object({\n action: z\n .string()\n .describe(\n 'Description of typing action, e.g. \"type foo into the email field\"',\n ),\n value: z.string().describe(valueDescription),\n }),\n )\n .min(1, \"Provide at least one field to fill\"),\n }),\n execute: async ({ fields }) => {\n try {\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: fillForm`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify(fields),\n type: \"object\",\n },\n },\n });\n const instruction = `Return observation results for the following actions: ${fields\n .map((f) => f.action)\n .join(\", \")}`;\n\n const observeOptions = executionModel\n ? { model: executionModel, timeout: toolTimeout }\n : { timeout: toolTimeout };\n const observeResults = await v3.observe(instruction, observeOptions);\n\n const completed = [] as unknown[];\n const replayableActions: Action[] = [];\n for (const res of observeResults) {\n const actOptions = variables\n ? { variables, timeout: toolTimeout }\n : { timeout: toolTimeout };\n const actResult = await v3.act(res, actOptions);\n completed.push(actResult);\n if (Array.isArray(actResult.actions)) {\n replayableActions.push(...(actResult.actions as Action[]));\n }\n }\n v3.recordAgentReplayStep({\n type: \"fillForm\",\n fields,\n observeResults,\n actions: replayableActions,\n });\n return {\n success: true,\n actions: completed,\n playwrightArguments: replayableActions,\n };\n } catch (error) {\n if (error instanceof TimeoutError) {\n ~~const~~ timeoutMessage = `TimeoutError while waiting for fillForm() to complete (it may continue executing in the background)`;\n v3.logger({\n category: \"agent\",\n message: timeoutMessage,\n level: 0,\n });\n return {\n success: false,\n error~~: `${timeoutMessage} — try filling fewer fields at once or use a different tool`,\n }~~;\n }\n return {\n success: false,\n error: error?.message ?? String(error),\n };\n }\n },\n });\n};\n"]}
1	+ {"version":3,"file":"fillform.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAE/D,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,EAAM,EACN,cAA0C,EAC1C,SAAqB,EACrB,WAAoB,EACpB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,+FAA+F,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACpI,CAAC,CAAC,8BAA8B,CAAC;IAEnC,OAAO,IAAI,CAAC;QACV,WAAW,EAAE,8GAA8G;QAC3H,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,oEAAoE,CACrE;gBACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;aAC7C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;SAChD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC5B,IAAI,CAAC;gBACH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,8BAA8B;oBACvC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;4BAC7B,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBACH,MAAM,WAAW,GAAG,yDAAyD,MAAM;qBAChF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;qBACpB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAEhB,MAAM,cAAc,GAAG,cAAc;oBACnC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE;oBACjD,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;gBAC7B,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;gBAErE,MAAM,SAAS,GAAG,EAAe,CAAC;gBAClC,MAAM,iBAAiB,GAAa,EAAE,CAAC;gBACvC,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;oBACjC,MAAM,UAAU,GAAG,SAAS;wBAC1B,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE;wBACrC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;oBAC7B,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;oBAChD,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;wBACrC,iBAAiB,CAAC,IAAI,CAAC,GAAI,SAAS,CAAC,OAAoB,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,UAAU;oBAChB,MAAM;oBACN,cAAc;oBACd,OAAO,EAAE,iBAAiB;iBAC3B,CAAC,CAAC;gBACH,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,SAAS;oBAClB,mBAAmB,EAAE,iBAAiB;iBACvC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;oBAClC,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC;iBACvC,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type { AgentModelConfig, Variables } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\nexport const fillFormTool = (\n v3: V3,\n executionModel?: string \| AgentModelConfig,\n variables?: Variables,\n toolTimeout?: number,\n) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const valueDescription = hasVariables\n ? `Text to type into the target. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n : \"Text to type into the target\";\n\n return tool({\n description: `📝 FORM FILL - MULTI-FIELD INPUT TOOL\\nFor any form with 2+ inputs/textareas. Faster than individual typing.`,\n inputSchema: z.object({\n fields: z\n .array(\n z.object({\n action: z\n .string()\n .describe(\n 'Description of typing action, e.g. \"type foo into the email field\"',\n ),\n value: z.string().describe(valueDescription),\n }),\n )\n .min(1, \"Provide at least one field to fill\"),\n }),\n execute: async ({ fields }) => {\n try {\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: fillForm`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify(fields),\n type: \"object\",\n },\n },\n });\n const instruction = `Return observation results for the following actions: ${fields\n .map((f) => f.action)\n .join(\", \")}`;\n\n const observeOptions = executionModel\n ? { model: executionModel, timeout: toolTimeout }\n : { timeout: toolTimeout };\n const observeResults = await v3.observe(instruction, observeOptions);\n\n const completed = [] as unknown[];\n const replayableActions: Action[] = [];\n for (const res of observeResults) {\n const actOptions = variables\n ? { variables, timeout: toolTimeout }\n : { timeout: toolTimeout };\n const actResult = await v3.act(res, actOptions);\n completed.push(actResult);\n if (Array.isArray(actResult.actions)) {\n replayableActions.push(...(actResult.actions as Action[]));\n }\n }\n v3.recordAgentReplayStep({\n type: \"fillForm\",\n fields,\n observeResults,\n actions: replayableActions,\n });\n return {\n success: true,\n actions: completed,\n playwrightArguments: replayableActions,\n };\n } catch (error) {\n if (error instanceof TimeoutError) {\n throw error;\n }\n return {\n success: false,\n error: error?.message ?? String(error),\n };\n }\n },\n });\n};\n"]}

package/dist/esm/lib/v3/agent/tools/index.d.ts CHANGED Viewed

@@ -44,8 +44,8 @@ export interface V3AgentToolOptions {
      */
     variables?: Variables;
     /**
-     * Timeout in milliseconds for tool calls that invoke v3 methods (act, extract, fillForm, ariaTree).
-     * Forwarded to the underlying v3 call's `timeout` option.
+     * Timeout in milliseconds for async tool calls.
+     * Applied to all tools that perform I/O (except wait and think).
      */
     toolTimeout?: number;
     /**

package/dist/esm/lib/v3/agent/tools/index.js CHANGED Viewed

@@ -16,6 +16,8 @@ import { fillFormVisionTool } from "./fillFormVision.js";
 import { thinkTool } from "./think.js";
 import { searchTool as browserbaseSearchTool } from "./browserbaseSearch.js";
 import { searchTool as braveSearchTool } from "./braveSearch.js";
+import { withTimeout } from "../../timeoutConfig.js";
+import { TimeoutError } from "../../types/public/sdkErrors.js";
 /**
  * Filters tools based on mode and explicit exclusions.
  * - 'dom' mode: Removes coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)
@@ -43,6 +45,40 @@ function filterTools(tools, mode, excludeTools) {
     }
     return filtered;
 }
+/**
+ * Wraps an AI SDK tool's execute function with a timeout guard.
+ * On timeout, returns `{ success: false, error: "TimeoutError: ..." }` to the LLM
+ * and logs the error. Also acts as a safety net for any uncaught errors.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function wrapToolWithTimeout(agentTool, toolName, v3, timeoutMs, timeoutHint) {
+    if (!timeoutMs || !agentTool.execute)
+        return agentTool;
+    const originalExecute = agentTool.execute;
+    return {
+        ...agentTool,
+        execute: async (...args) => {
+            try {
+                return await withTimeout(originalExecute(...args), timeoutMs, toolName);
+            }
+            catch (error) {
+                if (error instanceof TimeoutError) {
+                    const message = `TimeoutError: ${error.message}${timeoutHint ? ` ${timeoutHint}` : ""}`;
+                    v3.logger({
+                        category: "agent",
+                        message,
+                        level: 0,
+                    });
+                    return {
+                        success: false,
+                        error: message,
+                    };
+                }
+                throw error;
+            }
+        },
+    };
+}
 export function createAgentTools(v3, options) {
     const executionModel = options?.executionModel;
     const mode = options?.mode ?? "dom";
@@ -50,7 +86,13 @@ export function createAgentTools(v3, options) {
     const excludeTools = options?.excludeTools;
     const variables = options?.variables;
     const toolTimeout = options?.toolTimeout;
-    const allTools = {
+    const timeoutHints = {
+        act: "(it may continue executing in the background) — try using a different description for the action",
+        ariaTree: "— the page may be too large",
+        extract: "— try using a smaller or simpler schema",
+        fillForm: "(it may continue executing in the background) — try filling fewer fields at once or use a different tool",
+    };
+    const unwrappedTools = {
         act: actTool(v3, executionModel, variables, toolTimeout),
         ariaTree: ariaTreeTool(v3, toolTimeout),
         click: clickTool(v3, provider),
@@ -64,16 +106,22 @@ export function createAgentTools(v3, options) {
         navback: navBackTool(v3),
         screenshot: screenshotTool(v3),
         scroll: mode === "hybrid" ? scrollVisionTool(v3, provider) : scrollTool(v3),
-        think: thinkTool(),
         type: typeTool(v3, provider, variables),
-        wait: waitTool(v3, mode),
     };
     if (options?.useSearch && options.browserbaseApiKey) {
-        allTools.search = browserbaseSearchTool(v3, options.browserbaseApiKey);
+        unwrappedTools.search = browserbaseSearchTool(v3, options.browserbaseApiKey);
     }
     else if (process.env.BRAVE_API_KEY) {
-        allTools.search = braveSearchTool(v3);
+        unwrappedTools.search = braveSearchTool(v3);
     }
+    const allTools = {
+        ...Object.fromEntries(Object.entries(unwrappedTools).map(([name, t]) => [
+            name,
+            wrapToolWithTimeout(t, `${name}()`, v3, toolTimeout, timeoutHints[name]),
+        ])),
+        think: thinkTool(),
+        wait: waitTool(v3, mode),
+    };
     return filterTools(allTools, mode, excludeTools);
 }
 //# sourceMappingURL=index.js.map

package/dist/esm/lib/v3/agent/tools/index.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,UAAU,IAAI,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAC7E,OAAO,EAAE,UAAU,IAAI,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAmDjE;;;;;GAKG;AACH,SAAS,WAAW,CAClB,KAAc,EACd,IAAmB,EACnB,YAAuB;IAEvB,MAAM,QAAQ,GAAY,EAAE,GAAG,KAAK,EAAE,CAAC;IAEvC,uBAAuB;IACvB,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC,QAAQ,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,qBAAqB;QACrB,OAAO,QAAQ,CAAC,KAAK,CAAC;QACtB,OAAO,QAAQ,CAAC,IAAI,CAAC;QACrB,OAAO,QAAQ,CAAC,WAAW,CAAC;QAC5B,OAAO,QAAQ,CAAC,YAAY,CAAC;QAC7B,OAAO,QAAQ,CAAC,cAAc,CAAC;IACjC,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;YACpC,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,EAAM,EAAE,OAA4B;IACnE,MAAM,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAC/C,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,KAAK,CAAC;IACpC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IACnC,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;IAC3C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;IACrC,MAAM,WAAW,GAAG,OAAO,EAAE,WAAW,CAAC;IAEzC,MAAM,QAAQ,GAAY;QACxB,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,cAAc,EAAE,SAAS,EAAE,WAAW,CAAC;QACxD,QAAQ,EAAE,YAAY,CAAC,EAAE,EAAE,WAAW,CAAC;QACvC,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC9B,YAAY,EAAE,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC5C,WAAW,EAAE,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC1C,OAAO,EAAE,WAAW,CAAC,EAAE,EAAE,cAAc,EAAE,WAAW,CAAC;QACrD,QAAQ,EAAE,YAAY,CAAC,EAAE,EAAE,cAAc,EAAE,SAAS,EAAE,WAAW,CAAC;QAClE,cAAc,EAAE,kBAAkB,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC;QAC3D,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,OAAO,EAAE,WAAW,CAAC,EAAE,CAAC;QACxB,UAAU,EAAE,cAAc,CAAC,EAAE,CAAC;QAC9B,MAAM,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3E,KAAK,EAAE,SAAS,EAAE;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC;QACvC,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC;IAEF,IAAI,OAAO,EAAE,SAAS,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;QACpD,QAAQ,CAAC,MAAM,GAAG,qBAAqB,CAAC,EAAE,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACzE,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;QACrC,QAAQ,CAAC,MAAM,GAAG,eAAe,CAAC,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;AACnD,CAAC","sourcesContent":["import { gotoTool } from \"./goto.js\";\nimport { actTool } from \"./act.js\";\nimport { screenshotTool } from \"./screenshot.js\";\nimport { waitTool } from \"./wait.js\";\nimport { navBackTool } from \"./navback.js\";\nimport { ariaTreeTool } from \"./ariaTree.js\";\nimport { fillFormTool } from \"./fillform.js\";\nimport { scrollTool, scrollVisionTool } from \"./scroll.js\";\nimport { extractTool } from \"./extract.js\";\nimport { clickTool } from \"./click.js\";\nimport { typeTool } from \"./type.js\";\nimport { dragAndDropTool } from \"./dragAndDrop.js\";\nimport { clickAndHoldTool } from \"./clickAndHold.js\";\nimport { keysTool } from \"./keys.js\";\nimport { fillFormVisionTool } from \"./fillFormVision.js\";\nimport { thinkTool } from \"./think.js\";\nimport { searchTool as browserbaseSearchTool } from \"./browserbaseSearch.js\";\nimport { searchTool as braveSearchTool } from \"./braveSearch.js\";\n\nimport type { ToolSet, InferUITools } from \"ai\";\nimport type { V3 } from \"../../v3.js\";\nimport type { LogLine } from \"../../types/public/logs.js\";\nimport type {\n AgentToolMode,\n AgentModelConfig,\n Variables,\n} from \"../../types/public/agent.js\";\n\nexport interface V3AgentToolOptions {\n executionModel?: string \| AgentModelConfig;\n logger?: (message: LogLine) => void;\n /*\n Tool mode determines which set of tools are available.\n * - 'dom' (default): Uses DOM-based tools (act, fillForm) - removes coordinate-based tools\n * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, etc.) - removes fillForm\n /\n mode?: AgentToolMode;\n /\n The model provider. Used for model-specific coordinate handling\n /\n provider?: string;\n /\n Tools to exclude from the available toolset.\n * These tools will be filtered out after mode-based filtering.\n /\n excludeTools?: string[];\n /\n Variables available to the agent for use in act/type tools.\n * When provided, these tools will have an optional useVariable field.\n /\n variables?: Variables;\n /\n Timeout in milliseconds for tool calls that invoke v3 methods (act, extract, fillForm, ariaTree).\n * Forwarded to the underlying v3 call's `timeout` option.\n /\n toolTimeout?: number;\n /\n Whether to enable the Browserbase-powered web search tool.\n * Requires a valid Browserbase API key.\n /\n useSearch?: boolean;\n /\n The Browserbase API key used for the search tool.\n * Resolved from BROWSERBASE_API_KEY env var or the Stagehand constructor.\n /\n browserbaseApiKey?: string;\n}\n\n/\n Filters tools based on mode and explicit exclusions.\n * - 'dom' mode: Removes coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)\n * - 'hybrid' mode: Removes DOM-based form tool (fillForm) in favor of coordinate-based fillFormVision\n * - excludeTools: Additional tools to remove from the toolset\n /\nfunction filterTools(\n tools: ToolSet,\n mode: AgentToolMode,\n excludeTools?: string[],\n): ToolSet {\n const filtered: ToolSet = { ...tools };\n\n // Mode-based filtering\n if (mode === \"hybrid\") {\n delete filtered.fillForm;\n } else {\n // DOM mode (default)\n delete filtered.click;\n delete filtered.type;\n delete filtered.dragAndDrop;\n delete filtered.clickAndHold;\n delete filtered.fillFormVision;\n }\n\n if (excludeTools) {\n for (const toolName of excludeTools) {\n delete filtered[toolName];\n }\n }\n\n return filtered;\n}\n\nexport function createAgentTools(v3: V3, options?: V3AgentToolOptions) {\n const executionModel = options?.executionModel;\n const mode = options?.mode ?? \"dom\";\n const provider = options?.provider;\n const excludeTools = options?.excludeTools;\n const variables = options?.variables;\n const toolTimeout = options?.toolTimeout;\n\n const allTools: ToolSet = {\n act: actTool(v3, executionModel, variables, toolTimeout),\n ariaTree: ariaTreeTool(v3, toolTimeout),\n click: clickTool(v3, provider),\n clickAndHold: clickAndHoldTool(v3, provider),\n dragAndDrop: dragAndDropTool(v3, provider),\n extract: extractTool(v3, executionModel, toolTimeout),\n fillForm: fillFormTool(v3, executionModel, variables, toolTimeout),\n fillFormVision: fillFormVisionTool(v3, provider, variables),\n goto: gotoTool(v3),\n keys: keysTool(v3),\n navback: navBackTool(v3),\n screenshot: screenshotTool(v3),\n scroll: mode === \"hybrid\" ? scrollVisionTool(v3, provider) : scrollTool(v3),\n think: thinkTool(),\n type: typeTool(v3, provider, variables),\n wait: waitTool(v3, mode),\n };\n\n if (options?.useSearch && options.browserbaseApiKey) {\n allTools.search = browserbaseSearchTool(v3, options.browserbaseApiKey);\n } else if (process.env.BRAVE_API_KEY) {\n allTools.search = braveSearchTool(v3);\n }\n\n return filterTools(allTools, mode, excludeTools);\n}\n\nexport type AgentTools = ReturnType<typeof createAgentTools>;\n\n/\n Type map of all agent tools for strong typing of tool calls and results.\n * Note: `search` is optional — enabled via useSearch: true (Browserbase) or BRAVE_API_KEY env var (legacy).\n /\nexport type AgentToolTypesMap = {\n act: ReturnType<typeof actTool>;\n ariaTree: ReturnType<typeof ariaTreeTool>;\n click: ReturnType<typeof clickTool>;\n clickAndHold: ReturnType<typeof clickAndHoldTool>;\n dragAndDrop: ReturnType<typeof dragAndDropTool>;\n extract: ReturnType<typeof extractTool>;\n fillForm: ReturnType<typeof fillFormTool>;\n fillFormVision: ReturnType<typeof fillFormVisionTool>;\n goto: ReturnType<typeof gotoTool>;\n keys: ReturnType<typeof keysTool>;\n navback: ReturnType<typeof navBackTool>;\n screenshot: ReturnType<typeof screenshotTool>;\n scroll: ReturnType<typeof scrollTool> \| ReturnType<typeof scrollVisionTool>;\n search?:\n \| ReturnType<typeof browserbaseSearchTool>\n \| ReturnType<typeof braveSearchTool>;\n think: ReturnType<typeof thinkTool>;\n type: ReturnType<typeof typeTool>;\n wait: ReturnType<typeof waitTool>;\n};\n\n/\n Inferred UI tools type for type-safe tool inputs and outputs.\n * Use with UIMessage for full type safety in UI contexts.\n /\nexport type AgentUITools = InferUITools<AgentToolTypesMap>;\n\n/\n Union type for all possible agent tool calls.\n * Provides type-safe access to tool call arguments.\n /\nexport type AgentToolCall = {\n [K in keyof AgentToolTypesMap]: {\n toolName: K;\n toolCallId: string;\n args: AgentUITools[K][\"input\"];\n };\n}[keyof AgentToolTypesMap];\n\n/\n Union type for all possible agent tool results.\n * Provides type-safe access to tool result values.\n */\nexport type AgentToolResult = {\n [K in keyof AgentToolTypesMap]: {\n toolName: K;\n toolCallId: string;\n result: AgentUITools[K][\"output\"];\n };\n}[keyof AgentToolTypesMap];\n"]}
1	+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,UAAU,IAAI,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAC7E,OAAO,EAAE,UAAU,IAAI,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAUjE,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AA0C/D;;;;;GAKG;AACH,SAAS,WAAW,CAClB,KAAc,EACd,IAAmB,EACnB,YAAuB;IAEvB,MAAM,QAAQ,GAAY,EAAE,GAAG,KAAK,EAAE,CAAC;IAEvC,uBAAuB;IACvB,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC,QAAQ,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,qBAAqB;QACrB,OAAO,QAAQ,CAAC,KAAK,CAAC;QACtB,OAAO,QAAQ,CAAC,IAAI,CAAC;QACrB,OAAO,QAAQ,CAAC,WAAW,CAAC;QAC5B,OAAO,QAAQ,CAAC,YAAY,CAAC;QAC7B,OAAO,QAAQ,CAAC,cAAc,CAAC;IACjC,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;YACpC,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,8DAA8D;AAC9D,SAAS,mBAAmB,CAC1B,SAAY,EACZ,QAAgB,EAChB,EAAM,EACN,SAAkB,EAClB,WAAoB;IAEpB,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAEvD,MAAM,eAAe,GAAG,SAAS,CAAC,OAAO,CAAC;IAC1C,OAAO;QACL,GAAG,SAAS;QACZ,OAAO,EAAE,KAAK,EAAE,GAAG,IAAe,EAAE,EAAE;YACpC,IAAI,CAAC;gBACH,OAAO,MAAM,WAAW,CAAC,eAAe,CAAC,GAAG,IAAI,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;YAC1E,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;oBAClC,MAAM,OAAO,GAAG,iBAAiB,KAAK,CAAC,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;oBACxF,EAAE,CAAC,MAAM,CAAC;wBACR,QAAQ,EAAE,OAAO;wBACjB,OAAO;wBACP,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,KAAK,EAAE,OAAO;qBACf,CAAC;gBACJ,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;KACG,CAAC;AACT,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,EAAM,EAAE,OAA4B;IACnE,MAAM,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAC/C,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,KAAK,CAAC;IACpC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IACnC,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;IAC3C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;IACrC,MAAM,WAAW,GAAG,OAAO,EAAE,WAAW,CAAC;IAEzC,MAAM,YAAY,GAA2B;QAC3C,GAAG,EAAE,kGAAkG;QACvG,QAAQ,EAAE,6BAA6B;QACvC,OAAO,EAAE,yCAAyC;QAClD,QAAQ,EACN,0GAA0G;KAC7G,CAAC;IAEF,MAAM,cAAc,GAAY;QAC9B,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,cAAc,EAAE,SAAS,EAAE,WAAW,CAAC;QACxD,QAAQ,EAAE,YAAY,CAAC,EAAE,EAAE,WAAW,CAAC;QACvC,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC9B,YAAY,EAAE,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC5C,WAAW,EAAE,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC1C,OAAO,EAAE,WAAW,CAAC,EAAE,EAAE,cAAc,EAAE,WAAW,CAAC;QACrD,QAAQ,EAAE,YAAY,CAAC,EAAE,EAAE,cAAc,EAAE,SAAS,EAAE,WAAW,CAAC;QAClE,cAAc,EAAE,kBAAkB,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC;QAC3D,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,OAAO,EAAE,WAAW,CAAC,EAAE,CAAC;QACxB,UAAU,EAAE,cAAc,CAAC,EAAE,CAAC;QAC9B,MAAM,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3E,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC;KACxC,CAAC;IAEF,IAAI,OAAO,EAAE,SAAS,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;QACpD,cAAc,CAAC,MAAM,GAAG,qBAAqB,CAC3C,EAAE,EACF,OAAO,CAAC,iBAAiB,CAC1B,CAAC;IACJ,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;QACrC,cAAc,CAAC,MAAM,GAAG,eAAe,CAAC,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,QAAQ,GAAY;QACxB,GAAG,MAAM,CAAC,WAAW,CACnB,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;YAChD,IAAI;YACJ,mBAAmB,CACjB,CAAC,EACD,GAAG,IAAI,IAAI,EACX,EAAE,EACF,WAAW,EACX,YAAY,CAAC,IAAI,CAAC,CACnB;SACF,CAAC,CACH;QACD,KAAK,EAAE,SAAS,EAAE;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC;IAEF,OAAO,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;AACnD,CAAC","sourcesContent":["import { gotoTool } from \"./goto.js\";\nimport { actTool } from \"./act.js\";\nimport { screenshotTool } from \"./screenshot.js\";\nimport { waitTool } from \"./wait.js\";\nimport { navBackTool } from \"./navback.js\";\nimport { ariaTreeTool } from \"./ariaTree.js\";\nimport { fillFormTool } from \"./fillform.js\";\nimport { scrollTool, scrollVisionTool } from \"./scroll.js\";\nimport { extractTool } from \"./extract.js\";\nimport { clickTool } from \"./click.js\";\nimport { typeTool } from \"./type.js\";\nimport { dragAndDropTool } from \"./dragAndDrop.js\";\nimport { clickAndHoldTool } from \"./clickAndHold.js\";\nimport { keysTool } from \"./keys.js\";\nimport { fillFormVisionTool } from \"./fillFormVision.js\";\nimport { thinkTool } from \"./think.js\";\nimport { searchTool as browserbaseSearchTool } from \"./browserbaseSearch.js\";\nimport { searchTool as braveSearchTool } from \"./braveSearch.js\";\n\nimport type { ToolSet, InferUITools } from \"ai\";\nimport type { V3 } from \"../../v3.js\";\nimport type { LogLine } from \"../../types/public/logs.js\";\nimport type {\n AgentToolMode,\n AgentModelConfig,\n Variables,\n} from \"../../types/public/agent.js\";\nimport { withTimeout } from \"../../timeoutConfig.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\nexport interface V3AgentToolOptions {\n executionModel?: string \| AgentModelConfig;\n logger?: (message: LogLine) => void;\n /*\n Tool mode determines which set of tools are available.\n * - 'dom' (default): Uses DOM-based tools (act, fillForm) - removes coordinate-based tools\n * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, etc.) - removes fillForm\n /\n mode?: AgentToolMode;\n /\n The model provider. Used for model-specific coordinate handling\n /\n provider?: string;\n /\n Tools to exclude from the available toolset.\n * These tools will be filtered out after mode-based filtering.\n /\n excludeTools?: string[];\n /\n Variables available to the agent for use in act/type tools.\n * When provided, these tools will have an optional useVariable field.\n /\n variables?: Variables;\n /\n Timeout in milliseconds for async tool calls.\n * Applied to all tools that perform I/O (except wait and think).\n /\n toolTimeout?: number;\n /\n Whether to enable the Browserbase-powered web search tool.\n * Requires a valid Browserbase API key.\n /\n useSearch?: boolean;\n /\n The Browserbase API key used for the search tool.\n * Resolved from BROWSERBASE_API_KEY env var or the Stagehand constructor.\n /\n browserbaseApiKey?: string;\n}\n\n/\n Filters tools based on mode and explicit exclusions.\n * - 'dom' mode: Removes coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)\n * - 'hybrid' mode: Removes DOM-based form tool (fillForm) in favor of coordinate-based fillFormVision\n * - excludeTools: Additional tools to remove from the toolset\n /\nfunction filterTools(\n tools: ToolSet,\n mode: AgentToolMode,\n excludeTools?: string[],\n): ToolSet {\n const filtered: ToolSet = { ...tools };\n\n // Mode-based filtering\n if (mode === \"hybrid\") {\n delete filtered.fillForm;\n } else {\n // DOM mode (default)\n delete filtered.click;\n delete filtered.type;\n delete filtered.dragAndDrop;\n delete filtered.clickAndHold;\n delete filtered.fillFormVision;\n }\n\n if (excludeTools) {\n for (const toolName of excludeTools) {\n delete filtered[toolName];\n }\n }\n\n return filtered;\n}\n\n/\n Wraps an AI SDK tool's execute function with a timeout guard.\n * On timeout, returns `{ success: false, error: \"TimeoutError: ...\" }` to the LLM\n * and logs the error. Also acts as a safety net for any uncaught errors.\n /\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nfunction wrapToolWithTimeout<T extends Record<string, any>>(\n agentTool: T,\n toolName: string,\n v3: V3,\n timeoutMs?: number,\n timeoutHint?: string,\n): T {\n if (!timeoutMs \|\| !agentTool.execute) return agentTool;\n\n const originalExecute = agentTool.execute;\n return {\n ...agentTool,\n execute: async (...args: unknown[]) => {\n try {\n return await withTimeout(originalExecute(...args), timeoutMs, toolName);\n } catch (error) {\n if (error instanceof TimeoutError) {\n const message = `TimeoutError: ${error.message}${timeoutHint ? ` ${timeoutHint}` : \"\"}`;\n v3.logger({\n category: \"agent\",\n message,\n level: 0,\n });\n return {\n success: false,\n error: message,\n };\n }\n throw error;\n }\n },\n } as T;\n}\n\nexport function createAgentTools(v3: V3, options?: V3AgentToolOptions) {\n const executionModel = options?.executionModel;\n const mode = options?.mode ?? \"dom\";\n const provider = options?.provider;\n const excludeTools = options?.excludeTools;\n const variables = options?.variables;\n const toolTimeout = options?.toolTimeout;\n\n const timeoutHints: Record<string, string> = {\n act: \"(it may continue executing in the background) — try using a different description for the action\",\n ariaTree: \"— the page may be too large\",\n extract: \"— try using a smaller or simpler schema\",\n fillForm:\n \"(it may continue executing in the background) — try filling fewer fields at once or use a different tool\",\n };\n\n const unwrappedTools: ToolSet = {\n act: actTool(v3, executionModel, variables, toolTimeout),\n ariaTree: ariaTreeTool(v3, toolTimeout),\n click: clickTool(v3, provider),\n clickAndHold: clickAndHoldTool(v3, provider),\n dragAndDrop: dragAndDropTool(v3, provider),\n extract: extractTool(v3, executionModel, toolTimeout),\n fillForm: fillFormTool(v3, executionModel, variables, toolTimeout),\n fillFormVision: fillFormVisionTool(v3, provider, variables),\n goto: gotoTool(v3),\n keys: keysTool(v3),\n navback: navBackTool(v3),\n screenshot: screenshotTool(v3),\n scroll: mode === \"hybrid\" ? scrollVisionTool(v3, provider) : scrollTool(v3),\n type: typeTool(v3, provider, variables),\n };\n\n if (options?.useSearch && options.browserbaseApiKey) {\n unwrappedTools.search = browserbaseSearchTool(\n v3,\n options.browserbaseApiKey,\n );\n } else if (process.env.BRAVE_API_KEY) {\n unwrappedTools.search = braveSearchTool(v3);\n }\n\n const allTools: ToolSet = {\n ...Object.fromEntries(\n Object.entries(unwrappedTools).map(([name, t]) => [\n name,\n wrapToolWithTimeout(\n t,\n `${name}()`,\n v3,\n toolTimeout,\n timeoutHints[name],\n ),\n ]),\n ),\n think: thinkTool(),\n wait: waitTool(v3, mode),\n };\n\n return filterTools(allTools, mode, excludeTools);\n}\n\nexport type AgentTools = ReturnType<typeof createAgentTools>;\n\n/\n Type map of all agent tools for strong typing of tool calls and results.\n * Note: `search` is optional — enabled via useSearch: true (Browserbase) or BRAVE_API_KEY env var (legacy).\n /\nexport type AgentToolTypesMap = {\n act: ReturnType<typeof actTool>;\n ariaTree: ReturnType<typeof ariaTreeTool>;\n click: ReturnType<typeof clickTool>;\n clickAndHold: ReturnType<typeof clickAndHoldTool>;\n dragAndDrop: ReturnType<typeof dragAndDropTool>;\n extract: ReturnType<typeof extractTool>;\n fillForm: ReturnType<typeof fillFormTool>;\n fillFormVision: ReturnType<typeof fillFormVisionTool>;\n goto: ReturnType<typeof gotoTool>;\n keys: ReturnType<typeof keysTool>;\n navback: ReturnType<typeof navBackTool>;\n screenshot: ReturnType<typeof screenshotTool>;\n scroll: ReturnType<typeof scrollTool> \| ReturnType<typeof scrollVisionTool>;\n search?:\n \| ReturnType<typeof browserbaseSearchTool>\n \| ReturnType<typeof braveSearchTool>;\n think: ReturnType<typeof thinkTool>;\n type: ReturnType<typeof typeTool>;\n wait: ReturnType<typeof waitTool>;\n};\n\n/\n Inferred UI tools type for type-safe tool inputs and outputs.\n * Use with UIMessage for full type safety in UI contexts.\n /\nexport type AgentUITools = InferUITools<AgentToolTypesMap>;\n\n/\n Union type for all possible agent tool calls.\n * Provides type-safe access to tool call arguments.\n /\nexport type AgentToolCall = {\n [K in keyof AgentToolTypesMap]: {\n toolName: K;\n toolCallId: string;\n args: AgentUITools[K][\"input\"];\n };\n}[keyof AgentToolTypesMap];\n\n/\n Union type for all possible agent tool results.\n * Provides type-safe access to tool result values.\n */\nexport type AgentToolResult = {\n [K in keyof AgentToolTypesMap]: {\n toolName: K;\n toolCallId: string;\n result: AgentUITools[K][\"output\"];\n };\n}[keyof AgentToolTypesMap];\n"]}

package/dist/esm/lib/v3/agent/tools/keys.d.ts CHANGED Viewed

@@ -17,7 +17,7 @@ export declare const keysTool: (v3: V3) => import("ai").Tool<{
     error?: undefined;
 } | {
     success: boolean;
-    error: string;
+    error: any;
     method?: undefined;
     value?: undefined;
     times?: undefined;

package/dist/esm/lib/v3/agent/tools/keys.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"keys.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/keys.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,EAAE,CACjC,IAAI,CAAC;IACH,WAAW,EAAE;;;;kIAIiH;IAC9H,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACjC,KAAK,EAAE,CAAC;aACL,MAAM,EAAE;aACR,QAAQ,CACP,iEAAiE,CAClE;QACH,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC9B,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;QAC3C,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,0BAA0B;gBACnC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;wBAChD,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;YAEvC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBACzC,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,MAAM;oBACZ,WAAW,EAAE,SAAS,KAAK,GAAG;oBAC9B,mBAAmB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE;iBACpD,CAAC,CAAC;gBACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;YACjD,CAAC;YAED,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;gBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,MAAM;oBACZ,WAAW,EAAE,SAAS,KAAK,EAAE;oBAC7B,mBAAmB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE;iBACpD,CAAC,CAAC;gBACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;YACjD,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,MAAM,EAAE,EAAE,CAAC;QACpE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,~~EAAG~~,~~KAAe~~,CAAC,OAAO,EAAE,CAAC;~~QAC7D~~,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const keysTool = (v3: V3) =>\n tool({\n description: `Send keyboard input to the page without targeting a specific element. Unlike the type tool which clicks then types into coordinates, this sends keystrokes directly to wherever focus currently is.\n\nUse method=\"type\" to enter text into the currently focused element. Preferred when: input is already focused, text needs to flow across multiple fields (e.g., verification codes)\n\nUse method=\"press\" for navigation keys (Enter, Tab, Escape, Backspace, arrows) and keyboard shortcuts (Cmd+A, Ctrl+C, Shift+Tab).`,\n inputSchema: z.object({\n method: z.enum([\"press\", \"type\"]),\n value: z\n .string()\n .describe(\n \"The text to type, or the key/combo to press (Enter, Tab, Cmd+A)\",\n ),\n repeat: z.number().optional(),\n }),\n execute: async ({ method, value, repeat }) => {\n try {\n const page = await v3.context.awaitActivePage();\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: keys`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ method, value, repeat }),\n type: \"object\",\n },\n },\n });\n\n const times = Math.max(1, repeat ?? 1);\n\n if (method === \"type\") {\n for (let i = 0; i < times; i++) {\n await page.type(value, { delay: 100 });\n }\n v3.recordAgentReplayStep({\n type: \"keys\",\n instruction: `type \"${value}\"`,\n playwrightArguments: { method, text: value, times },\n });\n return { success: true, method, value, times };\n }\n\n if (method === \"press\") {\n for (let i = 0; i < times; i++) {\n await page.keyPress(value, { delay: 100 });\n }\n v3.recordAgentReplayStep({\n type: \"keys\",\n instruction: `press ${value}`,\n playwrightArguments: { method, keys: value, times },\n });\n return { success: true, method, value, times };\n }\n\n return { success: false, error: `Unsupported method: ${method}` };\n } catch (error) {\n return { success: false, error: (error ~~as Error)~~.message };\n }\n },\n });\n"]}
1	+ {"version":3,"file":"keys.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/keys.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,EAAE,CACjC,IAAI,CAAC;IACH,WAAW,EAAE;;;;kIAIiH;IAC9H,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACjC,KAAK,EAAE,CAAC;aACL,MAAM,EAAE;aACR,QAAQ,CACP,iEAAiE,CAClE;QACH,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC9B,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;QAC3C,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,0BAA0B;gBACnC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;wBAChD,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;YAEvC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBACzC,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,MAAM;oBACZ,WAAW,EAAE,SAAS,KAAK,GAAG;oBAC9B,mBAAmB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE;iBACpD,CAAC,CAAC;gBACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;YACjD,CAAC;YAED,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;gBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,MAAM;oBACZ,WAAW,EAAE,SAAS,KAAK,EAAE;oBAC7B,mBAAmB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE;iBACpD,CAAC,CAAC;gBACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;YACjD,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,MAAM,EAAE,EAAE,CAAC;QACpE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;QAClD,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const keysTool = (v3: V3) =>\n tool({\n description: `Send keyboard input to the page without targeting a specific element. Unlike the type tool which clicks then types into coordinates, this sends keystrokes directly to wherever focus currently is.\n\nUse method=\"type\" to enter text into the currently focused element. Preferred when: input is already focused, text needs to flow across multiple fields (e.g., verification codes)\n\nUse method=\"press\" for navigation keys (Enter, Tab, Escape, Backspace, arrows) and keyboard shortcuts (Cmd+A, Ctrl+C, Shift+Tab).`,\n inputSchema: z.object({\n method: z.enum([\"press\", \"type\"]),\n value: z\n .string()\n .describe(\n \"The text to type, or the key/combo to press (Enter, Tab, Cmd+A)\",\n ),\n repeat: z.number().optional(),\n }),\n execute: async ({ method, value, repeat }) => {\n try {\n const page = await v3.context.awaitActivePage();\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: keys`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ method, value, repeat }),\n type: \"object\",\n },\n },\n });\n\n const times = Math.max(1, repeat ?? 1);\n\n if (method === \"type\") {\n for (let i = 0; i < times; i++) {\n await page.type(value, { delay: 100 });\n }\n v3.recordAgentReplayStep({\n type: \"keys\",\n instruction: `type \"${value}\"`,\n playwrightArguments: { method, text: value, times },\n });\n return { success: true, method, value, times };\n }\n\n if (method === \"press\") {\n for (let i = 0; i < times; i++) {\n await page.keyPress(value, { delay: 100 });\n }\n v3.recordAgentReplayStep({\n type: \"keys\",\n instruction: `press ${value}`,\n playwrightArguments: { method, keys: value, times },\n });\n return { success: true, method, value, times };\n }\n\n return { success: false, error: `Unsupported method: ${method}` };\n } catch (error) {\n return { success: false, error: error.message };\n }\n },\n });\n"]}

package/dist/esm/lib/v3/agent/tools/type.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"type.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,SAAqB,EAAE,EAAE;IAC3E,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,YAAY;QAClC,CAAC,CAAC,oGAAoG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACzI,CAAC,CAAC,mCAAmC,CAAC;IAExC,OAAO,IAAI,CAAC;QACV,WAAW,EACT,2QAA2Q;QAC7Q,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,QAAQ,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,CACP,4HAA4H,CAC7H;YACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;YAC1C,WAAW,EAAE,CAAC;iBACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,QAAQ,CAAC,iDAAiD,CAAC;SAC/D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,WAAW,EACX,IAAI,GACL,EAA2B,EAAE;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;gBAEF,mDAAmD;gBACnD,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAExD,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,0BAA0B;oBACnC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;4BACzC,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;oBACvD,WAAW,EAAE,kBAAkB;iBAChC,CAAC,CAAC;gBAEH,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE5B,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;gBAE9D,0FAA0F;gBAC1F,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;oBAC3C,IAAI,eAAe,EAAE,CAAC;wBACpB,MAAM,MAAM,GAAW;4BACrB,QAAQ,EAAE,eAAe;4BACzB,WAAW,EAAE,QAAQ;4BACrB,MAAM,EAAE,MAAM;4BACd,SAAS,EAAE,CAAC,IAAI,CAAC;yBAClB,CAAC;wBACF,EAAE,CAAC,qBAAqB,CAAC;4BACvB,IAAI,EAAE,KAAK;4BACX,WAAW,EAAE,QAAQ;4BACrB,OAAO,EAAE,CAAC,MAAM,CAAC;4BACjB,iBAAiB,EAAE,QAAQ;yBAC5B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,QAAQ;oBACR,IAAI,EAAE,8FAA8F;oBACpG,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,~~iBAAkB~~,~~KAAe~~,CAAC,OAAO,EAAE;~~iBACnD~~,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;iBACxD,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n TypeToolResult,\n ModelOutputContentItem,\n Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const typeTool = (v3: V3, provider?: string, variables?: Variables) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const textDescription = hasVariables\n ? `The text to type into the element. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n : \"The text to type into the element\";\n\n return tool({\n description:\n \"Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n inputSchema: z.object({\n describe: z\n .string()\n .describe(\n \"Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description\",\n ),\n text: z.string().describe(textDescription),\n coordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to type into the element\"),\n }),\n execute: async ({\n describe,\n coordinates,\n text,\n }): Promise<TypeToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const processed = processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n\n // Substitute any %variableName% tokens in the text\n const actualText = substituteVariables(text, variables);\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: type`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ describe, text }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const xpath = await page.click(processed.x, processed.y, {\n returnXpath: shouldCollectXpath,\n });\n\n await page.type(actualText);\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n const action: Action = {\n selector: normalizedXpath,\n description: describe,\n method: \"type\",\n arguments: [text],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return {\n success: true,\n describe,\n text, // Return original text (with %variableName% tokens) to avoid exposing sensitive values to LLM\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error typing: ${(error ~~as Error)~~.message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n describe: result.describe,\n text: result.text,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n};\n"]}
1	+ {"version":3,"file":"type.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,SAAqB,EAAE,EAAE;IAC3E,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,YAAY;QAClC,CAAC,CAAC,oGAAoG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACzI,CAAC,CAAC,mCAAmC,CAAC;IAExC,OAAO,IAAI,CAAC;QACV,WAAW,EACT,2QAA2Q;QAC7Q,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,QAAQ,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,CACP,4HAA4H,CAC7H;YACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;YAC1C,WAAW,EAAE,CAAC;iBACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,QAAQ,CAAC,iDAAiD,CAAC;SAC/D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,WAAW,EACX,IAAI,GACL,EAA2B,EAAE;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;gBAEF,mDAAmD;gBACnD,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAExD,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,0BAA0B;oBACnC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;4BACzC,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;oBACvD,WAAW,EAAE,kBAAkB;iBAChC,CAAC,CAAC;gBAEH,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE5B,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;gBAE9D,0FAA0F;gBAC1F,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;oBAC3C,IAAI,eAAe,EAAE,CAAC;wBACpB,MAAM,MAAM,GAAW;4BACrB,QAAQ,EAAE,eAAe;4BACzB,WAAW,EAAE,QAAQ;4BACrB,MAAM,EAAE,MAAM;4BACd,SAAS,EAAE,CAAC,IAAI,CAAC;yBAClB,CAAC;wBACF,EAAE,CAAC,qBAAqB,CAAC;4BACvB,IAAI,EAAE,KAAK;4BACX,WAAW,EAAE,QAAQ;4BACrB,OAAO,EAAE,CAAC,MAAM,CAAC;4BACjB,iBAAiB,EAAE,QAAQ;yBAC5B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,QAAQ;oBACR,IAAI,EAAE,8FAA8F;oBACpG,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,iBAAiB,KAAK,CAAC,OAAO,EAAE;iBACxC,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;iBACxD,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n TypeToolResult,\n ModelOutputContentItem,\n Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const typeTool = (v3: V3, provider?: string, variables?: Variables) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const textDescription = hasVariables\n ? `The text to type into the element. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n : \"The text to type into the element\";\n\n return tool({\n description:\n \"Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n inputSchema: z.object({\n describe: z\n .string()\n .describe(\n \"Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description\",\n ),\n text: z.string().describe(textDescription),\n coordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to type into the element\"),\n }),\n execute: async ({\n describe,\n coordinates,\n text,\n }): Promise<TypeToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const processed = processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n\n // Substitute any %variableName% tokens in the text\n const actualText = substituteVariables(text, variables);\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: type`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ describe, text }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const xpath = await page.click(processed.x, processed.y, {\n returnXpath: shouldCollectXpath,\n });\n\n await page.type(actualText);\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n const action: Action = {\n selector: normalizedXpath,\n description: describe,\n method: \"type\",\n arguments: [text],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return {\n success: true,\n describe,\n text, // Return original text (with %variableName% tokens) to avoid exposing sensitive values to LLM\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error typing: ${error.message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n describe: result.describe,\n text: result.text,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n};\n"]}

package/dist/esm/lib/v3/api.js CHANGED Viewed

@@ -82,6 +82,7 @@ export class StagehandAPIClient {
             method: "POST",
             body: JSON.stringify(requestBody),
         });
+        console.log(JSON.stringify(requestBody, null, 2));
         if (sessionResponse.status === 401) {
             throw new StagehandAPIUnauthorizedError("Unauthorized. Ensure you provided a valid API key.");
         }
@@ -232,6 +233,9 @@ export class StagehandAPIClient {
             frameId,
             shouldCache,
         };
+        console.log(JSON.stringify(requestBody, null, 2));
+        console.log(JSON.stringify(requestBody, null, 2));
+        console.log(JSON.stringify(requestBody, null, 2));
         const result = await this.execute({
             method: "agentExecute",
             args: requestBody,
@@ -568,13 +572,16 @@ export class StagehandAPIClient {
         else {
             baseUrl = getApiUrlForRegion(this.region);
         }
-        const response = await this.fetchWithCookies(`${baseUrl}${path}`, {
+        const fetchUrl = `${baseUrl}${path}`;
+        const fetchOptions = {
             ...options,
             headers: {
                 ...defaultHeaders,
                 ...options.headers,
             },
-        });
+        };
+        console.log("Fetch request:", JSON.stringify({ url: fetchUrl, ...fetchOptions }, null, 2));
+        const response = await this.fetchWithCookies(fetchUrl, fetchOptions);
         return response;
     }
 }