npm - @browserbasehq/orca - Versions diffs - 3.7.0-preview.1 → 3.7.2-gemini-cua - Mend

@browserbasehq/orca 3.7.0-preview.1 → 3.7.2-gemini-cua

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (165) hide show

package/dist/esm/lib/v3/agent/tools/fillFormVision.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAI,CAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,CAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAEhD,qEAAqE;gBACrE,qEAAqE;gBACrE,kEAAkE;gBAClE,mEAAmE;gBACnE,qBAAqB;gBACrB,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBACtC,MAAM,SAAS,GAAG,kBAAkB,CAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;oBACF,OAAO;wBACL,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;oBAC/B,iEAAiE;oBACjE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAC5B,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB;wBACE,WAAW,EAAE,kBAAkB;qBAChC,CACF,CAAC;oBACF,oEAAoE;oBACpE,iCAAiC;oBACjC,MAAM,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;oBAE7D,wEAAwE;oBACxE,sEAAsE;oBACtE,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;6BACzB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,UAAU;oBAC/B,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAuB,KAAK,CAAC,OAAO,EAAE;iBAC9C,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gCACvB,KAAK,EAAE,MAAM,CAAC,KAAK;6BACpB,CAAC;yBACH;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n FillFormVisionToolResult,\n ModelOutputContentItem,\n Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n v3: V3,\n provider?: string,\n variables?: Variables,\n) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const valueDescription = hasVariables\n ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n : \"Text to type into the target field\";\n\n return tool({\n description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n inputSchema: z.object({\n fields: z\n .array(\n z.object({\n action: z\n .string()\n .describe(\n \"Description of the typing action, e.g. 'type foo into the bar field'\",\n ),\n value: z.string().describe(valueDescription),\n coordinates: z\n .object({\n x: z.number(),\n y: z.number(),\n })\n .describe(\"Coordinates of the target field\"),\n }),\n )\n .min(2, \"Provide at least two fields to fill\"),\n }),\n execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n\n // Process coordinates per field. Keep the original `value` (with any\n // `%variableName%` tokens) so substituted secrets never leak through\n // the tool result, replay cache, returned AgentResult.actions, or\n // anything that logs the action. Variables are substituted only at\n // typing time below.\n const safeFields = fields.map((field) => {\n const processed = processCoordinates(\n field.coordinates.x,\n field.coordinates.y,\n provider,\n v3,\n );\n return {\n action: field.action,\n value: field.value,\n coordinates: { x: processed.x, y: processed.y },\n };\n });\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: fillFormVision`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ fields }), // Don't log substituted values\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const actions: Action[] = [];\n\n for (const field of safeFields) {\n // Click the field, only requesting XPath when caching is enabled\n const xpath = await page.click(\n field.coordinates.x,\n field.coordinates.y,\n {\n returnXpath: shouldCollectXpath,\n },\n );\n // Substitute variables only at the moment of typing so the resolved\n // value never leaves this scope.\n await page.type(substituteVariables(field.value, variables));\n\n // Build Action with XPath for deterministic replay (only when caching).\n // Store the placeholder value so cache entries don't contain secrets.\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n actions.push({\n selector: normalizedXpath,\n description: field.action,\n method: \"type\",\n arguments: [field.value],\n });\n }\n }\n\n // Small delay between fields\n await new Promise((resolve) => setTimeout(resolve, 100));\n }\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n if (shouldCollectXpath && actions.length > 0) {\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: `Fill ${fields.length} form fields`,\n actions,\n actionDescription: `Fill ${fields.length} form fields`,\n });\n }\n\n return {\n success: true,\n playwrightArguments: safeFields,\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error filling form: ${error.message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n error: result.error,\n }),\n },\n ],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n fieldsCount: result.playwrightArguments?.length ?? 0,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n};\n"]}
1	+ {"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EACL,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAI,CAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,CAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC;oBACzC,CAAC,CAAC,MAAM,EAAE,CAAC,eAAe,EAAE;oBAC5B,CAAC,CAAC,SAAS,CAAC;gBAEd,qEAAqE;gBACrE,qEAAqE;gBACrE,kEAAkE;gBAClE,mEAAmE;gBACnE,qBAAqB;gBACrB,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAClC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;oBACzB,MAAM,SAAS,GAAG,MAAM,kBAAkB,CACxC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,EACF,QAAQ,CACT,CAAC;oBACF,OAAO;wBACL,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CACH,CAAC;gBAEF,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;oBAC/B,uEAAuE;oBACvE,uEAAuE;oBACvE,yEAAyE;oBACzE,wEAAwE;oBACxE,qEAAqE;oBACrE,MAAM,KAAK,GAAG,EAAE,CAAC,SAAS;wBACxB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE;4BACvD,WAAW,EAAE,kBAAkB;yBAChC,CAAC;wBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE;4BACzD,WAAW,EAAE,kBAAkB;yBAChC,CAAC,CAAC;oBACP,oEAAoE;oBACpE,iCAAiC;oBACjC,MAAM,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;oBAE7D,wEAAwE;oBACxE,sEAAsE;oBACtE,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;6BACzB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,UAAU;oBAC/B,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAuB,KAAK,CAAC,OAAO,EAAE;iBAC9C,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gCACvB,KAAK,EAAE,MAAM,CAAC,KAAK;6BACpB,CAAC;yBACH;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n FillFormVisionToolResult,\n ModelOutputContentItem,\n Variables,\n} from \"../../types/public/agent.js\";\nimport {\n isGoogleProvider,\n processCoordinates,\n} from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n v3: V3,\n provider?: string,\n variables?: Variables,\n) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const valueDescription = hasVariables\n ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n : \"Text to type into the target field\";\n\n return tool({\n description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n inputSchema: z.object({\n fields: z\n .array(\n z.object({\n action: z\n .string()\n .describe(\n \"Description of the typing action, e.g. 'type foo into the bar field'\",\n ),\n value: z.string().describe(valueDescription),\n coordinates: z\n .object({\n x: z.number(),\n y: z.number(),\n })\n .describe(\"Coordinates of the target field\"),\n }),\n )\n .min(2, \"Provide at least two fields to fill\"),\n }),\n execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const viewport = isGoogleProvider(provider)\n ? await v3.resolveViewport()\n : undefined;\n\n // Process coordinates per field. Keep the original `value` (with any\n // `%variableName%` tokens) so substituted secrets never leak through\n // the tool result, replay cache, returned AgentResult.actions, or\n // anything that logs the action. Variables are substituted only at\n // typing time below.\n const safeFields = await Promise.all(\n fields.map(async (field) => {\n const processed = await processCoordinates(\n field.coordinates.x,\n field.coordinates.y,\n provider,\n v3,\n viewport,\n );\n return {\n action: field.action,\n value: field.value,\n coordinates: { x: processed.x, y: processed.y },\n };\n }),\n );\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: fillFormVision`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ fields }), // Don't log substituted values\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const actions: Action[] = [];\n\n for (const field of safeFields) {\n // Focus the field, only requesting XPath when caching is enabled. On a\n // touch session this must be a trusted tap — a synthesized mouse click\n // never registers on touch-gated mobile layouts, so the field would stay\n // unfocused and the value go nowhere. The recorded step stays a \"type\",\n // whose replay focuses via the selector rather than a pointer event.\n const xpath = v3.usesTouch\n ? await page.tap(field.coordinates.x, field.coordinates.y, {\n returnXpath: shouldCollectXpath,\n })\n : await page.click(field.coordinates.x, field.coordinates.y, {\n returnXpath: shouldCollectXpath,\n });\n // Substitute variables only at the moment of typing so the resolved\n // value never leaves this scope.\n await page.type(substituteVariables(field.value, variables));\n\n // Build Action with XPath for deterministic replay (only when caching).\n // Store the placeholder value so cache entries don't contain secrets.\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n actions.push({\n selector: normalizedXpath,\n description: field.action,\n method: \"type\",\n arguments: [field.value],\n });\n }\n }\n\n // Small delay between fields\n await new Promise((resolve) => setTimeout(resolve, 100));\n }\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n if (shouldCollectXpath && actions.length > 0) {\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: `Fill ${fields.length} form fields`,\n actions,\n actionDescription: `Fill ${fields.length} form fields`,\n });\n }\n\n return {\n success: true,\n playwrightArguments: safeFields,\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error filling form: ${error.message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n error: result.error,\n }),\n },\n ],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n fieldsCount: result.playwrightArguments?.length ?? 0,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n};\n"]}

package/dist/esm/lib/v3/agent/tools/scroll.js CHANGED Viewed

@@ -81,7 +81,7 @@ export const scrollVisionTool = (v3, provider) => tool({
         let cx;
         let cy;
         if (coordinates) {
-            const processed = processCoordinates(coordinates[0], coordinates[1], provider, v3);
+            const processed = await processCoordinates(coordinates[0], coordinates[1], provider, v3);
             cx = processed.x;
             cy = processed.y;
         }

package/dist/esm/lib/v3/agent/tools/scroll.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"scroll.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/scroll.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,EAAE,CACnC,IAAI,CAAC;IACH,WAAW,EACT,iJAAiJ;IACnJ,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,UAAU,GAAG,EAAE,GAChB,EAA6B,EAAE;QAC9B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;oBAChD,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,YAAY,UAAU,KAAK,SAAS,KAAK,cAAc,KAAK;YACrE,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;aACtC;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEL;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAI,CAAC;IACH,WAAW,EAAE,6SAA6S;IAC1T,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,EAAE;aACV,QAAQ,CACP,4GAA4G,CAC7G;QACH,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,WAAW,EACX,UAAU,GAAG,EAAE,GAChB,EAAmC,EAAE;QACpC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,iEAAiE;QACjE,IAAI,EAAU,CAAC;QACf,IAAI,EAAU,CAAC;QACf,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YACF,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;YACjB,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACvB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,SAAS;wBACT,WAAW;wBACX,UAAU;wBACV,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;qBACtB,CAAC;oBACF,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEnE,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,WAAW;gBAClB,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,QAAQ,EAAE,KAAK,EAAE,GAAG;gBAC1D,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,EAAE;YAC1C,cAAc,EAAE,cAAc;YAC9B,gBAAgB;SACjB,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;iBACtC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type {\n ScrollToolResult,\n ScrollVisionToolResult,\n ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\n/*\n Simple scroll tool for DOM mode (non-grounding models).\n * No coordinates - scrolls from viewport center.\n /\nexport const scrollTool = (v3: V3) =>\n tool({\n description:\n \"Scroll the page up or down by a percentage of the viewport height. Default is 80%, and what should be typically used for general page scrolling\",\n inputSchema: z.object({\n direction: z.enum([\"up\", \"down\"]),\n percentage: z.number().min(1).max(200).optional(),\n }),\n execute: async ({\n direction,\n percentage = 80,\n }): Promise<ScrollToolResult> => {\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: scroll`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ direction, percentage }),\n type: \"object\",\n },\n },\n });\n\n const page = await v3.context.awaitActivePage();\n\n const { w, h } = await page.mainFrame().evaluate<{\n w: number;\n h: number;\n }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n const scrollDistance = Math.round((h percentage) / 100);\n const cx = Math.floor(w / 2);\n const cy = Math.floor(h / 2);\n const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n await page.scroll(cx, cy, 0, deltaY);\n\n v3.recordAgentReplayStep({\n type: \"scroll\",\n deltaX: 0,\n deltaY,\n anchor: { x: cx, y: cy },\n });\n\n return {\n success: true,\n message: `Scrolled ${percentage}% ${direction} (${scrollDistance}px)`,\n scrolledPixels: scrollDistance,\n };\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n return {\n type: \"json\",\n value: {\n success: result.success,\n message: result.message,\n scrolledPixels: result.scrolledPixels,\n },\n };\n },\n });\n\n/*\n Scroll tool for hybrid mode (grounding models).\n * Supports optional coordinates for scrolling within nested scrollable elements.\n /\nexport const scrollVisionTool = (v3: V3, provider?: string) =>\n tool({\n description: `Scroll the page up or down. For general page scrolling, no coordinates needed. Only provide coordinates when scrolling inside a nested scrollable element (e.g., a dropdown menu, modal with overflow, or scrollable sidebar). Default is 80%, and what should be typically used for general page scrolling`,\n inputSchema: z.object({\n direction: z.enum([\"up\", \"down\"]),\n coordinates: z\n .array(z.number())\n .optional()\n .describe(\n \"Only use coordinates for scrolling inside a nested scrollable element - provide (x, y) within that element\",\n ),\n percentage: z.number().min(1).max(200).optional(),\n }),\n execute: async ({\n direction,\n coordinates,\n percentage = 80,\n }): Promise<ScrollVisionToolResult> => {\n const page = await v3.context.awaitActivePage();\n\n const { w, h } = await page.mainFrame().evaluate<{\n w: number;\n h: number;\n }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n // Process coordinates if provided, otherwise use viewport center\n let cx: number;\n let cy: number;\n if (coordinates) {\n const processed = processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n cx = processed.x;\n cy = processed.y;\n } else {\n cx = Math.floor(w / 2);\n cy = Math.floor(h / 2);\n }\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: scroll`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({\n direction,\n coordinates,\n percentage,\n processed: { cx, cy },\n }),\n type: \"object\",\n },\n },\n });\n\n const scrollDistance = Math.round((h percentage) / 100);\n const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n await page.scroll(cx, cy, 0, deltaY);\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n v3.recordAgentReplayStep({\n type: \"scroll\",\n deltaX: 0,\n deltaY,\n anchor: { x: cx, y: cy },\n });\n\n return {\n success: true,\n message: coordinates\n ? `Scrolled ${percentage}% ${direction} at (${cx}, ${cy})`\n : `Scrolled ${percentage}% ${direction}`,\n scrolledPixels: scrollDistance,\n screenshotBase64,\n };\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n message: result.message,\n scrolledPixels: result.scrolledPixels,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n"]}
1	+ {"version":3,"file":"scroll.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/scroll.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,EAAE,CACnC,IAAI,CAAC;IACH,WAAW,EACT,iJAAiJ;IACnJ,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,UAAU,GAAG,EAAE,GAChB,EAA6B,EAAE;QAC9B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;oBAChD,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,YAAY,UAAU,KAAK,SAAS,KAAK,cAAc,KAAK;YACrE,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;aACtC;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEL;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAI,CAAC;IACH,WAAW,EAAE,6SAA6S;IAC1T,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,EAAE;aACV,QAAQ,CACP,4GAA4G,CAC7G;QACH,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,WAAW,EACX,UAAU,GAAG,EAAE,GAChB,EAAmC,EAAE;QACpC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,iEAAiE;QACjE,IAAI,EAAU,CAAC;QACf,IAAI,EAAU,CAAC;QACf,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,SAAS,GAAG,MAAM,kBAAkB,CACxC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YACF,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;YACjB,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACvB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,SAAS;wBACT,WAAW;wBACX,UAAU;wBACV,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;qBACtB,CAAC;oBACF,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEnE,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,WAAW;gBAClB,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,QAAQ,EAAE,KAAK,EAAE,GAAG;gBAC1D,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,EAAE;YAC1C,cAAc,EAAE,cAAc;YAC9B,gBAAgB;SACjB,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;iBACtC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type {\n ScrollToolResult,\n ScrollVisionToolResult,\n ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\n/*\n Simple scroll tool for DOM mode (non-grounding models).\n * No coordinates - scrolls from viewport center.\n /\nexport const scrollTool = (v3: V3) =>\n tool({\n description:\n \"Scroll the page up or down by a percentage of the viewport height. Default is 80%, and what should be typically used for general page scrolling\",\n inputSchema: z.object({\n direction: z.enum([\"up\", \"down\"]),\n percentage: z.number().min(1).max(200).optional(),\n }),\n execute: async ({\n direction,\n percentage = 80,\n }): Promise<ScrollToolResult> => {\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: scroll`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ direction, percentage }),\n type: \"object\",\n },\n },\n });\n\n const page = await v3.context.awaitActivePage();\n\n const { w, h } = await page.mainFrame().evaluate<{\n w: number;\n h: number;\n }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n const scrollDistance = Math.round((h percentage) / 100);\n const cx = Math.floor(w / 2);\n const cy = Math.floor(h / 2);\n const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n await page.scroll(cx, cy, 0, deltaY);\n\n v3.recordAgentReplayStep({\n type: \"scroll\",\n deltaX: 0,\n deltaY,\n anchor: { x: cx, y: cy },\n });\n\n return {\n success: true,\n message: `Scrolled ${percentage}% ${direction} (${scrollDistance}px)`,\n scrolledPixels: scrollDistance,\n };\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n return {\n type: \"json\",\n value: {\n success: result.success,\n message: result.message,\n scrolledPixels: result.scrolledPixels,\n },\n };\n },\n });\n\n/*\n Scroll tool for hybrid mode (grounding models).\n * Supports optional coordinates for scrolling within nested scrollable elements.\n /\nexport const scrollVisionTool = (v3: V3, provider?: string) =>\n tool({\n description: `Scroll the page up or down. For general page scrolling, no coordinates needed. Only provide coordinates when scrolling inside a nested scrollable element (e.g., a dropdown menu, modal with overflow, or scrollable sidebar). Default is 80%, and what should be typically used for general page scrolling`,\n inputSchema: z.object({\n direction: z.enum([\"up\", \"down\"]),\n coordinates: z\n .array(z.number())\n .optional()\n .describe(\n \"Only use coordinates for scrolling inside a nested scrollable element - provide (x, y) within that element\",\n ),\n percentage: z.number().min(1).max(200).optional(),\n }),\n execute: async ({\n direction,\n coordinates,\n percentage = 80,\n }): Promise<ScrollVisionToolResult> => {\n const page = await v3.context.awaitActivePage();\n\n const { w, h } = await page.mainFrame().evaluate<{\n w: number;\n h: number;\n }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n // Process coordinates if provided, otherwise use viewport center\n let cx: number;\n let cy: number;\n if (coordinates) {\n const processed = await processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n cx = processed.x;\n cy = processed.y;\n } else {\n cx = Math.floor(w / 2);\n cy = Math.floor(h / 2);\n }\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: scroll`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({\n direction,\n coordinates,\n percentage,\n processed: { cx, cy },\n }),\n type: \"object\",\n },\n },\n });\n\n const scrollDistance = Math.round((h percentage) / 100);\n const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n await page.scroll(cx, cy, 0, deltaY);\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n v3.recordAgentReplayStep({\n type: \"scroll\",\n deltaX: 0,\n deltaY,\n anchor: { x: cx, y: cy },\n });\n\n return {\n success: true,\n message: coordinates\n ? `Scrolled ${percentage}% ${direction} at (${cx}, ${cy})`\n : `Scrolled ${percentage}% ${direction}`,\n scrolledPixels: scrollDistance,\n screenshotBase64,\n };\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n message: result.message,\n scrolledPixels: result.scrolledPixels,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n"]}

package/dist/esm/lib/v3/agent/tools/type.js CHANGED Viewed

@@ -23,7 +23,7 @@ export const typeTool = (v3, provider, variables) => {
         execute: async ({ describe, coordinates, text, }) => {
             try {
                 const page = await v3.context.awaitActivePage();
-                const processed = processCoordinates(coordinates[0], coordinates[1], provider, v3);
+                const processed = await processCoordinates(coordinates[0], coordinates[1], provider, v3);
                 // Substitute any %variableName% tokens in the text
                 const actualText = substituteVariables(text, variables);
                 v3.logger({
@@ -39,9 +39,18 @@ export const typeTool = (v3, provider, variables) => {
                 });
                 // Only request XPath when caching is enabled to avoid unnecessary computation
                 const shouldCollectXpath = v3.isAgentReplayActive();
-                const xpath = await page.click(processed.x, processed.y, {
-                    returnXpath: shouldCollectXpath,
-                });
+                // Focus the field the way the session's input actually works: on a touch
+                // session a synthesized mouse click never registers on touch-gated mobile
+                // layouts, so the field would stay unfocused and the text go nowhere.
+                // Only the actuation changes — the recorded step is a "type", whose replay
+                // focuses via the selector rather than a pointer event.
+                const xpath = v3.usesTouch
+                    ? await page.tap(processed.x, processed.y, {
+                        returnXpath: shouldCollectXpath,
+                    })
+                    : await page.click(processed.x, processed.y, {
+                        returnXpath: shouldCollectXpath,
+                    });
                 await page.type(actualText);
                 const screenshotBase64 = await waitAndCaptureScreenshot(page);
                 // Record as an "act" step with proper Action for deterministic replay (only when caching)

package/dist/esm/lib/v3/agent/tools/type.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"type.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,SAAqB,EAAE,EAAE;IAC3E,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,YAAY;QAClC,CAAC,CAAC,oGAAoG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACzI,CAAC,CAAC,mCAAmC,CAAC;IAExC,OAAO,IAAI,CAAC;QACV,WAAW,EACT,2QAA2Q;QAC7Q,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,QAAQ,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,CACP,4HAA4H,CAC7H;YACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;YAC1C,WAAW,EAAE,CAAC;iBACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,QAAQ,CAAC,iDAAiD,CAAC;SAC/D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,WAAW,EACX,IAAI,GACL,EAA2B,EAAE;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,kBAAkB,~~CAClC~~,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;gBAEF,mDAAmD;gBACnD,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAExD,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,0BAA0B;oBACnC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;4BACzC,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;~~oBACvD~~,WAAW,EAAE,kBAAkB;~~iBAChC~~,CAAC,CAAC;~~gBAEH~~,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE5B,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;gBAE9D,0FAA0F;gBAC1F,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;oBAC3C,IAAI,eAAe,EAAE,CAAC;wBACpB,MAAM,MAAM,GAAW;4BACrB,QAAQ,EAAE,eAAe;4BACzB,WAAW,EAAE,QAAQ;4BACrB,MAAM,EAAE,MAAM;4BACd,SAAS,EAAE,CAAC,IAAI,CAAC;yBAClB,CAAC;wBACF,EAAE,CAAC,qBAAqB,CAAC;4BACvB,IAAI,EAAE,KAAK;4BACX,WAAW,EAAE,QAAQ;4BACrB,OAAO,EAAE,CAAC,MAAM,CAAC;4BACjB,iBAAiB,EAAE,QAAQ;yBAC5B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,QAAQ;oBACR,IAAI,EAAE,8FAA8F;oBACpG,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,iBAAiB,KAAK,CAAC,OAAO,EAAE;iBACxC,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;iBACxD,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n TypeToolResult,\n ModelOutputContentItem,\n Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const typeTool = (v3: V3, provider?: string, variables?: Variables) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const textDescription = hasVariables\n ? `The text to type into the element. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n : \"The text to type into the element\";\n\n return tool({\n description:\n \"Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n inputSchema: z.object({\n describe: z\n .string()\n .describe(\n \"Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description\",\n ),\n text: z.string().describe(textDescription),\n coordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to type into the element\"),\n }),\n execute: async ({\n describe,\n coordinates,\n text,\n }): Promise<TypeToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const processed = processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n\n // Substitute any %variableName% tokens in the text\n const actualText = substituteVariables(text, variables);\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: type`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ describe, text }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const xpath = await page.click(processed.x, processed.y, {\n returnXpath: shouldCollectXpath,\n });\n\n await page.type(actualText);\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n const action: Action = {\n selector: normalizedXpath,\n description: describe,\n method: \"type\",\n arguments: [text],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return {\n success: true,\n describe,\n text, // Return original text (with %variableName% tokens) to avoid exposing sensitive values to LLM\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error typing: ${error.message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n describe: result.describe,\n text: result.text,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n};\n"]}
1	+ {"version":3,"file":"type.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,SAAqB,EAAE,EAAE;IAC3E,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,YAAY;QAClC,CAAC,CAAC,oGAAoG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACzI,CAAC,CAAC,mCAAmC,CAAC;IAExC,OAAO,IAAI,CAAC;QACV,WAAW,EACT,2QAA2Q;QAC7Q,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,QAAQ,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,CACP,4HAA4H,CAC7H;YACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;YAC1C,WAAW,EAAE,CAAC;iBACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,QAAQ,CAAC,iDAAiD,CAAC;SAC/D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,WAAW,EACX,IAAI,GACL,EAA2B,EAAE;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,MAAM,kBAAkB,CACxC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;gBAEF,mDAAmD;gBACnD,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAExD,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,0BAA0B;oBACnC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;4BACzC,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,yEAAyE;gBACzE,0EAA0E;gBAC1E,sEAAsE;gBACtE,2EAA2E;gBAC3E,wDAAwD;gBACxD,MAAM,KAAK,GAAG,EAAE,CAAC,SAAS;oBACxB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;wBACvC,WAAW,EAAE,kBAAkB;qBAChC,CAAC;oBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;wBACzC,WAAW,EAAE,kBAAkB;qBAChC,CAAC,CAAC;gBAEP,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE5B,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;gBAE9D,0FAA0F;gBAC1F,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;oBAC3C,IAAI,eAAe,EAAE,CAAC;wBACpB,MAAM,MAAM,GAAW;4BACrB,QAAQ,EAAE,eAAe;4BACzB,WAAW,EAAE,QAAQ;4BACrB,MAAM,EAAE,MAAM;4BACd,SAAS,EAAE,CAAC,IAAI,CAAC;yBAClB,CAAC;wBACF,EAAE,CAAC,qBAAqB,CAAC;4BACvB,IAAI,EAAE,KAAK;4BACX,WAAW,EAAE,QAAQ;4BACrB,OAAO,EAAE,CAAC,MAAM,CAAC;4BACjB,iBAAiB,EAAE,QAAQ;yBAC5B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,QAAQ;oBACR,IAAI,EAAE,8FAA8F;oBACpG,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,iBAAiB,KAAK,CAAC,OAAO,EAAE;iBACxC,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;iBACxD,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n TypeToolResult,\n ModelOutputContentItem,\n Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const typeTool = (v3: V3, provider?: string, variables?: Variables) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const textDescription = hasVariables\n ? `The text to type into the element. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n : \"The text to type into the element\";\n\n return tool({\n description:\n \"Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n inputSchema: z.object({\n describe: z\n .string()\n .describe(\n \"Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description\",\n ),\n text: z.string().describe(textDescription),\n coordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to type into the element\"),\n }),\n execute: async ({\n describe,\n coordinates,\n text,\n }): Promise<TypeToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const processed = await processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n\n // Substitute any %variableName% tokens in the text\n const actualText = substituteVariables(text, variables);\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: type`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ describe, text }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n // Focus the field the way the session's input actually works: on a touch\n // session a synthesized mouse click never registers on touch-gated mobile\n // layouts, so the field would stay unfocused and the text go nowhere.\n // Only the actuation changes — the recorded step is a \"type\", whose replay\n // focuses via the selector rather than a pointer event.\n const xpath = v3.usesTouch\n ? await page.tap(processed.x, processed.y, {\n returnXpath: shouldCollectXpath,\n })\n : await page.click(processed.x, processed.y, {\n returnXpath: shouldCollectXpath,\n });\n\n await page.type(actualText);\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n const action: Action = {\n selector: normalizedXpath,\n description: describe,\n method: \"type\",\n arguments: [text],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return {\n success: true,\n describe,\n text, // Return original text (with %variableName% tokens) to avoid exposing sensitive values to LLM\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error typing: ${error.message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n describe: result.describe,\n text: result.text,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n};\n"]}

package/dist/esm/lib/v3/agent/utils/coordinateNormalization.d.ts CHANGED Viewed

@@ -7,7 +7,10 @@ export declare function normalizeGoogleCoordinates(x: number, y: number, viewpor
     x: number;
     y: number;
 };
-export declare function processCoordinates(x: number, y: number, provider?: string, v3?: V3): {
+export declare function processCoordinates(x: number, y: number, provider?: string, v3?: V3, resolvedViewport?: {
+    width: number;
+    height: number;
+}): Promise<{
     x: number;
     y: number;
-};
+}>;

package/dist/esm/lib/v3/agent/utils/coordinateNormalization.js CHANGED Viewed

@@ -1,5 +1,3 @@
-// Default viewport for Browserbase managed fingerprinting mode
-const STEALTH_VIEWPORT = { width: 1288, height: 711 };
 export function isGoogleProvider(provider) {
     if (!provider)
         return false;
@@ -15,11 +13,11 @@ export function normalizeGoogleCoordinates(x, y, viewport) {
         y: Math.floor((clampedY / 1000) * viewport.height),
     };
 }
-export function processCoordinates(x, y, provider, v3) {
-    if (isGoogleProvider(provider) && v3) {
-        // Browserbase managed fingerprinting uses a fixed viewport fallback.
-        const viewport = v3.isVerified ? STEALTH_VIEWPORT : v3.configuredViewport;
-        return normalizeGoogleCoordinates(x, y, viewport);
+export async function processCoordinates(x, y, provider, v3, resolvedViewport) {
+    if (isGoogleProvider(provider)) {
+        const viewport = resolvedViewport ?? (v3 ? await v3.resolveViewport() : undefined);
+        if (viewport)
+            return normalizeGoogleCoordinates(x, y, viewport);
     }
     return { x, y };
 }

package/dist/esm/lib/v3/agent/utils/coordinateNormalization.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"coordinateNormalization.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/coordinateNormalization.ts"],"names":[],"mappings":"AAEA~~,+DAA+D;AAC/D~~,MAAM,~~gBAAgB,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;AAEtD,MAAM,~~UAAU,gBAAgB,CAAC,QAAiB;IAChD,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5B,OAAO,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AACnD,CAAC;AAED,qEAAqE;AACrE,kCAAkC;AAClC,MAAM,UAAU,0BAA0B,CACxC,CAAS,EACT,CAAS,EACT,QAA2C;IAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO;QACL,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC;QACjD,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;KACnD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,~~CAChC~~,CAAS,EACT,CAAS,EACT,QAAiB,EACjB,EAAO;~~IAEP~~,IAAI,gBAAgB,CAAC,QAAQ,CAAC,~~IAAI,~~EAAE,~~EAAE,~~CAAC;~~QACrC~~,~~qEAAqE;QACrE,~~MAAM,QAAQ,~~GAAG~~,~~EAAE~~,CAAC,~~UAAU~~,CAAC,CAAC,CAAC,~~gBAAgB~~,~~CAAC~~,CAAC,~~CAAC~~,EAAE,CAAC,~~kBAAkB~~,CAAC;~~QAC1E~~,OAAO,0BAA0B,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;~~IACpD~~,CAAC;IACD,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;AAClB,CAAC","sourcesContent":["import type { V3 } from \"../../v3.js\";\n\~~n// Default viewport for Browserbase managed fingerprinting mode\nconst STEALTH_VIEWPORT = { width: 1288, height: 711 };\n\~~nexport function isGoogleProvider(provider?: string): boolean {\n if (!provider) return false;\n return provider.toLowerCase().includes(\"google\");\n}\n\n// Google returns coordinates in a 0-1000 range, we need to normalize\n// them to the viewport dimensions\nexport function normalizeGoogleCoordinates(\n x: number,\n y: number,\n viewport: { width: number; height: number },\n): { x: number; y: number } {\n const clampedX = Math.min(999, Math.max(0, x));\n const clampedY = Math.min(999, Math.max(0, y));\n return {\n x: Math.floor((clampedX / 1000) * viewport.width),\n y: Math.floor((clampedY / 1000) * viewport.height),\n };\n}\n\nexport function processCoordinates(\n x: number,\n y: number,\n provider?: string,\n v3?: V3,\n): { x: number; y: number } {\n if (isGoogleProvider(provider) ~~&& v3~~) {\n // ~~Browserbase managed fingerprinting uses a fixed~~ viewport ~~fallback.\~~n ~~const~~ ~~viewport~~ = v3~~.isVerified~~ ? ~~STEALTH_VIEWPORT~~ : ~~v3.configuredViewport~~;\n return normalizeGoogleCoordinates(x, y, viewport);\n }\n return { x, y };\n}\n"]}
1	+ {"version":3,"file":"coordinateNormalization.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/coordinateNormalization.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,gBAAgB,CAAC,QAAiB;IAChD,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5B,OAAO,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AACnD,CAAC;AAED,qEAAqE;AACrE,kCAAkC;AAClC,MAAM,UAAU,0BAA0B,CACxC,CAAS,EACT,CAAS,EACT,QAA2C;IAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO;QACL,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC;QACjD,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;KACnD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,CAAS,EACT,CAAS,EACT,QAAiB,EACjB,EAAO,EACP,gBAAoD;IAEpD,IAAI,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/B,MAAM,QAAQ,GACZ,gBAAgB,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACpE,IAAI,QAAQ;YAAE,OAAO,0BAA0B,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;AAClB,CAAC","sourcesContent":["import type { V3 } from \"../../v3.js\";\n\nexport function isGoogleProvider(provider?: string): boolean {\n if (!provider) return false;\n return provider.toLowerCase().includes(\"google\");\n}\n\n// Google returns coordinates in a 0-1000 range, we need to normalize\n// them to the viewport dimensions\nexport function normalizeGoogleCoordinates(\n x: number,\n y: number,\n viewport: { width: number; height: number },\n): { x: number; y: number } {\n const clampedX = Math.min(999, Math.max(0, x));\n const clampedY = Math.min(999, Math.max(0, y));\n return {\n x: Math.floor((clampedX / 1000) * viewport.width),\n y: Math.floor((clampedY / 1000) * viewport.height),\n };\n}\n\nexport async function processCoordinates(\n x: number,\n y: number,\n provider?: string,\n v3?: V3,\n resolvedViewport?: { width: number; height: number },\n): Promise<{ x: number; y: number }> {\n if (isGoogleProvider(provider)) {\n const viewport =\n resolvedViewport ?? (v3 ? await v3.resolveViewport() : undefined);\n if (viewport) return normalizeGoogleCoordinates(x, y, viewport);\n }\n return { x, y };\n}\n"]}

package/dist/esm/lib/v3/agent/utils/handleDoneToolCall.js CHANGED Viewed

@@ -3,6 +3,7 @@ import { z } from "zod";
 import { tool } from "ai";
 import { anthropicFallbacksOptions, rejectsForcedToolUse, } from "../../llm/anthropicOptions.js";
 import { getZFactory } from "../../../utils.js";
+import { openAIStoreProviderOptions } from "../../llm/providerOptions.js";
 /**
  * Deep-remove `undefined` values from the run history before it is re-submitted
  * to the forced "done" call.
@@ -105,6 +106,7 @@ Call the "done" tool with:
             : "Provide your final assessment.",
     };
     const modelId = typeof model === "string" ? model : model.modelId;
+    const modelProvider = typeof model === "string" ? undefined : model.provider;
     const fallbacks = anthropicFallbacksOptions(modelId);
     // Models whose always-on thinking rejects forced tool use go straight to
     // "auto" — the prompt already instructs calling "done", and the
@@ -119,7 +121,7 @@ Call the "done" tool with:
             : { type: "tool", toolName: "done" },
         providerOptions: {
             google: { mediaResolution: "MEDIA_RESOLUTION_HIGH" },
-            openai: { store: false },
+            ...openAIStoreProviderOptions(modelProvider),
             ...(fallbacks ? { anthropic: fallbacks } : {}),
         },
     });

package/dist/esm/lib/v3/agent/utils/handleDoneToolCall.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"handleDoneToolCall.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/handleDoneToolCall.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAwC,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAE1B,OAAO,EACL,yBAAyB,EACzB,oBAAoB,GACrB,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAUhD;;;;;;;;;;;;;;;;GAgBG;AACH,SAAS,kBAAkB,CAAI,KAAQ;IACrC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAiB,CAAC;IACjE,CAAC;IACD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,KAAK,KAAK,MAAM,CAAC,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjD,MAAM,GAAG,GAA4B,EAAE,CAAC;YACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,EAAE,CAAC;gBACtE,IAAI,CAAC,KAAK,SAAS;oBAAE,GAAG,CAAC,CAAC,CAAC,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC;YACtD,CAAC;YACD,OAAO,GAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,QAAwB;IAExB,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAiB;IAC5C,OAAO,OAAO,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,OAAO;aACf,MAAM,EAAE;aACR,QAAQ,CAAC,0DAA0D,CAAC;QACvE,YAAY,EAAE,OAAO;aAClB,OAAO,EAAE;aACT,QAAQ,CAAC,uDAAuD,CAAC;KACrE,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAMxC;IACC,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE5E,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,0BAA0B;QACnC,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IACH,4EAA4E;IAC5E,MAAM,OAAO,GAAG,YAAY;QAC1B,CAAC,CAAC,WAAW,CAAC,YAAkC,CAAC;QACjD,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,cAAc,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEpD,qEAAqE;IACrE,MAAM,cAAc,GAAG,YAAY;QACjC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,YAAY,CAAC,QAAQ,CAC3B,qDAAqD,CACtD;SACF,CAAC;QACJ,CAAC,CAAC,cAAc,CAAC;IAEnB,MAAM,kBAAkB,GAAG,YAAY;QACrC,CAAC,CAAC,4GAA4G,IAAI,CAAC,SAAS,CACxH,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,GAAG,CACpC,CAAC,CAAC,GAAG,EAAE,KAAK,CAA+B,EAAE,EAAE,CAAC;YAC9C,GAAG;YACH,KAAK,CAAC,WAAW,IAAI,gBAAgB;SACtC,CACF,CACF,EACD,IAAI,EACJ,CAAC,CACF,EAAE;QACL,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,YAAY,GAAG;;;GAGpB,WAAW;;mPAEqO,kBAAkB;;;;gDAIrN,YAAY,CAAC,CAAC,CAAC,wDAAwD,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAE7H,MAAM,QAAQ,GAAG,IAAI,CAAC;QACpB,WAAW,EAAE,YAAY;YACvB,CAAC,CAAC,uEAAuE;YACzE,CAAC,CAAC,+CAA+C;QACnD,WAAW,EAAE,cAAc;QAC3B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;QACtC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,GAAiB;QAC/B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,YAAY;YACnB,CAAC,CAAC,8DAA8D;YAChE,CAAC,CAAC,gCAAgC;KACrC,CAAC;IAEF,MAAM,OAAO,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;IAClE,MAAM,SAAS,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;IAErD,yEAAyE;IACzE,gEAAgE;IAChE,uDAAuD;IACvD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;QAChC,KAAK;QACL,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE,CAAC,GAAG,+BAA+B,CAAC,aAAa,CAAC,EAAE,UAAU,CAAC;QACzE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAa;QACpC,UAAU,EAAE,oBAAoB,CAAC,OAAO,CAAC;YACvC,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE;QACtC,eAAe,EAAE;YACf,MAAM,EAAE,EAAE,eAAe,EAAE,uBAAuB,EAAE;YACpD,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;YACxB,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/C;KACF,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IAC5E,MAAM,cAAc,GAAmB;QACrC,UAAU;QACV,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,IAAI,EAAE,CAAC;KACrC,CAAC;IAEF,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO;YACL,SAAS,EAAE,MAAM,CAAC,IAAI,IAAI,0BAA0B;YACpD,YAAY,EAAE,KAAK;YACnB,QAAQ,EAAE,cAAc;SACzB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,KAI1B,CAAC;IACF,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,gBAAgB;QACzB,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IAEH,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,QAAQ,EAAE,cAAc;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC;AACJ,CAAC","sourcesContent":["import { generateText, ModelMessage, LanguageModel, ToolSet } from \"ai\";\nimport { z } from \"zod\";\nimport { tool } from \"ai\";\nimport { LogLine } from \"../../types/public/logs.js\";\nimport {\n anthropicFallbacksOptions,\n rejectsForcedToolUse,\n} from \"../../llm/anthropicOptions.js\";\nimport { StagehandZodObject } from \"../../zodCompat.js\";\nimport { getZFactory } from \"../../../utils.js\";\nimport type { StagehandZodSchema } from \"../../zodCompat.js\";\n\ninterface DoneResult {\n reasoning: string;\n taskComplete: boolean;\n messages: ModelMessage[];\n output?: Record<string, unknown>;\n}\n\n/*\n Deep-remove `undefined` values from the run history before it is re-submitted\n * to the forced \"done\" call.\n \n The AI SDK validates re-submitted messages against its `ModelMessage` schema,\n * whose JSON-value schema rejects `undefined` (only null/string/number/boolean/\n * object/array are allowed). A custom tool that returns an object with an\n * optional field left `undefined` (e.g. `{ matchedExpected: undefined }`) lands\n * that `undefined` inside a tool-result `output.value`, and reasoning models can\n * leave `undefined` in part fields too. Either makes the forced \"done\" call\n * throw `Invalid prompt: messages must be a ModelMessage[]` (STG-2335) — a red\n * error that fires after the run has already completed. Stripping `undefined`\n * keeps the history valid without dropping any real content.\n \n Only plain objects and arrays are traversed; class instances (URL, typed\n * arrays for binary image data, Date, …) are passed through untouched.\n /\nfunction stripUndefinedDeep<T>(value: T): T {\n if (Array.isArray(value)) {\n return value.map((v) => stripUndefinedDeep(v)) as unknown as T;\n }\n if (value !== null && typeof value === \"object\") {\n const proto = Object.getPrototypeOf(value);\n if (proto === Object.prototype \|\| proto === null) {\n const out: Record<string, unknown> = {};\n for (const [k, v] of Object.entries(value as Record<string, unknown>)) {\n if (v !== undefined) out[k] = stripUndefinedDeep(v);\n }\n return out as T;\n }\n }\n return value;\n}\n\nexport function sanitizeMessagesForResubmission(\n messages: ModelMessage[],\n): ModelMessage[] {\n return stripUndefinedDeep(messages);\n}\n\nfunction buildBaseDoneSchema(factory: typeof z) {\n return factory.object({\n reasoning: factory\n .string()\n .describe(\"Brief summary of what actions were taken and the outcome\"),\n taskComplete: factory\n .boolean()\n .describe(\"true if the task was fully completed, false otherwise\"),\n });\n}\n\n/\n Force a done tool call at the end of an agent run.\n * This ensures we always get a structured final response,\n * even if the main loop ended without calling done.\n */\nexport async function handleDoneToolCall(options: {\n model: LanguageModel;\n inputMessages: ModelMessage[];\n instruction: string;\n outputSchema?: StagehandZodObject;\n logger: (message: LogLine) => void;\n}): Promise<DoneResult> {\n const { model, inputMessages, instruction, outputSchema, logger } = options;\n\n logger({\n category: \"agent\",\n message: \"Agent calling tool: done\",\n level: 1,\n });\n // Use the same Zod version as the user's outputSchema to avoid v3/v4 mixing\n const factory = outputSchema\n ? getZFactory(outputSchema as StagehandZodSchema)\n : z;\n const baseDoneSchema = buildBaseDoneSchema(factory);\n\n // Merge base done schema with user-provided output schema if present\n const doneToolSchema = outputSchema\n ? baseDoneSchema.extend({\n output: outputSchema.describe(\n \"The specific data the user requested from this task\",\n ),\n })\n : baseDoneSchema;\n\n const outputInstructions = outputSchema\n ? `\\n\\nThe user also requested the following information from this task. Provide it in the \"output\" field:\\n${JSON.stringify(\n Object.fromEntries(\n Object.entries(outputSchema.shape).map(\n ([key, value]: [string, StagehandZodSchema]) => [\n key,\n value.description \|\| \"no description\",\n ],\n ),\n ),\n null,\n 2,\n )}`\n : \"\";\n\n const systemPrompt = `You are a web automation assistant that was tasked with completing a task.\n\nThe task was:\n\"${instruction}\"\n\nReview what was accomplished and provide your final assessment in whether the task was completed successfully. you have been provided with the history of the actions taken so far, use this to determine if the task was completed successfully.${outputInstructions}\n\nCall the \"done\" tool with:\n1. A brief summary of what was done\n2. Whether the task was completed successfully${outputSchema ? \"\\n3. The requested output data based on what you found\" : \"\"}`;\n\n const doneTool = tool({\n description: outputSchema\n ? \"Complete the task with your assessment and the requested output data.\"\n : \"Complete the task with your final assessment.\",\n inputSchema: doneToolSchema,\n execute: async (params) => {\n return { success: true, ...params };\n },\n });\n\n const userPrompt: ModelMessage = {\n role: \"user\",\n content: outputSchema\n ? \"Provide your final assessment and the requested output data.\"\n : \"Provide your final assessment.\",\n };\n\n const modelId = typeof model === \"string\" ? model : model.modelId;\n const fallbacks = anthropicFallbacksOptions(modelId);\n\n // Models whose always-on thinking rejects forced tool use go straight to\n // \"auto\" — the prompt already instructs calling \"done\", and the\n // no-tool-call case below handles a plain-text answer.\n const result = await generateText({\n model,\n system: systemPrompt,\n messages: [...sanitizeMessagesForResubmission(inputMessages), userPrompt],\n tools: { done: doneTool } as ToolSet,\n toolChoice: rejectsForcedToolUse(modelId)\n ? \"auto\"\n : { type: \"tool\", toolName: \"done\" },\n providerOptions: {\n google: { mediaResolution: \"MEDIA_RESOLUTION_HIGH\" },\n openai: { store: false },\n ...(fallbacks ? { anthropic: fallbacks } : {}),\n },\n });\n\n const doneToolCall = result.toolCalls?.find((tc) => tc.toolName === \"done\");\n const outputMessages: ModelMessage[] = [\n userPrompt,\n ...(result.response?.messages \|\| []),\n ];\n\n if (!doneToolCall) {\n return {\n reasoning: result.text \|\| \"Task execution completed\",\n taskComplete: false,\n messages: outputMessages,\n };\n }\n\n const input = doneToolCall.input as {\n reasoning: string;\n taskComplete: boolean;\n output?: Record<string, unknown>;\n };\n logger({\n category: \"agent\",\n message: `Task completed`,\n level: 1,\n });\n\n return {\n reasoning: input.reasoning,\n taskComplete: input.taskComplete,\n messages: outputMessages,\n output: input.output,\n };\n}\n"]}
1	+ {"version":3,"file":"handleDoneToolCall.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/handleDoneToolCall.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAwC,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAE1B,OAAO,EACL,yBAAyB,EACzB,oBAAoB,GACrB,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,0BAA0B,EAAE,MAAM,8BAA8B,CAAC;AAS1E;;;;;;;;;;;;;;;;GAgBG;AACH,SAAS,kBAAkB,CAAI,KAAQ;IACrC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAiB,CAAC;IACjE,CAAC;IACD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,KAAK,KAAK,MAAM,CAAC,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjD,MAAM,GAAG,GAA4B,EAAE,CAAC;YACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,EAAE,CAAC;gBACtE,IAAI,CAAC,KAAK,SAAS;oBAAE,GAAG,CAAC,CAAC,CAAC,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC;YACtD,CAAC;YACD,OAAO,GAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,QAAwB;IAExB,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAiB;IAC5C,OAAO,OAAO,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,OAAO;aACf,MAAM,EAAE;aACR,QAAQ,CAAC,0DAA0D,CAAC;QACvE,YAAY,EAAE,OAAO;aAClB,OAAO,EAAE;aACT,QAAQ,CAAC,uDAAuD,CAAC;KACrE,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAMxC;IACC,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE5E,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,0BAA0B;QACnC,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IACH,4EAA4E;IAC5E,MAAM,OAAO,GAAG,YAAY;QAC1B,CAAC,CAAC,WAAW,CAAC,YAAkC,CAAC;QACjD,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,cAAc,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEpD,qEAAqE;IACrE,MAAM,cAAc,GAAG,YAAY;QACjC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,YAAY,CAAC,QAAQ,CAC3B,qDAAqD,CACtD;SACF,CAAC;QACJ,CAAC,CAAC,cAAc,CAAC;IAEnB,MAAM,kBAAkB,GAAG,YAAY;QACrC,CAAC,CAAC,4GAA4G,IAAI,CAAC,SAAS,CACxH,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,GAAG,CACpC,CAAC,CAAC,GAAG,EAAE,KAAK,CAA+B,EAAE,EAAE,CAAC;YAC9C,GAAG;YACH,KAAK,CAAC,WAAW,IAAI,gBAAgB;SACtC,CACF,CACF,EACD,IAAI,EACJ,CAAC,CACF,EAAE;QACL,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,YAAY,GAAG;;;GAGpB,WAAW;;mPAEqO,kBAAkB;;;;gDAIrN,YAAY,CAAC,CAAC,CAAC,wDAAwD,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAE7H,MAAM,QAAQ,GAAG,IAAI,CAAC;QACpB,WAAW,EAAE,YAAY;YACvB,CAAC,CAAC,uEAAuE;YACzE,CAAC,CAAC,+CAA+C;QACnD,WAAW,EAAE,cAAc;QAC3B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;QACtC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,GAAiB;QAC/B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,YAAY;YACnB,CAAC,CAAC,8DAA8D;YAChE,CAAC,CAAC,gCAAgC;KACrC,CAAC;IAEF,MAAM,OAAO,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;IAClE,MAAM,aAAa,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;IAC7E,MAAM,SAAS,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;IAErD,yEAAyE;IACzE,gEAAgE;IAChE,uDAAuD;IACvD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;QAChC,KAAK;QACL,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE,CAAC,GAAG,+BAA+B,CAAC,aAAa,CAAC,EAAE,UAAU,CAAC;QACzE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAa;QACpC,UAAU,EAAE,oBAAoB,CAAC,OAAO,CAAC;YACvC,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE;QACtC,eAAe,EAAE;YACf,MAAM,EAAE,EAAE,eAAe,EAAE,uBAAuB,EAAE;YACpD,GAAG,0BAA0B,CAAC,aAAa,CAAC;YAC5C,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/C;KACF,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IAC5E,MAAM,cAAc,GAAmB;QACrC,UAAU;QACV,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,IAAI,EAAE,CAAC;KACrC,CAAC;IAEF,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO;YACL,SAAS,EAAE,MAAM,CAAC,IAAI,IAAI,0BAA0B;YACpD,YAAY,EAAE,KAAK;YACnB,QAAQ,EAAE,cAAc;SACzB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,KAI1B,CAAC;IACF,MAAM,CAAC;QACL,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,gBAAgB;QACzB,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IAEH,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,QAAQ,EAAE,cAAc;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC;AACJ,CAAC","sourcesContent":["import { generateText, ModelMessage, LanguageModel, ToolSet } from \"ai\";\nimport { z } from \"zod\";\nimport { tool } from \"ai\";\nimport { LogLine } from \"../../types/public/logs.js\";\nimport {\n anthropicFallbacksOptions,\n rejectsForcedToolUse,\n} from \"../../llm/anthropicOptions.js\";\nimport { StagehandZodObject } from \"../../zodCompat.js\";\nimport { getZFactory } from \"../../../utils.js\";\nimport type { StagehandZodSchema } from \"../../zodCompat.js\";\nimport { openAIStoreProviderOptions } from \"../../llm/providerOptions.js\";\n\ninterface DoneResult {\n reasoning: string;\n taskComplete: boolean;\n messages: ModelMessage[];\n output?: Record<string, unknown>;\n}\n\n/*\n Deep-remove `undefined` values from the run history before it is re-submitted\n * to the forced \"done\" call.\n \n The AI SDK validates re-submitted messages against its `ModelMessage` schema,\n * whose JSON-value schema rejects `undefined` (only null/string/number/boolean/\n * object/array are allowed). A custom tool that returns an object with an\n * optional field left `undefined` (e.g. `{ matchedExpected: undefined }`) lands\n * that `undefined` inside a tool-result `output.value`, and reasoning models can\n * leave `undefined` in part fields too. Either makes the forced \"done\" call\n * throw `Invalid prompt: messages must be a ModelMessage[]` (STG-2335) — a red\n * error that fires after the run has already completed. Stripping `undefined`\n * keeps the history valid without dropping any real content.\n \n Only plain objects and arrays are traversed; class instances (URL, typed\n * arrays for binary image data, Date, …) are passed through untouched.\n /\nfunction stripUndefinedDeep<T>(value: T): T {\n if (Array.isArray(value)) {\n return value.map((v) => stripUndefinedDeep(v)) as unknown as T;\n }\n if (value !== null && typeof value === \"object\") {\n const proto = Object.getPrototypeOf(value);\n if (proto === Object.prototype \|\| proto === null) {\n const out: Record<string, unknown> = {};\n for (const [k, v] of Object.entries(value as Record<string, unknown>)) {\n if (v !== undefined) out[k] = stripUndefinedDeep(v);\n }\n return out as T;\n }\n }\n return value;\n}\n\nexport function sanitizeMessagesForResubmission(\n messages: ModelMessage[],\n): ModelMessage[] {\n return stripUndefinedDeep(messages);\n}\n\nfunction buildBaseDoneSchema(factory: typeof z) {\n return factory.object({\n reasoning: factory\n .string()\n .describe(\"Brief summary of what actions were taken and the outcome\"),\n taskComplete: factory\n .boolean()\n .describe(\"true if the task was fully completed, false otherwise\"),\n });\n}\n\n/\n Force a done tool call at the end of an agent run.\n * This ensures we always get a structured final response,\n * even if the main loop ended without calling done.\n */\nexport async function handleDoneToolCall(options: {\n model: LanguageModel;\n inputMessages: ModelMessage[];\n instruction: string;\n outputSchema?: StagehandZodObject;\n logger: (message: LogLine) => void;\n}): Promise<DoneResult> {\n const { model, inputMessages, instruction, outputSchema, logger } = options;\n\n logger({\n category: \"agent\",\n message: \"Agent calling tool: done\",\n level: 1,\n });\n // Use the same Zod version as the user's outputSchema to avoid v3/v4 mixing\n const factory = outputSchema\n ? getZFactory(outputSchema as StagehandZodSchema)\n : z;\n const baseDoneSchema = buildBaseDoneSchema(factory);\n\n // Merge base done schema with user-provided output schema if present\n const doneToolSchema = outputSchema\n ? baseDoneSchema.extend({\n output: outputSchema.describe(\n \"The specific data the user requested from this task\",\n ),\n })\n : baseDoneSchema;\n\n const outputInstructions = outputSchema\n ? `\\n\\nThe user also requested the following information from this task. Provide it in the \"output\" field:\\n${JSON.stringify(\n Object.fromEntries(\n Object.entries(outputSchema.shape).map(\n ([key, value]: [string, StagehandZodSchema]) => [\n key,\n value.description \|\| \"no description\",\n ],\n ),\n ),\n null,\n 2,\n )}`\n : \"\";\n\n const systemPrompt = `You are a web automation assistant that was tasked with completing a task.\n\nThe task was:\n\"${instruction}\"\n\nReview what was accomplished and provide your final assessment in whether the task was completed successfully. you have been provided with the history of the actions taken so far, use this to determine if the task was completed successfully.${outputInstructions}\n\nCall the \"done\" tool with:\n1. A brief summary of what was done\n2. Whether the task was completed successfully${outputSchema ? \"\\n3. The requested output data based on what you found\" : \"\"}`;\n\n const doneTool = tool({\n description: outputSchema\n ? \"Complete the task with your assessment and the requested output data.\"\n : \"Complete the task with your final assessment.\",\n inputSchema: doneToolSchema,\n execute: async (params) => {\n return { success: true, ...params };\n },\n });\n\n const userPrompt: ModelMessage = {\n role: \"user\",\n content: outputSchema\n ? \"Provide your final assessment and the requested output data.\"\n : \"Provide your final assessment.\",\n };\n\n const modelId = typeof model === \"string\" ? model : model.modelId;\n const modelProvider = typeof model === \"string\" ? undefined : model.provider;\n const fallbacks = anthropicFallbacksOptions(modelId);\n\n // Models whose always-on thinking rejects forced tool use go straight to\n // \"auto\" — the prompt already instructs calling \"done\", and the\n // no-tool-call case below handles a plain-text answer.\n const result = await generateText({\n model,\n system: systemPrompt,\n messages: [...sanitizeMessagesForResubmission(inputMessages), userPrompt],\n tools: { done: doneTool } as ToolSet,\n toolChoice: rejectsForcedToolUse(modelId)\n ? \"auto\"\n : { type: \"tool\", toolName: \"done\" },\n providerOptions: {\n google: { mediaResolution: \"MEDIA_RESOLUTION_HIGH\" },\n ...openAIStoreProviderOptions(modelProvider),\n ...(fallbacks ? { anthropic: fallbacks } : {}),\n },\n });\n\n const doneToolCall = result.toolCalls?.find((tc) => tc.toolName === \"done\");\n const outputMessages: ModelMessage[] = [\n userPrompt,\n ...(result.response?.messages \|\| []),\n ];\n\n if (!doneToolCall) {\n return {\n reasoning: result.text \|\| \"Task execution completed\",\n taskComplete: false,\n messages: outputMessages,\n };\n }\n\n const input = doneToolCall.input as {\n reasoning: string;\n taskComplete: boolean;\n output?: Record<string, unknown>;\n };\n logger({\n category: \"agent\",\n message: `Task completed`,\n level: 1,\n });\n\n return {\n reasoning: input.reasoning,\n taskComplete: input.taskComplete,\n messages: outputMessages,\n output: input.output,\n };\n}\n"]}

package/dist/esm/lib/v3/api.d.ts CHANGED Viewed

@@ -91,7 +91,7 @@ export declare class StagehandAPIClient {
     private latestAgentCacheEntry;
     private warnedStagehandBaseUrl;
     constructor({ apiKey, projectId, logger, serverCache, }: StagehandAPIConstructorParams);
-    init({ modelName, modelApiKey, defaultModelConfig, domSettleTimeoutMs, verbose, systemPrompt, selfHeal, browserbaseSessionCreateParams, browserbaseSessionID, }: ClientSessionStartParams): Promise<Api.SessionStartResult>;
+    init({ modelName, modelApiKey, defaultModelConfig, domSettleTimeoutMs, verbose, systemPrompt, selfHeal, useTouch, browserbaseSessionCreateParams, browserbaseSessionID, }: ClientSessionStartParams): Promise<Api.SessionStartResult>;
     act({ input, options, frameId, }: ClientActParameters): Promise<ActResult>;
     extract<T extends StagehandZodSchema>({ instruction, schema: zodSchema, options, frameId, }: ClientExtractParameters): Promise<ExtractResult<T>>;
     observe({ instruction, options, frameId, }: ClientObserveParameters): Promise<ObserveResult>;

package/dist/esm/lib/v3/api.js CHANGED Viewed

@@ -1,5 +1,6 @@
 import makeFetchCookie from "fetch-cookie";
 import { getInheritableModelOptions, hasModelProviderAuth, loadApiKeyFromEnv, } from "../utils.js";
+import { isAutoModel } from "../modelUtils.js";
 import { STAGEHAND_VERSION } from "../version.js";
 import { StagehandAPIError, StagehandAPIUnauthorizedError, StagehandHttpError, StagehandResponseBodyError, StagehandResponseParseError, StagehandServerError, ExperimentalNotConfiguredError, } from "./types/public/index.js";
 import { toJsonSchema } from "./zodCompat.js";
@@ -51,7 +52,7 @@ export class StagehandAPIClient {
         // Create a single cookie jar instance that will persist across all requests
         this.fetchWithCookies = makeFetchCookie(fetch);
     }
-    async init({ modelName, modelApiKey, defaultModelConfig, domSettleTimeoutMs, verbose, systemPrompt, selfHeal, browserbaseSessionCreateParams, browserbaseSessionID,
+    async init({ modelName, modelApiKey, defaultModelConfig, domSettleTimeoutMs, verbose, systemPrompt, selfHeal, useTouch, browserbaseSessionCreateParams, browserbaseSessionID,
     // browser,  TODO for local browsers
      }) {
         this.modelApiKey = modelApiKey;
@@ -76,6 +77,7 @@ export class StagehandAPIClient {
             verbose,
             systemPrompt,
             selfHeal,
+            useTouch,
             browserbaseSessionCreateParams,
             browserbaseSessionID,
             // browser, TODO: only send when connected to local fastify
@@ -412,6 +414,14 @@ export class StagehandAPIClient {
      * model provider differs from the one used to init the session.
      */
     prepareModelConfig(model) {
+        // "auto" delegates model selection to the API, so never inherit the
+        // default model's provider config or API key — the server picks the
+        // provider. Explicit per-call options are still passed through.
+        if (isAutoModel(model)) {
+            return typeof model === "string"
+                ? { modelName: model }
+                : { ...model };
+        }
         if (typeof model === "string") {
             // Extract provider from model string (e.g., "openai/gpt-5-nano" -> "openai")
             const provider = this.getModelProvider(model);