@browserbasehq/orca 3.7.2-preview.0 → 3.7.2-preview.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/lib/v3/agent/tools/click.js +1 -1
- package/dist/cjs/lib/v3/agent/tools/click.js.map +1 -1
- package/dist/cjs/lib/v3/agent/tools/clickAndHold.js +1 -1
- package/dist/cjs/lib/v3/agent/tools/clickAndHold.js.map +1 -1
- package/dist/cjs/lib/v3/agent/tools/dragAndDrop.js +2 -2
- package/dist/cjs/lib/v3/agent/tools/dragAndDrop.js.map +1 -1
- package/dist/cjs/lib/v3/agent/tools/fillFormVision.js +6 -3
- package/dist/cjs/lib/v3/agent/tools/fillFormVision.js.map +1 -1
- package/dist/cjs/lib/v3/agent/tools/scroll.js +1 -1
- package/dist/cjs/lib/v3/agent/tools/scroll.js.map +1 -1
- package/dist/cjs/lib/v3/agent/tools/type.js +1 -1
- package/dist/cjs/lib/v3/agent/tools/type.js.map +1 -1
- package/dist/cjs/lib/v3/agent/utils/coordinateNormalization.d.ts +5 -2
- package/dist/cjs/lib/v3/agent/utils/coordinateNormalization.js +5 -7
- package/dist/cjs/lib/v3/agent/utils/coordinateNormalization.js.map +1 -1
- package/dist/cjs/lib/v3/handlers/v3CuaAgentHandler.js +2 -20
- package/dist/cjs/lib/v3/handlers/v3CuaAgentHandler.js.map +1 -1
- package/dist/cjs/lib/v3/llm/aisdk.js +3 -3
- package/dist/cjs/lib/v3/llm/aisdk.js.map +1 -1
- package/dist/cjs/lib/v3/v3.d.ts +6 -0
- package/dist/cjs/lib/v3/v3.js +28 -13
- package/dist/cjs/lib/v3/v3.js.map +1 -1
- package/dist/esm/lib/v3/agent/tools/click.js +1 -1
- package/dist/esm/lib/v3/agent/tools/click.js.map +1 -1
- package/dist/esm/lib/v3/agent/tools/clickAndHold.js +1 -1
- package/dist/esm/lib/v3/agent/tools/clickAndHold.js.map +1 -1
- package/dist/esm/lib/v3/agent/tools/dragAndDrop.js +2 -2
- package/dist/esm/lib/v3/agent/tools/dragAndDrop.js.map +1 -1
- package/dist/esm/lib/v3/agent/tools/fillFormVision.js +7 -4
- package/dist/esm/lib/v3/agent/tools/fillFormVision.js.map +1 -1
- package/dist/esm/lib/v3/agent/tools/scroll.js +1 -1
- package/dist/esm/lib/v3/agent/tools/scroll.js.map +1 -1
- package/dist/esm/lib/v3/agent/tools/type.js +1 -1
- package/dist/esm/lib/v3/agent/tools/type.js.map +1 -1
- package/dist/esm/lib/v3/agent/utils/coordinateNormalization.d.ts +5 -2
- package/dist/esm/lib/v3/agent/utils/coordinateNormalization.js +5 -7
- package/dist/esm/lib/v3/agent/utils/coordinateNormalization.js.map +1 -1
- package/dist/esm/lib/v3/handlers/v3CuaAgentHandler.js +2 -20
- package/dist/esm/lib/v3/handlers/v3CuaAgentHandler.js.map +1 -1
- package/dist/esm/lib/v3/llm/aisdk.js +3 -3
- package/dist/esm/lib/v3/llm/aisdk.js.map +1 -1
- package/dist/esm/lib/v3/v3.d.ts +6 -0
- package/dist/esm/lib/v3/v3.js +28 -13
- package/dist/esm/lib/v3/v3.js.map +1 -1
- package/package.json +1 -1
|
@@ -16,7 +16,7 @@ export const clickTool = (v3, provider) => tool({
|
|
|
16
16
|
execute: async ({ describe, coordinates }) => {
|
|
17
17
|
try {
|
|
18
18
|
const page = await v3.context.awaitActivePage();
|
|
19
|
-
const processed = processCoordinates(coordinates[0], coordinates[1], provider, v3);
|
|
19
|
+
const processed = await processCoordinates(coordinates[0], coordinates[1], provider, v3);
|
|
20
20
|
v3.logger({
|
|
21
21
|
category: "agent",
|
|
22
22
|
message: `Agent calling tool: click`,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAI,CAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,
|
|
1
|
+
{"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAI,CAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,MAAM,kBAAkB,CACxC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,0EAA0E;YAC1E,6EAA6E;YAC7E,4EAA4E;YAC5E,4EAA4E;YAC5E,MAAM,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;YAC9B,MAAM,KAAK,GAAG,QAAQ;gBACpB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;oBACvC,WAAW,EAAE,kBAAkB;iBAChC,CAAC;gBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;oBACzC,WAAW,EAAE,kBAAkB;iBAChC,CAAC,CAAC;YAEP,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,mEAAmE;wBACnE,oDAAoD;wBACpD,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO;wBAClC,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAmB,KAAK,CAAC,OAAO,EAAE;aAC1C,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n ClickToolResult,\n ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n tool({\n description:\n \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n inputSchema: z.object({\n describe: z\n .string()\n .describe(\n \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n ),\n coordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to click on\"),\n }),\n execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const processed = await processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: click`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ describe }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n // On a touch session, actuate as a trusted tap rather than a mouse click:\n // mobile layouts commonly gate their handlers on touch/pointer events, where\n // a synthesized mouse click never registers (a size selector highlights but\n // still reports \"please choose a size\"). Mirrors the CUA handler's routing.\n const useTouch = v3.usesTouch;\n const xpath = useTouch\n ? await page.tap(processed.x, processed.y, {\n returnXpath: shouldCollectXpath,\n })\n : await page.click(processed.x, processed.y, {\n returnXpath: shouldCollectXpath,\n });\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n const action: Action = {\n selector: normalizedXpath,\n description: describe,\n // Replay the same input kind that was recorded, so a cached mobile\n // run does not silently fall back to a mouse click.\n method: useTouch ? \"tap\" : \"click\",\n arguments: [],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return {\n success: true,\n describe,\n coordinates: [processed.x, processed.y],\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error clicking: ${error.message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false || result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n describe: result.describe,\n coordinates: result.coordinates,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n"]}
|
|
@@ -18,7 +18,7 @@ export const clickAndHoldTool = (v3, provider) => tool({
|
|
|
18
18
|
execute: async ({ describe, coordinates, duration }) => {
|
|
19
19
|
try {
|
|
20
20
|
const page = await v3.context.awaitActivePage();
|
|
21
|
-
const processed = processCoordinates(coordinates[0], coordinates[1], provider, v3);
|
|
21
|
+
const processed = await processCoordinates(coordinates[0], coordinates[1], provider, v3);
|
|
22
22
|
v3.logger({
|
|
23
23
|
category: "agent",
|
|
24
24
|
message: `Agent calling tool: clickAndHold`,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"clickAndHold.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/clickAndHold.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAI,CAAC;IACH,WAAW,EAAE,oDAAoD;IACjE,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CAAC,kDAAkD,CAAC;QAC/D,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,EAAE;QACrD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,
|
|
1
|
+
{"version":3,"file":"clickAndHold.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/clickAndHold.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAI,CAAC;IACH,WAAW,EAAE,oDAAoD;IACjE,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CAAC,kDAAkD,CAAC;QAC/D,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,EAAE;QACrD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,MAAM,kBAAkB,CACxC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,kCAAkC;gBAC3C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;4BACR,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YAEpD,sFAAsF;YACtF,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACpC,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACrD,CAAC;YAEF,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,cAAc;wBACtB,SAAS,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;qBAC9B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,+BAA+B,KAAK,CAAC,OAAO,EAAE;aACtD,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\n\nexport const clickAndHoldTool = (v3: V3, provider?: string) =>\n tool({\n description: \"Click and hold on an element using its coordinates\",\n inputSchema: z.object({\n describe: z\n .string()\n .describe(\n \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n ),\n duration: z\n .number()\n .describe(\"The duration to hold the element in milliseconds\"),\n coordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to click on\"),\n }),\n execute: async ({ describe, coordinates, duration }) => {\n try {\n const page = await v3.context.awaitActivePage();\n const processed = await processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: clickAndHold`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({\n describe,\n duration,\n }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n\n // Use dragAndDrop from same point to same point with delay to simulate click and hold\n const [xpath] = await page.dragAndDrop(\n processed.x,\n processed.y,\n processed.x,\n processed.y,\n { delay: duration, returnXpath: shouldCollectXpath },\n );\n\n // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n const action: Action = {\n selector: normalizedXpath,\n description: describe,\n method: \"clickAndHold\",\n arguments: [String(duration)],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return { success: true, describe };\n } catch (error) {\n return {\n success: false,\n error: `Error clicking and holding: ${error.message}`,\n };\n }\n },\n });\n"]}
|
|
@@ -17,8 +17,8 @@ export const dragAndDropTool = (v3, provider) => tool({
|
|
|
17
17
|
execute: async ({ describe, startCoordinates, endCoordinates, }) => {
|
|
18
18
|
try {
|
|
19
19
|
const page = await v3.context.awaitActivePage();
|
|
20
|
-
const processedStart = processCoordinates(startCoordinates[0], startCoordinates[1], provider, v3);
|
|
21
|
-
const processedEnd = processCoordinates(endCoordinates[0], endCoordinates[1], provider, v3);
|
|
20
|
+
const processedStart = await processCoordinates(startCoordinates[0], startCoordinates[1], provider, v3);
|
|
21
|
+
const processedEnd = await processCoordinates(endCoordinates[0], endCoordinates[1], provider, v3);
|
|
22
22
|
v3.logger({
|
|
23
23
|
category: "agent",
|
|
24
24
|
message: `Agent calling tool: dragAndDrop`,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dragAndDrop.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/dragAndDrop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC3D,IAAI,CAAC;IACH,WAAW,EACT,kNAAkN;IACpN,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACtE,gBAAgB,EAAE,CAAC;aAChB,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,wDAAwD,CAAC;QACrE,cAAc,EAAE,CAAC;aACd,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oDAAoD,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,gBAAgB,EAChB,cAAc,GACf,EAAkC,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,kBAAkB,
|
|
1
|
+
{"version":3,"file":"dragAndDrop.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/dragAndDrop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC3D,IAAI,CAAC;IACH,WAAW,EACT,kNAAkN;IACpN,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACtE,gBAAgB,EAAE,CAAC;aAChB,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,wDAAwD,CAAC;QACrE,cAAc,EAAE,CAAC;aACd,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oDAAoD,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,gBAAgB,EAChB,cAAc,GACf,EAAkC,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAC7C,gBAAgB,CAAC,CAAC,CAAC,EACnB,gBAAgB,CAAC,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;YACF,MAAM,YAAY,GAAG,MAAM,kBAAkB,CAC3C,cAAc,CAAC,CAAC,CAAC,EACjB,cAAc,CAAC,CAAC,CAAC,EACjB,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,iCAAiC;gBAC1C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACjD,cAAc,CAAC,CAAC,EAChB,cAAc,CAAC,CAAC,EAChB,YAAY,CAAC,CAAC,EACd,YAAY,CAAC,CAAC,EACd,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,cAAc,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;gBAC1C,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,cAAc;wBACxB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,aAAa;wBACrB,SAAS,EAAE,CAAC,YAAY,CAAC;qBAC1B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAmB,KAAK,CAAC,OAAO,EAAE;aAC1C,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBAC1B,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n DragAndDropToolResult,\n ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const dragAndDropTool = (v3: V3, provider?: string) =>\n tool({\n description:\n \"Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n inputSchema: z.object({\n describe: z.string().describe(\"Describe the element to drag and drop\"),\n startCoordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to start the drag and drop from\"),\n endCoordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to end the drag and drop at\"),\n }),\n execute: async ({\n describe,\n startCoordinates,\n endCoordinates,\n }): Promise<DragAndDropToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const processedStart = await processCoordinates(\n startCoordinates[0],\n startCoordinates[1],\n provider,\n v3,\n );\n const processedEnd = await processCoordinates(\n endCoordinates[0],\n endCoordinates[1],\n provider,\n v3,\n );\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: dragAndDrop`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({\n describe,\n }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const [fromXpath, toXpath] = await page.dragAndDrop(\n processedStart.x,\n processedStart.y,\n processedEnd.x,\n processedEnd.y,\n { returnXpath: shouldCollectXpath },\n );\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedFrom = ensureXPath(fromXpath);\n const normalizedTo = ensureXPath(toXpath);\n if (normalizedFrom && normalizedTo) {\n const action: Action = {\n selector: normalizedFrom,\n description: describe,\n method: \"dragAndDrop\",\n arguments: [normalizedTo],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return {\n success: true,\n describe,\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error dragging: ${error.message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false || result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n describe: result.describe,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n"]}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { tool } from "ai";
|
|
2
2
|
import { z } from "zod";
|
|
3
|
-
import { processCoordinates } from "../utils/coordinateNormalization.js";
|
|
3
|
+
import { isGoogleProvider, processCoordinates, } from "../utils/coordinateNormalization.js";
|
|
4
4
|
import { ensureXPath } from "../utils/xpath.js";
|
|
5
5
|
import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
|
|
6
6
|
import { substituteVariables } from "../utils/variables.js";
|
|
@@ -46,19 +46,22 @@ MANDATORY USE CASES (always use fillFormVision for these):
|
|
|
46
46
|
execute: async ({ fields }) => {
|
|
47
47
|
try {
|
|
48
48
|
const page = await v3.context.awaitActivePage();
|
|
49
|
+
const viewport = isGoogleProvider(provider)
|
|
50
|
+
? await v3.resolveViewport()
|
|
51
|
+
: undefined;
|
|
49
52
|
// Process coordinates per field. Keep the original `value` (with any
|
|
50
53
|
// `%variableName%` tokens) so substituted secrets never leak through
|
|
51
54
|
// the tool result, replay cache, returned AgentResult.actions, or
|
|
52
55
|
// anything that logs the action. Variables are substituted only at
|
|
53
56
|
// typing time below.
|
|
54
|
-
const safeFields = fields.map((field) => {
|
|
55
|
-
const processed = processCoordinates(field.coordinates.x, field.coordinates.y, provider, v3);
|
|
57
|
+
const safeFields = await Promise.all(fields.map(async (field) => {
|
|
58
|
+
const processed = await processCoordinates(field.coordinates.x, field.coordinates.y, provider, v3, viewport);
|
|
56
59
|
return {
|
|
57
60
|
action: field.action,
|
|
58
61
|
value: field.value,
|
|
59
62
|
coordinates: { x: processed.x, y: processed.y },
|
|
60
63
|
};
|
|
61
|
-
});
|
|
64
|
+
}));
|
|
62
65
|
v3.logger({
|
|
63
66
|
category: "agent",
|
|
64
67
|
message: `Agent calling tool: fillFormVision`,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAI,CAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,CAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAEhD,qEAAqE;gBACrE,qEAAqE;gBACrE,kEAAkE;gBAClE,mEAAmE;gBACnE,qBAAqB;gBACrB,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBACtC,MAAM,SAAS,GAAG,kBAAkB,CAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;oBACF,OAAO;wBACL,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;oBAC/B,uEAAuE;oBACvE,uEAAuE;oBACvE,yEAAyE;oBACzE,wEAAwE;oBACxE,qEAAqE;oBACrE,MAAM,KAAK,GAAG,EAAE,CAAC,SAAS;wBACxB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE;4BACvD,WAAW,EAAE,kBAAkB;yBAChC,CAAC;wBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE;4BACzD,WAAW,EAAE,kBAAkB;yBAChC,CAAC,CAAC;oBACP,oEAAoE;oBACpE,iCAAiC;oBACjC,MAAM,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;oBAE7D,wEAAwE;oBACxE,sEAAsE;oBACtE,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;6BACzB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,UAAU;oBAC/B,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAuB,KAAK,CAAC,OAAO,EAAE;iBAC9C,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gCACvB,KAAK,EAAE,MAAM,CAAC,KAAK;6BACpB,CAAC;yBACH;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n FillFormVisionToolResult,\n ModelOutputContentItem,\n Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n v3: V3,\n provider?: string,\n variables?: Variables,\n) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const valueDescription = hasVariables\n ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n : \"Text to type into the target field\";\n\n return tool({\n description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n inputSchema: z.object({\n fields: z\n .array(\n z.object({\n action: z\n .string()\n .describe(\n \"Description of the typing action, e.g. 'type foo into the bar field'\",\n ),\n value: z.string().describe(valueDescription),\n coordinates: z\n .object({\n x: z.number(),\n y: z.number(),\n })\n .describe(\"Coordinates of the target field\"),\n }),\n )\n .min(2, \"Provide at least two fields to fill\"),\n }),\n execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n\n // Process coordinates per field. Keep the original `value` (with any\n // `%variableName%` tokens) so substituted secrets never leak through\n // the tool result, replay cache, returned AgentResult.actions, or\n // anything that logs the action. Variables are substituted only at\n // typing time below.\n const safeFields = fields.map((field) => {\n const processed = processCoordinates(\n field.coordinates.x,\n field.coordinates.y,\n provider,\n v3,\n );\n return {\n action: field.action,\n value: field.value,\n coordinates: { x: processed.x, y: processed.y },\n };\n });\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: fillFormVision`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ fields }), // Don't log substituted values\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const actions: Action[] = [];\n\n for (const field of safeFields) {\n // Focus the field, only requesting XPath when caching is enabled. On a\n // touch session this must be a trusted tap — a synthesized mouse click\n // never registers on touch-gated mobile layouts, so the field would stay\n // unfocused and the value go nowhere. The recorded step stays a \"type\",\n // whose replay focuses via the selector rather than a pointer event.\n const xpath = v3.usesTouch\n ? await page.tap(field.coordinates.x, field.coordinates.y, {\n returnXpath: shouldCollectXpath,\n })\n : await page.click(field.coordinates.x, field.coordinates.y, {\n returnXpath: shouldCollectXpath,\n });\n // Substitute variables only at the moment of typing so the resolved\n // value never leaves this scope.\n await page.type(substituteVariables(field.value, variables));\n\n // Build Action with XPath for deterministic replay (only when caching).\n // Store the placeholder value so cache entries don't contain secrets.\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n actions.push({\n selector: normalizedXpath,\n description: field.action,\n method: \"type\",\n arguments: [field.value],\n });\n }\n }\n\n // Small delay between fields\n await new Promise((resolve) => setTimeout(resolve, 100));\n }\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n if (shouldCollectXpath && actions.length > 0) {\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: `Fill ${fields.length} form fields`,\n actions,\n actionDescription: `Fill ${fields.length} form fields`,\n });\n }\n\n return {\n success: true,\n playwrightArguments: safeFields,\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error filling form: ${error.message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false || result.error !== undefined) {\n return {\n type: \"content\",\n value: [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n error: result.error,\n }),\n },\n ],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n fieldsCount: result.playwrightArguments?.length ?? 0,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n};\n"]}
|
|
1
|
+
{"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EACL,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAI,CAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,CAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC;oBACzC,CAAC,CAAC,MAAM,EAAE,CAAC,eAAe,EAAE;oBAC5B,CAAC,CAAC,SAAS,CAAC;gBAEd,qEAAqE;gBACrE,qEAAqE;gBACrE,kEAAkE;gBAClE,mEAAmE;gBACnE,qBAAqB;gBACrB,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAClC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;oBACzB,MAAM,SAAS,GAAG,MAAM,kBAAkB,CACxC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,EACF,QAAQ,CACT,CAAC;oBACF,OAAO;wBACL,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CACH,CAAC;gBAEF,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;oBAC/B,uEAAuE;oBACvE,uEAAuE;oBACvE,yEAAyE;oBACzE,wEAAwE;oBACxE,qEAAqE;oBACrE,MAAM,KAAK,GAAG,EAAE,CAAC,SAAS;wBACxB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE;4BACvD,WAAW,EAAE,kBAAkB;yBAChC,CAAC;wBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE;4BACzD,WAAW,EAAE,kBAAkB;yBAChC,CAAC,CAAC;oBACP,oEAAoE;oBACpE,iCAAiC;oBACjC,MAAM,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;oBAE7D,wEAAwE;oBACxE,sEAAsE;oBACtE,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;6BACzB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,UAAU;oBAC/B,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAuB,KAAK,CAAC,OAAO,EAAE;iBAC9C,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gCACvB,KAAK,EAAE,MAAM,CAAC,KAAK;6BACpB,CAAC;yBACH;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n FillFormVisionToolResult,\n ModelOutputContentItem,\n Variables,\n} from \"../../types/public/agent.js\";\nimport {\n isGoogleProvider,\n processCoordinates,\n} from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n v3: V3,\n provider?: string,\n variables?: Variables,\n) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const valueDescription = hasVariables\n ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n : \"Text to type into the target field\";\n\n return tool({\n description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n inputSchema: z.object({\n fields: z\n .array(\n z.object({\n action: z\n .string()\n .describe(\n \"Description of the typing action, e.g. 'type foo into the bar field'\",\n ),\n value: z.string().describe(valueDescription),\n coordinates: z\n .object({\n x: z.number(),\n y: z.number(),\n })\n .describe(\"Coordinates of the target field\"),\n }),\n )\n .min(2, \"Provide at least two fields to fill\"),\n }),\n execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const viewport = isGoogleProvider(provider)\n ? await v3.resolveViewport()\n : undefined;\n\n // Process coordinates per field. Keep the original `value` (with any\n // `%variableName%` tokens) so substituted secrets never leak through\n // the tool result, replay cache, returned AgentResult.actions, or\n // anything that logs the action. Variables are substituted only at\n // typing time below.\n const safeFields = await Promise.all(\n fields.map(async (field) => {\n const processed = await processCoordinates(\n field.coordinates.x,\n field.coordinates.y,\n provider,\n v3,\n viewport,\n );\n return {\n action: field.action,\n value: field.value,\n coordinates: { x: processed.x, y: processed.y },\n };\n }),\n );\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: fillFormVision`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ fields }), // Don't log substituted values\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const actions: Action[] = [];\n\n for (const field of safeFields) {\n // Focus the field, only requesting XPath when caching is enabled. On a\n // touch session this must be a trusted tap — a synthesized mouse click\n // never registers on touch-gated mobile layouts, so the field would stay\n // unfocused and the value go nowhere. The recorded step stays a \"type\",\n // whose replay focuses via the selector rather than a pointer event.\n const xpath = v3.usesTouch\n ? await page.tap(field.coordinates.x, field.coordinates.y, {\n returnXpath: shouldCollectXpath,\n })\n : await page.click(field.coordinates.x, field.coordinates.y, {\n returnXpath: shouldCollectXpath,\n });\n // Substitute variables only at the moment of typing so the resolved\n // value never leaves this scope.\n await page.type(substituteVariables(field.value, variables));\n\n // Build Action with XPath for deterministic replay (only when caching).\n // Store the placeholder value so cache entries don't contain secrets.\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n actions.push({\n selector: normalizedXpath,\n description: field.action,\n method: \"type\",\n arguments: [field.value],\n });\n }\n }\n\n // Small delay between fields\n await new Promise((resolve) => setTimeout(resolve, 100));\n }\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n if (shouldCollectXpath && actions.length > 0) {\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: `Fill ${fields.length} form fields`,\n actions,\n actionDescription: `Fill ${fields.length} form fields`,\n });\n }\n\n return {\n success: true,\n playwrightArguments: safeFields,\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error filling form: ${error.message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false || result.error !== undefined) {\n return {\n type: \"content\",\n value: [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n error: result.error,\n }),\n },\n ],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n fieldsCount: result.playwrightArguments?.length ?? 0,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n};\n"]}
|
|
@@ -81,7 +81,7 @@ export const scrollVisionTool = (v3, provider) => tool({
|
|
|
81
81
|
let cx;
|
|
82
82
|
let cy;
|
|
83
83
|
if (coordinates) {
|
|
84
|
-
const processed = processCoordinates(coordinates[0], coordinates[1], provider, v3);
|
|
84
|
+
const processed = await processCoordinates(coordinates[0], coordinates[1], provider, v3);
|
|
85
85
|
cx = processed.x;
|
|
86
86
|
cy = processed.y;
|
|
87
87
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scroll.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/scroll.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,EAAE,CACnC,IAAI,CAAC;IACH,WAAW,EACT,iJAAiJ;IACnJ,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,UAAU,GAAG,EAAE,GAChB,EAA6B,EAAE;QAC9B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;oBAChD,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,YAAY,UAAU,KAAK,SAAS,KAAK,cAAc,KAAK;YACrE,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;aACtC;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEL;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAI,CAAC;IACH,WAAW,EAAE,6SAA6S;IAC1T,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,EAAE;aACV,QAAQ,CACP,4GAA4G,CAC7G;QACH,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,WAAW,EACX,UAAU,GAAG,EAAE,GAChB,EAAmC,EAAE;QACpC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,iEAAiE;QACjE,IAAI,EAAU,CAAC;QACf,IAAI,EAAU,CAAC;QACf,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YACF,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;YACjB,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACvB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,SAAS;wBACT,WAAW;wBACX,UAAU;wBACV,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;qBACtB,CAAC;oBACF,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEnE,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,WAAW;gBAClB,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,QAAQ,EAAE,KAAK,EAAE,GAAG;gBAC1D,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,EAAE;YAC1C,cAAc,EAAE,cAAc;YAC9B,gBAAgB;SACjB,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;iBACtC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type {\n ScrollToolResult,\n ScrollVisionToolResult,\n ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\n/**\n * Simple scroll tool for DOM mode (non-grounding models).\n * No coordinates - scrolls from viewport center.\n */\nexport const scrollTool = (v3: V3) =>\n tool({\n description:\n \"Scroll the page up or down by a percentage of the viewport height. Default is 80%, and what should be typically used for general page scrolling\",\n inputSchema: z.object({\n direction: z.enum([\"up\", \"down\"]),\n percentage: z.number().min(1).max(200).optional(),\n }),\n execute: async ({\n direction,\n percentage = 80,\n }): Promise<ScrollToolResult> => {\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: scroll`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ direction, percentage }),\n type: \"object\",\n },\n },\n });\n\n const page = await v3.context.awaitActivePage();\n\n const { w, h } = await page.mainFrame().evaluate<{\n w: number;\n h: number;\n }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n const scrollDistance = Math.round((h * percentage) / 100);\n const cx = Math.floor(w / 2);\n const cy = Math.floor(h / 2);\n const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n await page.scroll(cx, cy, 0, deltaY);\n\n v3.recordAgentReplayStep({\n type: \"scroll\",\n deltaX: 0,\n deltaY,\n anchor: { x: cx, y: cy },\n });\n\n return {\n success: true,\n message: `Scrolled ${percentage}% ${direction} (${scrollDistance}px)`,\n scrolledPixels: scrollDistance,\n };\n },\n toModelOutput: (result) => {\n if (result.success === false || result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n return {\n type: \"json\",\n value: {\n success: result.success,\n message: result.message,\n scrolledPixels: result.scrolledPixels,\n },\n };\n },\n });\n\n/**\n * Scroll tool for hybrid mode (grounding models).\n * Supports optional coordinates for scrolling within nested scrollable elements.\n */\nexport const scrollVisionTool = (v3: V3, provider?: string) =>\n tool({\n description: `Scroll the page up or down. For general page scrolling, no coordinates needed. Only provide coordinates when scrolling inside a nested scrollable element (e.g., a dropdown menu, modal with overflow, or scrollable sidebar). Default is 80%, and what should be typically used for general page scrolling`,\n inputSchema: z.object({\n direction: z.enum([\"up\", \"down\"]),\n coordinates: z\n .array(z.number())\n .optional()\n .describe(\n \"Only use coordinates for scrolling inside a nested scrollable element - provide (x, y) within that element\",\n ),\n percentage: z.number().min(1).max(200).optional(),\n }),\n execute: async ({\n direction,\n coordinates,\n percentage = 80,\n }): Promise<ScrollVisionToolResult> => {\n const page = await v3.context.awaitActivePage();\n\n const { w, h } = await page.mainFrame().evaluate<{\n w: number;\n h: number;\n }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n // Process coordinates if provided, otherwise use viewport center\n let cx: number;\n let cy: number;\n if (coordinates) {\n const processed = processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n cx = processed.x;\n cy = processed.y;\n } else {\n cx = Math.floor(w / 2);\n cy = Math.floor(h / 2);\n }\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: scroll`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({\n direction,\n coordinates,\n percentage,\n processed: { cx, cy },\n }),\n type: \"object\",\n },\n },\n });\n\n const scrollDistance = Math.round((h * percentage) / 100);\n const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n await page.scroll(cx, cy, 0, deltaY);\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n v3.recordAgentReplayStep({\n type: \"scroll\",\n deltaX: 0,\n deltaY,\n anchor: { x: cx, y: cy },\n });\n\n return {\n success: true,\n message: coordinates\n ? `Scrolled ${percentage}% ${direction} at (${cx}, ${cy})`\n : `Scrolled ${percentage}% ${direction}`,\n scrolledPixels: scrollDistance,\n screenshotBase64,\n };\n },\n toModelOutput: (result) => {\n if (result.success === false || result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n message: result.message,\n scrolledPixels: result.scrolledPixels,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n"]}
|
|
1
|
+
{"version":3,"file":"scroll.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/scroll.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,EAAE,CACnC,IAAI,CAAC;IACH,WAAW,EACT,iJAAiJ;IACnJ,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,UAAU,GAAG,EAAE,GAChB,EAA6B,EAAE;QAC9B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;oBAChD,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,YAAY,UAAU,KAAK,SAAS,KAAK,cAAc,KAAK;YACrE,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;aACtC;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEL;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAI,CAAC;IACH,WAAW,EAAE,6SAA6S;IAC1T,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,EAAE;aACV,QAAQ,CACP,4GAA4G,CAC7G;QACH,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,WAAW,EACX,UAAU,GAAG,EAAE,GAChB,EAAmC,EAAE;QACpC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,iEAAiE;QACjE,IAAI,EAAU,CAAC;QACf,IAAI,EAAU,CAAC;QACf,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,SAAS,GAAG,MAAM,kBAAkB,CACxC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YACF,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;YACjB,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACvB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,SAAS;wBACT,WAAW;wBACX,UAAU;wBACV,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;qBACtB,CAAC;oBACF,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEnE,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,WAAW;gBAClB,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,QAAQ,EAAE,KAAK,EAAE,GAAG;gBAC1D,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,EAAE;YAC1C,cAAc,EAAE,cAAc;YAC9B,gBAAgB;SACjB,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;iBACtC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type {\n ScrollToolResult,\n ScrollVisionToolResult,\n ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\n/**\n * Simple scroll tool for DOM mode (non-grounding models).\n * No coordinates - scrolls from viewport center.\n */\nexport const scrollTool = (v3: V3) =>\n tool({\n description:\n \"Scroll the page up or down by a percentage of the viewport height. Default is 80%, and what should be typically used for general page scrolling\",\n inputSchema: z.object({\n direction: z.enum([\"up\", \"down\"]),\n percentage: z.number().min(1).max(200).optional(),\n }),\n execute: async ({\n direction,\n percentage = 80,\n }): Promise<ScrollToolResult> => {\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: scroll`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ direction, percentage }),\n type: \"object\",\n },\n },\n });\n\n const page = await v3.context.awaitActivePage();\n\n const { w, h } = await page.mainFrame().evaluate<{\n w: number;\n h: number;\n }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n const scrollDistance = Math.round((h * percentage) / 100);\n const cx = Math.floor(w / 2);\n const cy = Math.floor(h / 2);\n const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n await page.scroll(cx, cy, 0, deltaY);\n\n v3.recordAgentReplayStep({\n type: \"scroll\",\n deltaX: 0,\n deltaY,\n anchor: { x: cx, y: cy },\n });\n\n return {\n success: true,\n message: `Scrolled ${percentage}% ${direction} (${scrollDistance}px)`,\n scrolledPixels: scrollDistance,\n };\n },\n toModelOutput: (result) => {\n if (result.success === false || result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n return {\n type: \"json\",\n value: {\n success: result.success,\n message: result.message,\n scrolledPixels: result.scrolledPixels,\n },\n };\n },\n });\n\n/**\n * Scroll tool for hybrid mode (grounding models).\n * Supports optional coordinates for scrolling within nested scrollable elements.\n */\nexport const scrollVisionTool = (v3: V3, provider?: string) =>\n tool({\n description: `Scroll the page up or down. For general page scrolling, no coordinates needed. Only provide coordinates when scrolling inside a nested scrollable element (e.g., a dropdown menu, modal with overflow, or scrollable sidebar). Default is 80%, and what should be typically used for general page scrolling`,\n inputSchema: z.object({\n direction: z.enum([\"up\", \"down\"]),\n coordinates: z\n .array(z.number())\n .optional()\n .describe(\n \"Only use coordinates for scrolling inside a nested scrollable element - provide (x, y) within that element\",\n ),\n percentage: z.number().min(1).max(200).optional(),\n }),\n execute: async ({\n direction,\n coordinates,\n percentage = 80,\n }): Promise<ScrollVisionToolResult> => {\n const page = await v3.context.awaitActivePage();\n\n const { w, h } = await page.mainFrame().evaluate<{\n w: number;\n h: number;\n }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n // Process coordinates if provided, otherwise use viewport center\n let cx: number;\n let cy: number;\n if (coordinates) {\n const processed = await processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n cx = processed.x;\n cy = processed.y;\n } else {\n cx = Math.floor(w / 2);\n cy = Math.floor(h / 2);\n }\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: scroll`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({\n direction,\n coordinates,\n percentage,\n processed: { cx, cy },\n }),\n type: \"object\",\n },\n },\n });\n\n const scrollDistance = Math.round((h * percentage) / 100);\n const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n await page.scroll(cx, cy, 0, deltaY);\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n v3.recordAgentReplayStep({\n type: \"scroll\",\n deltaX: 0,\n deltaY,\n anchor: { x: cx, y: cy },\n });\n\n return {\n success: true,\n message: coordinates\n ? `Scrolled ${percentage}% ${direction} at (${cx}, ${cy})`\n : `Scrolled ${percentage}% ${direction}`,\n scrolledPixels: scrollDistance,\n screenshotBase64,\n };\n },\n toModelOutput: (result) => {\n if (result.success === false || result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n message: result.message,\n scrolledPixels: result.scrolledPixels,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n"]}
|
|
@@ -23,7 +23,7 @@ export const typeTool = (v3, provider, variables) => {
|
|
|
23
23
|
execute: async ({ describe, coordinates, text, }) => {
|
|
24
24
|
try {
|
|
25
25
|
const page = await v3.context.awaitActivePage();
|
|
26
|
-
const processed = processCoordinates(coordinates[0], coordinates[1], provider, v3);
|
|
26
|
+
const processed = await processCoordinates(coordinates[0], coordinates[1], provider, v3);
|
|
27
27
|
// Substitute any %variableName% tokens in the text
|
|
28
28
|
const actualText = substituteVariables(text, variables);
|
|
29
29
|
v3.logger({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"type.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,SAAqB,EAAE,EAAE;IAC3E,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,YAAY;QAClC,CAAC,CAAC,oGAAoG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACzI,CAAC,CAAC,mCAAmC,CAAC;IAExC,OAAO,IAAI,CAAC;QACV,WAAW,EACT,2QAA2Q;QAC7Q,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,QAAQ,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,CACP,4HAA4H,CAC7H;YACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;YAC1C,WAAW,EAAE,CAAC;iBACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,QAAQ,CAAC,iDAAiD,CAAC;SAC/D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,WAAW,EACX,IAAI,GACL,EAA2B,EAAE;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,kBAAkB,
|
|
1
|
+
{"version":3,"file":"type.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,SAAqB,EAAE,EAAE;IAC3E,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,YAAY;QAClC,CAAC,CAAC,oGAAoG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACzI,CAAC,CAAC,mCAAmC,CAAC;IAExC,OAAO,IAAI,CAAC;QACV,WAAW,EACT,2QAA2Q;QAC7Q,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,QAAQ,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,CACP,4HAA4H,CAC7H;YACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;YAC1C,WAAW,EAAE,CAAC;iBACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,QAAQ,CAAC,iDAAiD,CAAC;SAC/D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,WAAW,EACX,IAAI,GACL,EAA2B,EAAE;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,MAAM,kBAAkB,CACxC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;gBAEF,mDAAmD;gBACnD,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAExD,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,0BAA0B;oBACnC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;4BACzC,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,yEAAyE;gBACzE,0EAA0E;gBAC1E,sEAAsE;gBACtE,2EAA2E;gBAC3E,wDAAwD;gBACxD,MAAM,KAAK,GAAG,EAAE,CAAC,SAAS;oBACxB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;wBACvC,WAAW,EAAE,kBAAkB;qBAChC,CAAC;oBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;wBACzC,WAAW,EAAE,kBAAkB;qBAChC,CAAC,CAAC;gBAEP,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE5B,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;gBAE9D,0FAA0F;gBAC1F,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;oBAC3C,IAAI,eAAe,EAAE,CAAC;wBACpB,MAAM,MAAM,GAAW;4BACrB,QAAQ,EAAE,eAAe;4BACzB,WAAW,EAAE,QAAQ;4BACrB,MAAM,EAAE,MAAM;4BACd,SAAS,EAAE,CAAC,IAAI,CAAC;yBAClB,CAAC;wBACF,EAAE,CAAC,qBAAqB,CAAC;4BACvB,IAAI,EAAE,KAAK;4BACX,WAAW,EAAE,QAAQ;4BACrB,OAAO,EAAE,CAAC,MAAM,CAAC;4BACjB,iBAAiB,EAAE,QAAQ;yBAC5B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,QAAQ;oBACR,IAAI,EAAE,8FAA8F;oBACpG,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,iBAAiB,KAAK,CAAC,OAAO,EAAE;iBACxC,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;iBACxD,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n TypeToolResult,\n ModelOutputContentItem,\n Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const typeTool = (v3: V3, provider?: string, variables?: Variables) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const textDescription = hasVariables\n ? `The text to type into the element. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n : \"The text to type into the element\";\n\n return tool({\n description:\n \"Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n inputSchema: z.object({\n describe: z\n .string()\n .describe(\n \"Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description\",\n ),\n text: z.string().describe(textDescription),\n coordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to type into the element\"),\n }),\n execute: async ({\n describe,\n coordinates,\n text,\n }): Promise<TypeToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const processed = await processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n\n // Substitute any %variableName% tokens in the text\n const actualText = substituteVariables(text, variables);\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: type`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ describe, text }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n // Focus the field the way the session's input actually works: on a touch\n // session a synthesized mouse click never registers on touch-gated mobile\n // layouts, so the field would stay unfocused and the text go nowhere.\n // Only the actuation changes — the recorded step is a \"type\", whose replay\n // focuses via the selector rather than a pointer event.\n const xpath = v3.usesTouch\n ? await page.tap(processed.x, processed.y, {\n returnXpath: shouldCollectXpath,\n })\n : await page.click(processed.x, processed.y, {\n returnXpath: shouldCollectXpath,\n });\n\n await page.type(actualText);\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n const action: Action = {\n selector: normalizedXpath,\n description: describe,\n method: \"type\",\n arguments: [text],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return {\n success: true,\n describe,\n text, // Return original text (with %variableName% tokens) to avoid exposing sensitive values to LLM\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error typing: ${error.message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false || result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n describe: result.describe,\n text: result.text,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n};\n"]}
|
|
@@ -7,7 +7,10 @@ export declare function normalizeGoogleCoordinates(x: number, y: number, viewpor
|
|
|
7
7
|
x: number;
|
|
8
8
|
y: number;
|
|
9
9
|
};
|
|
10
|
-
export declare function processCoordinates(x: number, y: number, provider?: string, v3?: V3
|
|
10
|
+
export declare function processCoordinates(x: number, y: number, provider?: string, v3?: V3, resolvedViewport?: {
|
|
11
|
+
width: number;
|
|
12
|
+
height: number;
|
|
13
|
+
}): Promise<{
|
|
11
14
|
x: number;
|
|
12
15
|
y: number;
|
|
13
|
-
}
|
|
16
|
+
}>;
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
// Default viewport for Browserbase managed fingerprinting mode
|
|
2
|
-
const STEALTH_VIEWPORT = { width: 1288, height: 711 };
|
|
3
1
|
export function isGoogleProvider(provider) {
|
|
4
2
|
if (!provider)
|
|
5
3
|
return false;
|
|
@@ -15,11 +13,11 @@ export function normalizeGoogleCoordinates(x, y, viewport) {
|
|
|
15
13
|
y: Math.floor((clampedY / 1000) * viewport.height),
|
|
16
14
|
};
|
|
17
15
|
}
|
|
18
|
-
export function processCoordinates(x, y, provider, v3) {
|
|
19
|
-
if (isGoogleProvider(provider)
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
16
|
+
export async function processCoordinates(x, y, provider, v3, resolvedViewport) {
|
|
17
|
+
if (isGoogleProvider(provider)) {
|
|
18
|
+
const viewport = resolvedViewport ?? (v3 ? await v3.resolveViewport() : undefined);
|
|
19
|
+
if (viewport)
|
|
20
|
+
return normalizeGoogleCoordinates(x, y, viewport);
|
|
23
21
|
}
|
|
24
22
|
return { x, y };
|
|
25
23
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"coordinateNormalization.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/coordinateNormalization.ts"],"names":[],"mappings":"AAEA
|
|
1
|
+
{"version":3,"file":"coordinateNormalization.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/coordinateNormalization.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,gBAAgB,CAAC,QAAiB;IAChD,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5B,OAAO,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AACnD,CAAC;AAED,qEAAqE;AACrE,kCAAkC;AAClC,MAAM,UAAU,0BAA0B,CACxC,CAAS,EACT,CAAS,EACT,QAA2C;IAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO;QACL,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC;QACjD,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;KACnD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,CAAS,EACT,CAAS,EACT,QAAiB,EACjB,EAAO,EACP,gBAAoD;IAEpD,IAAI,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/B,MAAM,QAAQ,GACZ,gBAAgB,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACpE,IAAI,QAAQ;YAAE,OAAO,0BAA0B,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;AAClB,CAAC","sourcesContent":["import type { V3 } from \"../../v3.js\";\n\nexport function isGoogleProvider(provider?: string): boolean {\n if (!provider) return false;\n return provider.toLowerCase().includes(\"google\");\n}\n\n// Google returns coordinates in a 0-1000 range, we need to normalize\n// them to the viewport dimensions\nexport function normalizeGoogleCoordinates(\n x: number,\n y: number,\n viewport: { width: number; height: number },\n): { x: number; y: number } {\n const clampedX = Math.min(999, Math.max(0, x));\n const clampedY = Math.min(999, Math.max(0, y));\n return {\n x: Math.floor((clampedX / 1000) * viewport.width),\n y: Math.floor((clampedY / 1000) * viewport.height),\n };\n}\n\nexport async function processCoordinates(\n x: number,\n y: number,\n provider?: string,\n v3?: V3,\n resolvedViewport?: { width: number; height: number },\n): Promise<{ x: number; y: number }> {\n if (isGoogleProvider(provider)) {\n const viewport =\n resolvedViewport ?? (v3 ? await v3.resolveViewport() : undefined);\n if (viewport) return normalizeGoogleCoordinates(x, y, viewport);\n }\n return { x, y };\n}\n"]}
|
|
@@ -599,26 +599,8 @@ export class V3CuaAgentHandler {
|
|
|
599
599
|
});
|
|
600
600
|
}
|
|
601
601
|
async updateClientViewport() {
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
// Browserbase managed fingerprinting uses a fixed 1288x711 fallback.
|
|
605
|
-
if (this.agentClient instanceof GoogleCUAClient) {
|
|
606
|
-
const dims = this.v3.isVerified
|
|
607
|
-
? { width: 1288, height: 711 }
|
|
608
|
-
: this.v3.configuredViewport;
|
|
609
|
-
this.agentClient.setViewport(dims.width, dims.height);
|
|
610
|
-
}
|
|
611
|
-
else {
|
|
612
|
-
// For other clients, use actual window dimensions
|
|
613
|
-
const page = await this.v3.context.awaitActivePage();
|
|
614
|
-
const { w, h } = await page.mainFrame().evaluate("({ w: window.innerWidth, h: window.innerHeight })");
|
|
615
|
-
if (w && h)
|
|
616
|
-
this.agentClient.setViewport(w, h);
|
|
617
|
-
}
|
|
618
|
-
}
|
|
619
|
-
catch {
|
|
620
|
-
//
|
|
621
|
-
}
|
|
602
|
+
const { width, height } = await this.v3.resolveViewport();
|
|
603
|
+
this.agentClient.setViewport(width, height);
|
|
622
604
|
}
|
|
623
605
|
async updateClientUrl() {
|
|
624
606
|
try {
|