@browserbasehq/orca 3.7.2-preview.0 → 3.7.2-preview.2

package/dist/cjs/lib/inference.js

@@ -244,25 +244,31 @@ async function observe({ instruction, domElements, llmClient, userProvidedInstru
     };
 }
 async function act({ instruction, domElements, llmClient, userProvidedInstructions, logger, logInferenceToFile = false, }) {
+    const actionSchema = zod_1.z.object({
+        elementId: zod_1.z
+            .string()
+            .regex(/^\d+-\d+$/)
+            .describe("the ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'. for example, '0-76' or '16-21'"),
+        description: zod_1.z
+            .string()
+            .describe("a description of the accessible element and its purpose"),
+        method: zod_1.z
+            .enum(
+        // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly
+        Object.values(handlers_js_1.SupportedUnderstudyAction))
+            .describe("the candidate method/action to interact with the element. Select one of the available Understudy interaction methods."),
+        arguments: zod_1.z.array(zod_1.z
+            .string()
+            .describe("the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.")),
+    });
     const actSchema = zod_1.z.object({
-        action: zod_1.z
-            .object({
-            elementId: zod_1.z
-                .string()
-                .regex(/^\d+-\d+$/)
-                .describe("the ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'. for example, '0-76' or '16-21'"),
-            description: zod_1.z
-                .string()
-                .describe("a description of the accessible element and its purpose"),
-            method: zod_1.z
-                .enum(
-            // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly
-            Object.values(handlers_js_1.SupportedUnderstudyAction))
-                .describe("the candidate method/action to interact with the element. Select one of the available Understudy interaction methods."),
-            arguments: zod_1.z.array(zod_1.z
-                .string()
-                .describe("the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.")),
-        })
+        // OpenAI strict mode requires additionalProperties: false on every object.
+        // The AI SDK's Zod 4 converter omits it inside anyOf, so set it in the JSON
+        // schema only; parsing still strips unknown keys. .meta() is Zod 4 only;
+        // the Zod 3 converter already emits it.
+        action: (typeof actionSchema.meta === "function"
+            ? actionSchema.meta({ additionalProperties: false })
+            : actionSchema)
             .nullable()
             .describe("The element to act on. Return null if no element on the page matches the instruction — do NOT fabricate or guess an element, and never emit empty strings or placeholder values."),
         twoStep: zod_1.z.boolean(),

package/dist/cjs/lib/inference.js.map

@@ -1 +1 @@
-{"version":3,"file":"inference.js","sourceRoot":"","sources":["../../../lib/inference.ts"],"names":[],"mappings":";;AAiCA,0BA8NC;AAED,0BAsJC;AAED,kBA6IC;AAtiBD,6BAAwB;AAGxB,4DAAqE;AACrE,2CAQqB;AACrB,iEAAgF;AAKhF,gEAA2E;AAE3E,iEAA+E;AAK/E,SAAS,cAAc,CAAI,OAAmB,EAAE,SAAiB;IAC/D,OAAO,IAAA,8BAAW,EAChB,OAAO,EACP,IAAA,kCAAe,EAAC,YAAY,CAAC,EAC7B,OAAO,SAAS,EAAE,CACnB,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,OAAO,CAA+B,EAC1D,WAAW,EACX,WAAW,EACX,MAAM,EACN,SAAS,EACT,MAAM,EACN,wBAAwB,EACxB,kBAAkB,GAAG,KAAK,EAC1B,UAAU,GAUX;IACC,MAAM,cAAc,GAAG,OAAC,CAAC,MAAM,CAAC;QAC9B,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,oEAAoE,CACrE;QACH,SAAS,EAAE,OAAC;aACT,OAAO,EAAE;aACT,QAAQ,CACP,iHAAiH,CAClH;KACJ,CAAC,CAAC;IAKH,MAAM,gBAAgB,GAAG,SAAS,CAAC,IAAI,KAAK,WAAW,CAAC;IACxD,IAAI,UAAU,IAAI,SAAS,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC7C,MAAM,IAAI,4CAA6B,CACrC,sEAAsE,CACvE,CAAC;IACJ,CAAC;IACD,MAAM,iBAAiB,GAAG,UAAU;QAClC,CAAC,CAAC,yBAAyB,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;QAC1D,CAAC,CAAC,SAAS,CAAC;IAEd,MAAM,mBAAmB,GAAkB;QACzC,IAAA,oCAAwB,EACtB,gBAAgB,EAChB,wBAAwB,EACxB,OAAO,CAAC,iBAAiB,CAAC,CAC3B;QACD,IAAA,kCAAsB,EACpB,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,iBAAiB,CAClB;KACF,CAAC;IAEF,IAAI,eAAe,GAAG,EAAE,CAAC;IACzB,IAAI,oBAAoB,GAAG,EAAE,CAAC;IAC9B,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,iBAAiB,EACjB,cAAc,EACd;YACE,SAAS,EAAE,SAAS;YACpB,QAAQ,EAAE,mBAAmB;SAC9B,CACF,CAAC;QACF,eAAe,GAAG,QAAQ,CAAC;QAC3B,oBAAoB,GAAG,SAAS,CAAC;IACnC,CAAC;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,MAAM,kBAAkB,GAAG,MAAM,cAAc,CAC7C,SAAS,CAAC,oBAAoB,CAAqB;QACjD,OAAO,EAAE;YACP,QAAQ,EAAE,mBAAmB;YAC7B,cAAc,EAAE;gBACd,MAAM;gBACN,IAAI,EAAE,YAAY;aACnB;YACD,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,EACF,SAAS,CACV,CAAC;IACF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,kBAAkB,CAAC;IAExE,IAAI,mBAA2B,CAAC;IAChC,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAA,8CAAuB,EAC1C,iBAAiB,EACjB,kBAAkB,EAClB;YACE,aAAa,EAAE,SAAS;YACxB,WAAW,EAAE,aAAa;SAC3B,CACF,CAAC;QACF,mBAAmB,GAAG,QAAQ,CAAC;QAE/B,IAAA,oCAAa,EAAC,SAAS,EAAE;YACvB,sBAAsB,EAAE,SAAS;YACjC,SAAS,EAAE,oBAAoB;YAC/B,cAAc,EAAE,eAAe;YAC/B,eAAe,EAAE,mBAAmB;YACpC,aAAa,EAAE,YAAY,EAAE,aAAa,IAAI,CAAC;YAC/C,iBAAiB,EAAE,YAAY,EAAE,iBAAiB,IAAI,CAAC;YACvD,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,IAAI,CAAC;YACrD,mBAAmB,EAAE,YAAY,EAAE,mBAAmB,IAAI,CAAC;YAC3D,iBAAiB,EAAE,cAAc,GAAG,gBAAgB;SACrD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,oBAAoB,GAAkB;QAC1C,IAAA,qCAAyB,GAAE;QAC3B,IAAA,+BAAmB,EAAC,WAAW,EAAE,aAAa,CAAC;KAChD,CAAC;IAEF,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAC1B,IAAI,qBAAqB,GAAG,EAAE,CAAC;IAC/B,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,iBAAiB,EACjB,eAAe,EACf;YACE,SAAS,EAAE,UAAU;YACrB,QAAQ,EAAE,oBAAoB;SAC/B,CACF,CAAC;QACF,gBAAgB,GAAG,QAAQ,CAAC;QAC5B,qBAAqB,GAAG,SAAS,CAAC;IACpC,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACrC,MAAM,gBAAgB,GAAG,MAAM,cAAc,CAC3C,SAAS,CAAC,oBAAoB,CAAmB;QAC/C,OAAO,EAAE;YACP,QAAQ,EAAE,oBAAoB;YAC9B,cAAc,EAAE;gBACd,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,cAAc;aACvB;YACD,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,EACF,kBAAkB,CACnB,CAAC;IACF,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEnC,MAAM,EACJ,IAAI,EAAE,EACJ,SAAS,EAAE,yBAAyB,EACpC,QAAQ,EAAE,wBAAwB,GACnC,EACD,KAAK,EAAE,qBAAqB,GAC7B,GAAG,gBAAgB,CAAC;IAErB,IAAI,oBAA4B,CAAC;IACjC,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAA,8CAAuB,EAC1C,iBAAiB,EACjB,mBAAmB,EACnB;YACE,aAAa,EAAE,UAAU;YACzB,SAAS,EAAE,yBAAyB;YACpC,QAAQ,EAAE,wBAAwB;SACnC,CACF,CAAC;QACF,oBAAoB,GAAG,QAAQ,CAAC;QAEhC,IAAA,oCAAa,EAAC,SAAS,EAAE;YACvB,sBAAsB,EAAE,UAAU;YAClC,SAAS,EAAE,qBAAqB;YAChC,cAAc,EAAE,gBAAgB;YAChC,eAAe,EAAE,oBAAoB;YACrC,aAAa,EAAE,qBAAqB,EAAE,aAAa,IAAI,CAAC;YACxD,iBAAiB,EAAE,qBAAqB,EAAE,iBAAiB,IAAI,CAAC;YAChE,gBAAgB,EAAE,qBAAqB,EAAE,gBAAgB,IAAI,CAAC;YAC9D,mBAAmB,EAAE,qBAAqB,EAAE,mBAAmB,IAAI,CAAC;YACpE,iBAAiB,EAAE,eAAe,GAAG,iBAAiB;SACvD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,iBAAiB,GACrB,CAAC,YAAY,EAAE,aAAa,IAAI,CAAC,CAAC;QAClC,CAAC,qBAAqB,EAAE,aAAa,IAAI,CAAC,CAAC,CAAC;IAE9C,MAAM,qBAAqB,GACzB,CAAC,YAAY,EAAE,iBAAiB,IAAI,CAAC,CAAC;QACtC,CAAC,qBAAqB,EAAE,iBAAiB,IAAI,CAAC,CAAC,CAAC;IAElD,MAAM,oBAAoB,GACxB,cAAc,GAAG,gBAAgB,GAAG,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAAC;IAC5E,MAAM,oBAAoB,GACxB,CAAC,YAAY,EAAE,gBAAgB,IAAI,CAAC,CAAC;QACrC,CAAC,qBAAqB,EAAE,gBAAgB,IAAI,CAAC,CAAC,CAAC;IACjD,MAAM,sBAAsB,GAC1B,CAAC,YAAY,EAAE,mBAAmB,IAAI,CAAC,CAAC;QACxC,CAAC,qBAAqB,EAAE,mBAAmB,IAAI,CAAC,CAAC,CAAC;IAEpD,OAAO;QACL,GAAG,aAAa;QAChB,QAAQ,EAAE;YACR,SAAS,EAAE,yBAAyB;YACpC,QAAQ,EAAE,wBAAwB;SACnC;QACD,aAAa,EAAE,iBAAiB;QAChC,iBAAiB,EAAE,qBAAqB;QACxC,gBAAgB,EAAE,oBAAoB;QACtC,mBAAmB,EAAE,sBAAsB;QAC3C,iBAAiB,EAAE,oBAAoB;KACxC,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,OAAO,CAAC,EAC5B,WAAW,EACX,WAAW,EACX,SAAS,EACT,wBAAwB,EACxB,MAAM,EACN,kBAAkB,GAAG,KAAK,EAC1B,gBAAgB,EAChB,SAAS,GAUV;IACC,MAAM,aAAa,GAAG,OAAC,CAAC,MAAM,CAAC;QAC7B,QAAQ,EAAE,OAAC;aACR,KAAK,CACJ,OAAC,CAAC,MAAM,CAAC;YACP,SAAS,EAAE,OAAC;iBACT,MAAM,EAAE;iBACR,KAAK,CAAC,WAAW,CAAC;iBAClB,QAAQ,CACP,0OAA0O,CAC3O;YACH,WAAW,EAAE,OAAC;iBACX,MAAM,EAAE;iBACR,QAAQ,CACP,yDAAyD,CAC1D;YACH,MAAM,EAAE,OAAC;iBACN,IAAI;YACH,yGAAyG;YACzG,MAAM,CAAC,MAAM,CAAC,uCAAyB,CAGtC,CACF;iBACA,QAAQ,CACP,uHAAuH,CACxH;YACH,SAAS,EAAE,OAAC,CAAC,KAAK,CAChB,OAAC;iBACE,MAAM,EAAE;iBACR,QAAQ,CACP,iJAAiJ,CAClJ,CACJ;SACF,CAAC,CACH;aACA,QAAQ,CAAC,4DAA4D,CAAC;KAC1E,CAAC,CAAC;IAIH,MAAM,QAAQ,GAAkB;QAC9B,IAAA,oCAAwB,EACtB,wBAAwB,EACxB,gBAAgB,EAChB,SAAS,CACV;QACD,IAAA,mCAAuB,EAAC,WAAW,EAAE,WAAW,CAAC;KAClD,CAAC;IAEF,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,iBAAiB,EACjB,cAAc,EACd;YACE,SAAS,EAAE,SAAS;YACpB,QAAQ;SACT,CACF,CAAC;QACF,QAAQ,GAAG,QAAQ,CAAC;QACpB,aAAa,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAkB;QACxE,OAAO,EAAE;YACP,QAAQ;YACR,cAAc,EAAE;gBACd,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,aAAa;aACpB;YACD,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,WAAW,GAAG,GAAG,GAAG,KAAK,CAAC;IAEhC,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;IAC/D,MAAM,YAAY,GAAG,YAAY,EAAE,aAAa,IAAI,CAAC,CAAC;IACtD,MAAM,gBAAgB,GAAG,YAAY,EAAE,iBAAiB,IAAI,CAAC,CAAC;IAC9D,MAAM,eAAe,GAAG,YAAY,EAAE,gBAAgB,IAAI,CAAC,CAAC;IAC5D,MAAM,iBAAiB,GAAG,YAAY,EAAE,mBAAmB,IAAI,CAAC,CAAC;IAEjE,IAAI,YAAoB,CAAC;IACzB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAA,8CAAuB,EAC5D,iBAAiB,EACjB,kBAAkB,EAClB;YACE,aAAa,EAAE,SAAS;YACxB,WAAW,EAAE,WAAW;SACzB,CACF,CAAC;QACF,YAAY,GAAG,gBAAgB,CAAC;QAEhC,IAAA,oCAAa,EAAC,SAAS,EAAE;YACvB,CAAC,wBAAwB,CAAC,EAAE,SAAS;YACrC,SAAS,EAAE,aAAa;YACxB,cAAc,EAAE,QAAQ;YACxB,eAAe,EAAE,YAAY;YAC7B,aAAa,EAAE,YAAY;YAC3B,iBAAiB,EAAE,gBAAgB;YACnC,gBAAgB,EAAE,eAAe;YACjC,mBAAmB,EAAE,iBAAiB;YACtC,iBAAiB,EAAE,WAAW;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,cAAc,GAClB,WAAW,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG;YACX,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,WAAW,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC;YACnC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;YACzB,SAAS,EAAE,EAAE,CAAC,SAAS;SACxB,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,IAAI,EAAE,CAAC;IAEX,OAAO;QACL,QAAQ,EAAE,cAAc;QACxB,aAAa,EAAE,YAAY;QAC3B,iBAAiB,EAAE,gBAAgB;QACnC,gBAAgB,EAAE,eAAe;QACjC,mBAAmB,EAAE,iBAAiB;QACtC,iBAAiB,EAAE,WAAW;KAC/B,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,GAAG,CAAC,EACxB,WAAW,EACX,WAAW,EACX,SAAS,EACT,wBAAwB,EACxB,MAAM,EACN,kBAAkB,GAAG,KAAK,GAQ3B;IACC,MAAM,SAAS,GAAG,OAAC,CAAC,MAAM,CAAC;QACzB,MAAM,EAAE,OAAC;aACN,MAAM,CAAC;YACN,SAAS,EAAE,OAAC;iBACT,MAAM,EAAE;iBACR,KAAK,CAAC,WAAW,CAAC;iBAClB,QAAQ,CACP,4KAA4K,CAC7K;YACH,WAAW,EAAE,OAAC;iBACX,MAAM,EAAE;iBACR,QAAQ,CAAC,yDAAyD,CAAC;YACtE,MAAM,EAAE,OAAC;iBACN,IAAI;YACH,yGAAyG;YACzG,MAAM,CAAC,MAAM,CAAC,uCAAyB,CAGtC,CACF;iBACA,QAAQ,CACP,uHAAuH,CACxH;YACH,SAAS,EAAE,OAAC,CAAC,KAAK,CAChB,OAAC;iBACE,MAAM,EAAE;iBACR,QAAQ,CACP,iJAAiJ,CAClJ,CACJ;SACF,CAAC;aACD,QAAQ,EAAE;aACV,QAAQ,CACP,kLAAkL,CACnL;QACH,OAAO,EAAE,OAAC,CAAC,OAAO,EAAE;KACrB,CAAC,CAAC;IAIH,MAAM,QAAQ,GAAkB;QAC9B,IAAA,gCAAoB,EAAC,wBAAwB,CAAC;QAC9C,IAAA,mCAAuB,EAAC,WAAW,EAAE,WAAW,CAAC;KAClD,CAAC;IAEF,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,aAAa,EACb,UAAU,EACV;YACE,SAAS,EAAE,KAAK;YAChB,QAAQ;SACT,CACF,CAAC;QACF,QAAQ,GAAG,QAAQ,CAAC;QACpB,aAAa,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAc;QACpE,OAAO,EAAE;YACP,QAAQ;YACR,cAAc,EAAE;gBACd,MAAM,EAAE,SAAS;gBACjB,IAAI,EAAE,KAAK;aACZ;YACD,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,WAAW,GAAG,GAAG,GAAG,KAAK,CAAC;IAEhC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC;IACvD,MAAM,YAAY,GAAG,QAAQ,EAAE,aAAa,IAAI,CAAC,CAAC;IAClD,MAAM,gBAAgB,GAAG,QAAQ,EAAE,iBAAiB,IAAI,CAAC,CAAC;IAC1D,MAAM,eAAe,GAAG,QAAQ,EAAE,gBAAgB,IAAI,CAAC,CAAC;IACxD,MAAM,iBAAiB,GAAG,QAAQ,EAAE,mBAAmB,IAAI,CAAC,CAAC;IAE7D,IAAI,YAAoB,CAAC;IACzB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAA,8CAAuB,EAC5D,aAAa,EACb,cAAc,EACd;YACE,aAAa,EAAE,KAAK;YACpB,WAAW,EAAE,OAAO;SACrB,CACF,CAAC;QACF,YAAY,GAAG,gBAAgB,CAAC;QAEhC,IAAA,oCAAa,EAAC,KAAK,EAAE;YACnB,CAAC,oBAAoB,CAAC,EAAE,KAAK;YAC7B,SAAS,EAAE,aAAa;YACxB,cAAc,EAAE,QAAQ;YACxB,eAAe,EAAE,YAAY;YAC7B,aAAa,EAAE,YAAY;YAC3B,iBAAiB,EAAE,gBAAgB;YACnC,gBAAgB,EAAE,eAAe;YACjC,mBAAmB,EAAE,iBAAiB;YACtC,iBAAiB,EAAE,WAAW;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM;QAClC,CAAC,CAAC;YACE,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS;YACnC,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC;YAC/C,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;YACrC,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS;SACpC;QACH,CAAC,CAAC,SAAS,CAAC;IAEd,OAAO;QACL,OAAO,EAAE,aAAa;QACtB,aAAa,EAAE,YAAY;QAC3B,iBAAiB,EAAE,gBAAgB;QACnC,gBAAgB,EAAE,eAAe;QACjC,mBAAmB,EAAE,iBAAiB;QACtC,iBAAiB,EAAE,WAAW;QAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC;AACJ,CAAC","sourcesContent":["import { z } from \"zod\";\nimport { LogLine } from \"./v3/types/public/logs.js\";\nimport { ChatMessage, LLMClient } from \"./v3/llm/LLMClient.js\";\nimport { getEnvTimeoutMs, withTimeout } from \"./v3/timeoutConfig.js\";\nimport {\n  buildActSystemPrompt,\n  buildExtractSystemPrompt,\n  buildExtractUserPrompt,\n  buildMetadataPrompt,\n  buildMetadataSystemPrompt,\n  buildObserveSystemPrompt,\n  buildObserveUserMessage,\n} from \"./prompt.js\";\nimport { appendSummary, writeTimestampedTxtFile } from \"./inferenceLogUtils.js\";\nimport type {\n  InferStagehandSchema,\n  StagehandZodObject,\n} from \"./v3/zodCompat.js\";\nimport { SupportedUnderstudyAction } from \"./v3/types/private/handlers.js\";\nimport type { Variables } from \"./v3/types/public/agent.js\";\nimport { StagehandInvalidArgumentError } from \"./v3/types/public/sdkErrors.js\";\n\n// Re-export for backward compatibility\nexport type { LLMParsedResponse, LLMUsage } from \"./v3/llm/LLMClient.js\";\n\nfunction withLlmTimeout<T>(promise: Promise<T>, operation: string): Promise<T> {\n  return withTimeout(\n    promise,\n    getEnvTimeoutMs(\"LLM_MAX_MS\"),\n    `LLM ${operation}`,\n  );\n}\n\nexport async function extract<T extends StagehandZodObject>({\n  instruction,\n  domElements,\n  schema,\n  llmClient,\n  logger,\n  userProvidedInstructions,\n  logInferenceToFile = false,\n  screenshot,\n}: {\n  instruction: string;\n  domElements: string;\n  schema: T;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n  screenshot?: Buffer;\n}) {\n  const metadataSchema = z.object({\n    progress: z\n      .string()\n      .describe(\n        \"progress of what has been extracted so far, as concise as possible\",\n      ),\n    completed: z\n      .boolean()\n      .describe(\n        \"true if the goal is now accomplished. Use this conservatively, only when sure that the goal has been completed.\",\n      ),\n  });\n\n  type ExtractionResponse = InferStagehandSchema<T>;\n  type MetadataResponse = z.infer<typeof metadataSchema>;\n\n  const isUsingAnthropic = llmClient.type === \"anthropic\";\n  if (screenshot && llmClient.type !== \"aisdk\") {\n    throw new StagehandInvalidArgumentError(\n      \"extract({ screenshot: true }) is only supported with AI SDK clients.\",\n    );\n  }\n  const screenshotDataUrl = screenshot\n    ? `data:image/png;base64,${screenshot.toString(\"base64\")}`\n    : undefined;\n\n  const extractCallMessages: ChatMessage[] = [\n    buildExtractSystemPrompt(\n      isUsingAnthropic,\n      userProvidedInstructions,\n      Boolean(screenshotDataUrl),\n    ),\n    buildExtractUserPrompt(\n      instruction,\n      domElements,\n      isUsingAnthropic,\n      screenshotDataUrl,\n    ),\n  ];\n\n  let extractCallFile = \"\";\n  let extractCallTimestamp = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"extract_call\",\n      {\n        modelCall: \"extract\",\n        messages: extractCallMessages,\n      },\n    );\n    extractCallFile = fileName;\n    extractCallTimestamp = timestamp;\n  }\n\n  const extractStartTime = Date.now();\n  const extractionResponse = await withLlmTimeout(\n    llmClient.createChatCompletion<ExtractionResponse>({\n      options: {\n        messages: extractCallMessages,\n        response_model: {\n          schema,\n          name: \"Extraction\",\n        },\n        top_p: 1,\n        frequency_penalty: 0,\n        presence_penalty: 0,\n      },\n      logger,\n    }),\n    \"extract\",\n  );\n  const extractEndTime = Date.now();\n\n  const { data: extractedData, usage: extractUsage } = extractionResponse;\n\n  let extractResponseFile: string;\n  if (logInferenceToFile) {\n    const { fileName } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"extract_response\",\n      {\n        modelResponse: \"extract\",\n        rawResponse: extractedData,\n      },\n    );\n    extractResponseFile = fileName;\n\n    appendSummary(\"extract\", {\n      extract_inference_type: \"extract\",\n      timestamp: extractCallTimestamp,\n      LLM_input_file: extractCallFile,\n      LLM_output_file: extractResponseFile,\n      prompt_tokens: extractUsage?.prompt_tokens ?? 0,\n      completion_tokens: extractUsage?.completion_tokens ?? 0,\n      reasoning_tokens: extractUsage?.reasoning_tokens ?? 0,\n      cached_input_tokens: extractUsage?.cached_input_tokens ?? 0,\n      inference_time_ms: extractEndTime - extractStartTime,\n    });\n  }\n\n  const metadataCallMessages: ChatMessage[] = [\n    buildMetadataSystemPrompt(),\n    buildMetadataPrompt(instruction, extractedData),\n  ];\n\n  let metadataCallFile = \"\";\n  let metadataCallTimestamp = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"metadata_call\",\n      {\n        modelCall: \"metadata\",\n        messages: metadataCallMessages,\n      },\n    );\n    metadataCallFile = fileName;\n    metadataCallTimestamp = timestamp;\n  }\n\n  const metadataStartTime = Date.now();\n  const metadataResponse = await withLlmTimeout(\n    llmClient.createChatCompletion<MetadataResponse>({\n      options: {\n        messages: metadataCallMessages,\n        response_model: {\n          name: \"Metadata\",\n          schema: metadataSchema,\n        },\n        top_p: 1,\n        frequency_penalty: 0,\n        presence_penalty: 0,\n      },\n      logger,\n    }),\n    \"extract metadata\",\n  );\n  const metadataEndTime = Date.now();\n\n  const {\n    data: {\n      completed: metadataResponseCompleted,\n      progress: metadataResponseProgress,\n    },\n    usage: metadataResponseUsage,\n  } = metadataResponse;\n\n  let metadataResponseFile: string;\n  if (logInferenceToFile) {\n    const { fileName } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"metadata_response\",\n      {\n        modelResponse: \"metadata\",\n        completed: metadataResponseCompleted,\n        progress: metadataResponseProgress,\n      },\n    );\n    metadataResponseFile = fileName;\n\n    appendSummary(\"extract\", {\n      extract_inference_type: \"metadata\",\n      timestamp: metadataCallTimestamp,\n      LLM_input_file: metadataCallFile,\n      LLM_output_file: metadataResponseFile,\n      prompt_tokens: metadataResponseUsage?.prompt_tokens ?? 0,\n      completion_tokens: metadataResponseUsage?.completion_tokens ?? 0,\n      reasoning_tokens: metadataResponseUsage?.reasoning_tokens ?? 0,\n      cached_input_tokens: metadataResponseUsage?.cached_input_tokens ?? 0,\n      inference_time_ms: metadataEndTime - metadataStartTime,\n    });\n  }\n\n  const totalPromptTokens =\n    (extractUsage?.prompt_tokens ?? 0) +\n    (metadataResponseUsage?.prompt_tokens ?? 0);\n\n  const totalCompletionTokens =\n    (extractUsage?.completion_tokens ?? 0) +\n    (metadataResponseUsage?.completion_tokens ?? 0);\n\n  const totalInferenceTimeMs =\n    extractEndTime - extractStartTime + (metadataEndTime - metadataStartTime);\n  const totalReasoningTokens =\n    (extractUsage?.reasoning_tokens ?? 0) +\n    (metadataResponseUsage?.reasoning_tokens ?? 0);\n  const totalCachedInputTokens =\n    (extractUsage?.cached_input_tokens ?? 0) +\n    (metadataResponseUsage?.cached_input_tokens ?? 0);\n\n  return {\n    ...extractedData,\n    metadata: {\n      completed: metadataResponseCompleted,\n      progress: metadataResponseProgress,\n    },\n    prompt_tokens: totalPromptTokens,\n    completion_tokens: totalCompletionTokens,\n    reasoning_tokens: totalReasoningTokens,\n    cached_input_tokens: totalCachedInputTokens,\n    inference_time_ms: totalInferenceTimeMs,\n  };\n}\n\nexport async function observe({\n  instruction,\n  domElements,\n  llmClient,\n  userProvidedInstructions,\n  logger,\n  logInferenceToFile = false,\n  supportedActions,\n  variables,\n}: {\n  instruction: string;\n  domElements: string;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n  supportedActions?: string[];\n  variables?: Variables;\n}) {\n  const observeSchema = z.object({\n    elements: z\n      .array(\n        z.object({\n          elementId: z\n            .string()\n            .regex(/^\\d+-\\d+$/)\n            .describe(\n              \"the exact ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'. For example, if the accessibility tree shows [0-18372], return '0-18372', not '18372'.\",\n            ),\n          description: z\n            .string()\n            .describe(\n              \"a description of the accessible element and its purpose\",\n            ),\n          method: z\n            .enum(\n              // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly\n              Object.values(SupportedUnderstudyAction) as unknown as readonly [\n                string,\n                ...string[],\n              ],\n            )\n            .describe(\n              `the candidate method/action to interact with the element. Select one of the available Understudy interaction methods.`,\n            ),\n          arguments: z.array(\n            z\n              .string()\n              .describe(\n                \"the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.\",\n              ),\n          ),\n        }),\n      )\n      .describe(\"an array of accessible elements that match the instruction\"),\n  });\n\n  type ObserveResponse = z.infer<typeof observeSchema>;\n\n  const messages: ChatMessage[] = [\n    buildObserveSystemPrompt(\n      userProvidedInstructions,\n      supportedActions,\n      variables,\n    ),\n    buildObserveUserMessage(instruction, domElements),\n  ];\n\n  let callTimestamp = \"\";\n  let callFile = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      `observe_summary`,\n      `observe_call`,\n      {\n        modelCall: \"observe\",\n        messages,\n      },\n    );\n    callFile = fileName;\n    callTimestamp = timestamp;\n  }\n\n  const start = Date.now();\n  const rawResponse = await llmClient.createChatCompletion<ObserveResponse>({\n    options: {\n      messages,\n      response_model: {\n        schema: observeSchema,\n        name: \"Observation\",\n      },\n      top_p: 1,\n      frequency_penalty: 0,\n      presence_penalty: 0,\n    },\n    logger,\n  });\n  const end = Date.now();\n  const usageTimeMs = end - start;\n\n  const { data: observeData, usage: observeUsage } = rawResponse;\n  const promptTokens = observeUsage?.prompt_tokens ?? 0;\n  const completionTokens = observeUsage?.completion_tokens ?? 0;\n  const reasoningTokens = observeUsage?.reasoning_tokens ?? 0;\n  const cachedInputTokens = observeUsage?.cached_input_tokens ?? 0;\n\n  let responseFile: string;\n  if (logInferenceToFile) {\n    const { fileName: responseFileName } = writeTimestampedTxtFile(\n      `observe_summary`,\n      `observe_response`,\n      {\n        modelResponse: \"observe\",\n        rawResponse: observeData,\n      },\n    );\n    responseFile = responseFileName;\n\n    appendSummary(\"observe\", {\n      [`observe_inference_type`]: \"observe\",\n      timestamp: callTimestamp,\n      LLM_input_file: callFile,\n      LLM_output_file: responseFile,\n      prompt_tokens: promptTokens,\n      completion_tokens: completionTokens,\n      reasoning_tokens: reasoningTokens,\n      cached_input_tokens: cachedInputTokens,\n      inference_time_ms: usageTimeMs,\n    });\n  }\n\n  const parsedElements =\n    observeData.elements?.map((el) => {\n      const base = {\n        elementId: el.elementId,\n        description: String(el.description),\n        method: String(el.method),\n        arguments: el.arguments,\n      };\n      return base;\n    }) ?? [];\n\n  return {\n    elements: parsedElements,\n    prompt_tokens: promptTokens,\n    completion_tokens: completionTokens,\n    reasoning_tokens: reasoningTokens,\n    cached_input_tokens: cachedInputTokens,\n    inference_time_ms: usageTimeMs,\n  };\n}\n\nexport async function act({\n  instruction,\n  domElements,\n  llmClient,\n  userProvidedInstructions,\n  logger,\n  logInferenceToFile = false,\n}: {\n  instruction: string;\n  domElements: string;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n}) {\n  const actSchema = z.object({\n    action: z\n      .object({\n        elementId: z\n          .string()\n          .regex(/^\\d+-\\d+$/)\n          .describe(\n            \"the ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'. for example, '0-76' or '16-21'\",\n          ),\n        description: z\n          .string()\n          .describe(\"a description of the accessible element and its purpose\"),\n        method: z\n          .enum(\n            // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly\n            Object.values(SupportedUnderstudyAction) as unknown as readonly [\n              string,\n              ...string[],\n            ],\n          )\n          .describe(\n            \"the candidate method/action to interact with the element. Select one of the available Understudy interaction methods.\",\n          ),\n        arguments: z.array(\n          z\n            .string()\n            .describe(\n              \"the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.\",\n            ),\n        ),\n      })\n      .nullable()\n      .describe(\n        \"The element to act on. Return null if no element on the page matches the instruction — do NOT fabricate or guess an element, and never emit empty strings or placeholder values.\",\n      ),\n    twoStep: z.boolean(),\n  });\n\n  type ActResponse = z.infer<typeof actSchema>;\n\n  const messages: ChatMessage[] = [\n    buildActSystemPrompt(userProvidedInstructions),\n    buildObserveUserMessage(instruction, domElements),\n  ];\n\n  let callTimestamp = \"\";\n  let callFile = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      `act_summary`,\n      `act_call`,\n      {\n        modelCall: \"act\",\n        messages,\n      },\n    );\n    callFile = fileName;\n    callTimestamp = timestamp;\n  }\n\n  const start = Date.now();\n  const rawResponse = await llmClient.createChatCompletion<ActResponse>({\n    options: {\n      messages,\n      response_model: {\n        schema: actSchema,\n        name: \"act\",\n      },\n      top_p: 1,\n      frequency_penalty: 0,\n      presence_penalty: 0,\n    },\n    logger,\n  });\n  const end = Date.now();\n  const usageTimeMs = end - start;\n\n  const { data: actData, usage: actUsage } = rawResponse;\n  const promptTokens = actUsage?.prompt_tokens ?? 0;\n  const completionTokens = actUsage?.completion_tokens ?? 0;\n  const reasoningTokens = actUsage?.reasoning_tokens ?? 0;\n  const cachedInputTokens = actUsage?.cached_input_tokens ?? 0;\n\n  let responseFile: string;\n  if (logInferenceToFile) {\n    const { fileName: responseFileName } = writeTimestampedTxtFile(\n      `act_summary`,\n      `act_response`,\n      {\n        modelResponse: \"act\",\n        rawResponse: actData,\n      },\n    );\n    responseFile = responseFileName;\n\n    appendSummary(\"act\", {\n      [`act_inference_type`]: \"act\",\n      timestamp: callTimestamp,\n      LLM_input_file: callFile,\n      LLM_output_file: responseFile,\n      prompt_tokens: promptTokens,\n      completion_tokens: completionTokens,\n      reasoning_tokens: reasoningTokens,\n      cached_input_tokens: cachedInputTokens,\n      inference_time_ms: usageTimeMs,\n    });\n  }\n\n  const parsedElement = actData.action\n    ? {\n        elementId: actData.action.elementId,\n        description: String(actData.action.description),\n        method: String(actData.action.method),\n        arguments: actData.action.arguments,\n      }\n    : undefined;\n\n  return {\n    element: parsedElement,\n    prompt_tokens: promptTokens,\n    completion_tokens: completionTokens,\n    reasoning_tokens: reasoningTokens,\n    cached_input_tokens: cachedInputTokens,\n    inference_time_ms: usageTimeMs,\n    twoStep: actData.twoStep,\n  };\n}\n"]}
+{"version":3,"file":"inference.js","sourceRoot":"","sources":["../../../lib/inference.ts"],"names":[],"mappings":";;AAiCA,0BA8NC;AAED,0BAsJC;AAED,kBAqJC;AA9iBD,6BAAwB;AAGxB,4DAAqE;AACrE,2CAQqB;AACrB,iEAAgF;AAKhF,gEAA2E;AAE3E,iEAA+E;AAK/E,SAAS,cAAc,CAAI,OAAmB,EAAE,SAAiB;IAC/D,OAAO,IAAA,8BAAW,EAChB,OAAO,EACP,IAAA,kCAAe,EAAC,YAAY,CAAC,EAC7B,OAAO,SAAS,EAAE,CACnB,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,OAAO,CAA+B,EAC1D,WAAW,EACX,WAAW,EACX,MAAM,EACN,SAAS,EACT,MAAM,EACN,wBAAwB,EACxB,kBAAkB,GAAG,KAAK,EAC1B,UAAU,GAUX;IACC,MAAM,cAAc,GAAG,OAAC,CAAC,MAAM,CAAC;QAC9B,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,oEAAoE,CACrE;QACH,SAAS,EAAE,OAAC;aACT,OAAO,EAAE;aACT,QAAQ,CACP,iHAAiH,CAClH;KACJ,CAAC,CAAC;IAKH,MAAM,gBAAgB,GAAG,SAAS,CAAC,IAAI,KAAK,WAAW,CAAC;IACxD,IAAI,UAAU,IAAI,SAAS,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC7C,MAAM,IAAI,4CAA6B,CACrC,sEAAsE,CACvE,CAAC;IACJ,CAAC;IACD,MAAM,iBAAiB,GAAG,UAAU;QAClC,CAAC,CAAC,yBAAyB,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;QAC1D,CAAC,CAAC,SAAS,CAAC;IAEd,MAAM,mBAAmB,GAAkB;QACzC,IAAA,oCAAwB,EACtB,gBAAgB,EAChB,wBAAwB,EACxB,OAAO,CAAC,iBAAiB,CAAC,CAC3B;QACD,IAAA,kCAAsB,EACpB,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,iBAAiB,CAClB;KACF,CAAC;IAEF,IAAI,eAAe,GAAG,EAAE,CAAC;IACzB,IAAI,oBAAoB,GAAG,EAAE,CAAC;IAC9B,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,iBAAiB,EACjB,cAAc,EACd;YACE,SAAS,EAAE,SAAS;YACpB,QAAQ,EAAE,mBAAmB;SAC9B,CACF,CAAC;QACF,eAAe,GAAG,QAAQ,CAAC;QAC3B,oBAAoB,GAAG,SAAS,CAAC;IACnC,CAAC;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,MAAM,kBAAkB,GAAG,MAAM,cAAc,CAC7C,SAAS,CAAC,oBAAoB,CAAqB;QACjD,OAAO,EAAE;YACP,QAAQ,EAAE,mBAAmB;YAC7B,cAAc,EAAE;gBACd,MAAM;gBACN,IAAI,EAAE,YAAY;aACnB;YACD,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,EACF,SAAS,CACV,CAAC;IACF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,kBAAkB,CAAC;IAExE,IAAI,mBAA2B,CAAC;IAChC,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAA,8CAAuB,EAC1C,iBAAiB,EACjB,kBAAkB,EAClB;YACE,aAAa,EAAE,SAAS;YACxB,WAAW,EAAE,aAAa;SAC3B,CACF,CAAC;QACF,mBAAmB,GAAG,QAAQ,CAAC;QAE/B,IAAA,oCAAa,EAAC,SAAS,EAAE;YACvB,sBAAsB,EAAE,SAAS;YACjC,SAAS,EAAE,oBAAoB;YAC/B,cAAc,EAAE,eAAe;YAC/B,eAAe,EAAE,mBAAmB;YACpC,aAAa,EAAE,YAAY,EAAE,aAAa,IAAI,CAAC;YAC/C,iBAAiB,EAAE,YAAY,EAAE,iBAAiB,IAAI,CAAC;YACvD,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,IAAI,CAAC;YACrD,mBAAmB,EAAE,YAAY,EAAE,mBAAmB,IAAI,CAAC;YAC3D,iBAAiB,EAAE,cAAc,GAAG,gBAAgB;SACrD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,oBAAoB,GAAkB;QAC1C,IAAA,qCAAyB,GAAE;QAC3B,IAAA,+BAAmB,EAAC,WAAW,EAAE,aAAa,CAAC;KAChD,CAAC;IAEF,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAC1B,IAAI,qBAAqB,GAAG,EAAE,CAAC;IAC/B,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,iBAAiB,EACjB,eAAe,EACf;YACE,SAAS,EAAE,UAAU;YACrB,QAAQ,EAAE,oBAAoB;SAC/B,CACF,CAAC;QACF,gBAAgB,GAAG,QAAQ,CAAC;QAC5B,qBAAqB,GAAG,SAAS,CAAC;IACpC,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACrC,MAAM,gBAAgB,GAAG,MAAM,cAAc,CAC3C,SAAS,CAAC,oBAAoB,CAAmB;QAC/C,OAAO,EAAE;YACP,QAAQ,EAAE,oBAAoB;YAC9B,cAAc,EAAE;gBACd,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,cAAc;aACvB;YACD,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,EACF,kBAAkB,CACnB,CAAC;IACF,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEnC,MAAM,EACJ,IAAI,EAAE,EACJ,SAAS,EAAE,yBAAyB,EACpC,QAAQ,EAAE,wBAAwB,GACnC,EACD,KAAK,EAAE,qBAAqB,GAC7B,GAAG,gBAAgB,CAAC;IAErB,IAAI,oBAA4B,CAAC;IACjC,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAA,8CAAuB,EAC1C,iBAAiB,EACjB,mBAAmB,EACnB;YACE,aAAa,EAAE,UAAU;YACzB,SAAS,EAAE,yBAAyB;YACpC,QAAQ,EAAE,wBAAwB;SACnC,CACF,CAAC;QACF,oBAAoB,GAAG,QAAQ,CAAC;QAEhC,IAAA,oCAAa,EAAC,SAAS,EAAE;YACvB,sBAAsB,EAAE,UAAU;YAClC,SAAS,EAAE,qBAAqB;YAChC,cAAc,EAAE,gBAAgB;YAChC,eAAe,EAAE,oBAAoB;YACrC,aAAa,EAAE,qBAAqB,EAAE,aAAa,IAAI,CAAC;YACxD,iBAAiB,EAAE,qBAAqB,EAAE,iBAAiB,IAAI,CAAC;YAChE,gBAAgB,EAAE,qBAAqB,EAAE,gBAAgB,IAAI,CAAC;YAC9D,mBAAmB,EAAE,qBAAqB,EAAE,mBAAmB,IAAI,CAAC;YACpE,iBAAiB,EAAE,eAAe,GAAG,iBAAiB;SACvD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,iBAAiB,GACrB,CAAC,YAAY,EAAE,aAAa,IAAI,CAAC,CAAC;QAClC,CAAC,qBAAqB,EAAE,aAAa,IAAI,CAAC,CAAC,CAAC;IAE9C,MAAM,qBAAqB,GACzB,CAAC,YAAY,EAAE,iBAAiB,IAAI,CAAC,CAAC;QACtC,CAAC,qBAAqB,EAAE,iBAAiB,IAAI,CAAC,CAAC,CAAC;IAElD,MAAM,oBAAoB,GACxB,cAAc,GAAG,gBAAgB,GAAG,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAAC;IAC5E,MAAM,oBAAoB,GACxB,CAAC,YAAY,EAAE,gBAAgB,IAAI,CAAC,CAAC;QACrC,CAAC,qBAAqB,EAAE,gBAAgB,IAAI,CAAC,CAAC,CAAC;IACjD,MAAM,sBAAsB,GAC1B,CAAC,YAAY,EAAE,mBAAmB,IAAI,CAAC,CAAC;QACxC,CAAC,qBAAqB,EAAE,mBAAmB,IAAI,CAAC,CAAC,CAAC;IAEpD,OAAO;QACL,GAAG,aAAa;QAChB,QAAQ,EAAE;YACR,SAAS,EAAE,yBAAyB;YACpC,QAAQ,EAAE,wBAAwB;SACnC;QACD,aAAa,EAAE,iBAAiB;QAChC,iBAAiB,EAAE,qBAAqB;QACxC,gBAAgB,EAAE,oBAAoB;QACtC,mBAAmB,EAAE,sBAAsB;QAC3C,iBAAiB,EAAE,oBAAoB;KACxC,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,OAAO,CAAC,EAC5B,WAAW,EACX,WAAW,EACX,SAAS,EACT,wBAAwB,EACxB,MAAM,EACN,kBAAkB,GAAG,KAAK,EAC1B,gBAAgB,EAChB,SAAS,GAUV;IACC,MAAM,aAAa,GAAG,OAAC,CAAC,MAAM,CAAC;QAC7B,QAAQ,EAAE,OAAC;aACR,KAAK,CACJ,OAAC,CAAC,MAAM,CAAC;YACP,SAAS,EAAE,OAAC;iBACT,MAAM,EAAE;iBACR,KAAK,CAAC,WAAW,CAAC;iBAClB,QAAQ,CACP,0OAA0O,CAC3O;YACH,WAAW,EAAE,OAAC;iBACX,MAAM,EAAE;iBACR,QAAQ,CACP,yDAAyD,CAC1D;YACH,MAAM,EAAE,OAAC;iBACN,IAAI;YACH,yGAAyG;YACzG,MAAM,CAAC,MAAM,CAAC,uCAAyB,CAGtC,CACF;iBACA,QAAQ,CACP,uHAAuH,CACxH;YACH,SAAS,EAAE,OAAC,CAAC,KAAK,CAChB,OAAC;iBACE,MAAM,EAAE;iBACR,QAAQ,CACP,iJAAiJ,CAClJ,CACJ;SACF,CAAC,CACH;aACA,QAAQ,CAAC,4DAA4D,CAAC;KAC1E,CAAC,CAAC;IAIH,MAAM,QAAQ,GAAkB;QAC9B,IAAA,oCAAwB,EACtB,wBAAwB,EACxB,gBAAgB,EAChB,SAAS,CACV;QACD,IAAA,mCAAuB,EAAC,WAAW,EAAE,WAAW,CAAC;KAClD,CAAC;IAEF,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,iBAAiB,EACjB,cAAc,EACd;YACE,SAAS,EAAE,SAAS;YACpB,QAAQ;SACT,CACF,CAAC;QACF,QAAQ,GAAG,QAAQ,CAAC;QACpB,aAAa,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAkB;QACxE,OAAO,EAAE;YACP,QAAQ;YACR,cAAc,EAAE;gBACd,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,aAAa;aACpB;YACD,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,WAAW,GAAG,GAAG,GAAG,KAAK,CAAC;IAEhC,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;IAC/D,MAAM,YAAY,GAAG,YAAY,EAAE,aAAa,IAAI,CAAC,CAAC;IACtD,MAAM,gBAAgB,GAAG,YAAY,EAAE,iBAAiB,IAAI,CAAC,CAAC;IAC9D,MAAM,eAAe,GAAG,YAAY,EAAE,gBAAgB,IAAI,CAAC,CAAC;IAC5D,MAAM,iBAAiB,GAAG,YAAY,EAAE,mBAAmB,IAAI,CAAC,CAAC;IAEjE,IAAI,YAAoB,CAAC;IACzB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAA,8CAAuB,EAC5D,iBAAiB,EACjB,kBAAkB,EAClB;YACE,aAAa,EAAE,SAAS;YACxB,WAAW,EAAE,WAAW;SACzB,CACF,CAAC;QACF,YAAY,GAAG,gBAAgB,CAAC;QAEhC,IAAA,oCAAa,EAAC,SAAS,EAAE;YACvB,CAAC,wBAAwB,CAAC,EAAE,SAAS;YACrC,SAAS,EAAE,aAAa;YACxB,cAAc,EAAE,QAAQ;YACxB,eAAe,EAAE,YAAY;YAC7B,aAAa,EAAE,YAAY;YAC3B,iBAAiB,EAAE,gBAAgB;YACnC,gBAAgB,EAAE,eAAe;YACjC,mBAAmB,EAAE,iBAAiB;YACtC,iBAAiB,EAAE,WAAW;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,cAAc,GAClB,WAAW,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG;YACX,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,WAAW,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC;YACnC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;YACzB,SAAS,EAAE,EAAE,CAAC,SAAS;SACxB,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,IAAI,EAAE,CAAC;IAEX,OAAO;QACL,QAAQ,EAAE,cAAc;QACxB,aAAa,EAAE,YAAY;QAC3B,iBAAiB,EAAE,gBAAgB;QACnC,gBAAgB,EAAE,eAAe;QACjC,mBAAmB,EAAE,iBAAiB;QACtC,iBAAiB,EAAE,WAAW;KAC/B,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,GAAG,CAAC,EACxB,WAAW,EACX,WAAW,EACX,SAAS,EACT,wBAAwB,EACxB,MAAM,EACN,kBAAkB,GAAG,KAAK,GAQ3B;IACC,MAAM,YAAY,GAAG,OAAC,CAAC,MAAM,CAAC;QAC5B,SAAS,EAAE,OAAC;aACT,MAAM,EAAE;aACR,KAAK,CAAC,WAAW,CAAC;aAClB,QAAQ,CACP,4KAA4K,CAC7K;QACH,WAAW,EAAE,OAAC;aACX,MAAM,EAAE;aACR,QAAQ,CAAC,yDAAyD,CAAC;QACtE,MAAM,EAAE,OAAC;aACN,IAAI;QACH,yGAAyG;QACzG,MAAM,CAAC,MAAM,CAAC,uCAAyB,CAGtC,CACF;aACA,QAAQ,CACP,uHAAuH,CACxH;QACH,SAAS,EAAE,OAAC,CAAC,KAAK,CAChB,OAAC;aACE,MAAM,EAAE;aACR,QAAQ,CACP,iJAAiJ,CAClJ,CACJ;KACF,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,OAAC,CAAC,MAAM,CAAC;QACzB,2EAA2E;QAC3E,4EAA4E;QAC5E,yEAAyE;QACzE,wCAAwC;QACxC,MAAM,EAAE,CAAC,OAAO,YAAY,CAAC,IAAI,KAAK,UAAU;YAC9C,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;YACpD,CAAC,CAAC,YAAY,CACf;aACE,QAAQ,EAAE;aACV,QAAQ,CACP,kLAAkL,CACnL;QACH,OAAO,EAAE,OAAC,CAAC,OAAO,EAAE;KACrB,CAAC,CAAC;IAIH,MAAM,QAAQ,GAAkB;QAC9B,IAAA,gCAAoB,EAAC,wBAAwB,CAAC;QAC9C,IAAA,mCAAuB,EAAC,WAAW,EAAE,WAAW,CAAC;KAClD,CAAC;IAEF,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,aAAa,EACb,UAAU,EACV;YACE,SAAS,EAAE,KAAK;YAChB,QAAQ;SACT,CACF,CAAC;QACF,QAAQ,GAAG,QAAQ,CAAC;QACpB,aAAa,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAc;QACpE,OAAO,EAAE;YACP,QAAQ;YACR,cAAc,EAAE;gBACd,MAAM,EAAE,SAAS;gBACjB,IAAI,EAAE,KAAK;aACZ;YACD,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,WAAW,GAAG,GAAG,GAAG,KAAK,CAAC;IAEhC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC;IACvD,MAAM,YAAY,GAAG,QAAQ,EAAE,aAAa,IAAI,CAAC,CAAC;IAClD,MAAM,gBAAgB,GAAG,QAAQ,EAAE,iBAAiB,IAAI,CAAC,CAAC;IAC1D,MAAM,eAAe,GAAG,QAAQ,EAAE,gBAAgB,IAAI,CAAC,CAAC;IACxD,MAAM,iBAAiB,GAAG,QAAQ,EAAE,mBAAmB,IAAI,CAAC,CAAC;IAE7D,IAAI,YAAoB,CAAC;IACzB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAA,8CAAuB,EAC5D,aAAa,EACb,cAAc,EACd;YACE,aAAa,EAAE,KAAK;YACpB,WAAW,EAAE,OAAO;SACrB,CACF,CAAC;QACF,YAAY,GAAG,gBAAgB,CAAC;QAEhC,IAAA,oCAAa,EAAC,KAAK,EAAE;YACnB,CAAC,oBAAoB,CAAC,EAAE,KAAK;YAC7B,SAAS,EAAE,aAAa;YACxB,cAAc,EAAE,QAAQ;YACxB,eAAe,EAAE,YAAY;YAC7B,aAAa,EAAE,YAAY;YAC3B,iBAAiB,EAAE,gBAAgB;YACnC,gBAAgB,EAAE,eAAe;YACjC,mBAAmB,EAAE,iBAAiB;YACtC,iBAAiB,EAAE,WAAW;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM;QAClC,CAAC,CAAC;YACE,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS;YACnC,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC;YAC/C,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;YACrC,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS;SACpC;QACH,CAAC,CAAC,SAAS,CAAC;IAEd,OAAO;QACL,OAAO,EAAE,aAAa;QACtB,aAAa,EAAE,YAAY;QAC3B,iBAAiB,EAAE,gBAAgB;QACnC,gBAAgB,EAAE,eAAe;QACjC,mBAAmB,EAAE,iBAAiB;QACtC,iBAAiB,EAAE,WAAW;QAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC;AACJ,CAAC","sourcesContent":["import { z } from \"zod\";\nimport { LogLine } from \"./v3/types/public/logs.js\";\nimport { ChatMessage, LLMClient } from \"./v3/llm/LLMClient.js\";\nimport { getEnvTimeoutMs, withTimeout } from \"./v3/timeoutConfig.js\";\nimport {\n  buildActSystemPrompt,\n  buildExtractSystemPrompt,\n  buildExtractUserPrompt,\n  buildMetadataPrompt,\n  buildMetadataSystemPrompt,\n  buildObserveSystemPrompt,\n  buildObserveUserMessage,\n} from \"./prompt.js\";\nimport { appendSummary, writeTimestampedTxtFile } from \"./inferenceLogUtils.js\";\nimport type {\n  InferStagehandSchema,\n  StagehandZodObject,\n} from \"./v3/zodCompat.js\";\nimport { SupportedUnderstudyAction } from \"./v3/types/private/handlers.js\";\nimport type { Variables } from \"./v3/types/public/agent.js\";\nimport { StagehandInvalidArgumentError } from \"./v3/types/public/sdkErrors.js\";\n\n// Re-export for backward compatibility\nexport type { LLMParsedResponse, LLMUsage } from \"./v3/llm/LLMClient.js\";\n\nfunction withLlmTimeout<T>(promise: Promise<T>, operation: string): Promise<T> {\n  return withTimeout(\n    promise,\n    getEnvTimeoutMs(\"LLM_MAX_MS\"),\n    `LLM ${operation}`,\n  );\n}\n\nexport async function extract<T extends StagehandZodObject>({\n  instruction,\n  domElements,\n  schema,\n  llmClient,\n  logger,\n  userProvidedInstructions,\n  logInferenceToFile = false,\n  screenshot,\n}: {\n  instruction: string;\n  domElements: string;\n  schema: T;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n  screenshot?: Buffer;\n}) {\n  const metadataSchema = z.object({\n    progress: z\n      .string()\n      .describe(\n        \"progress of what has been extracted so far, as concise as possible\",\n      ),\n    completed: z\n      .boolean()\n      .describe(\n        \"true if the goal is now accomplished. Use this conservatively, only when sure that the goal has been completed.\",\n      ),\n  });\n\n  type ExtractionResponse = InferStagehandSchema<T>;\n  type MetadataResponse = z.infer<typeof metadataSchema>;\n\n  const isUsingAnthropic = llmClient.type === \"anthropic\";\n  if (screenshot && llmClient.type !== \"aisdk\") {\n    throw new StagehandInvalidArgumentError(\n      \"extract({ screenshot: true }) is only supported with AI SDK clients.\",\n    );\n  }\n  const screenshotDataUrl = screenshot\n    ? `data:image/png;base64,${screenshot.toString(\"base64\")}`\n    : undefined;\n\n  const extractCallMessages: ChatMessage[] = [\n    buildExtractSystemPrompt(\n      isUsingAnthropic,\n      userProvidedInstructions,\n      Boolean(screenshotDataUrl),\n    ),\n    buildExtractUserPrompt(\n      instruction,\n      domElements,\n      isUsingAnthropic,\n      screenshotDataUrl,\n    ),\n  ];\n\n  let extractCallFile = \"\";\n  let extractCallTimestamp = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"extract_call\",\n      {\n        modelCall: \"extract\",\n        messages: extractCallMessages,\n      },\n    );\n    extractCallFile = fileName;\n    extractCallTimestamp = timestamp;\n  }\n\n  const extractStartTime = Date.now();\n  const extractionResponse = await withLlmTimeout(\n    llmClient.createChatCompletion<ExtractionResponse>({\n      options: {\n        messages: extractCallMessages,\n        response_model: {\n          schema,\n          name: \"Extraction\",\n        },\n        top_p: 1,\n        frequency_penalty: 0,\n        presence_penalty: 0,\n      },\n      logger,\n    }),\n    \"extract\",\n  );\n  const extractEndTime = Date.now();\n\n  const { data: extractedData, usage: extractUsage } = extractionResponse;\n\n  let extractResponseFile: string;\n  if (logInferenceToFile) {\n    const { fileName } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"extract_response\",\n      {\n        modelResponse: \"extract\",\n        rawResponse: extractedData,\n      },\n    );\n    extractResponseFile = fileName;\n\n    appendSummary(\"extract\", {\n      extract_inference_type: \"extract\",\n      timestamp: extractCallTimestamp,\n      LLM_input_file: extractCallFile,\n      LLM_output_file: extractResponseFile,\n      prompt_tokens: extractUsage?.prompt_tokens ?? 0,\n      completion_tokens: extractUsage?.completion_tokens ?? 0,\n      reasoning_tokens: extractUsage?.reasoning_tokens ?? 0,\n      cached_input_tokens: extractUsage?.cached_input_tokens ?? 0,\n      inference_time_ms: extractEndTime - extractStartTime,\n    });\n  }\n\n  const metadataCallMessages: ChatMessage[] = [\n    buildMetadataSystemPrompt(),\n    buildMetadataPrompt(instruction, extractedData),\n  ];\n\n  let metadataCallFile = \"\";\n  let metadataCallTimestamp = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"metadata_call\",\n      {\n        modelCall: \"metadata\",\n        messages: metadataCallMessages,\n      },\n    );\n    metadataCallFile = fileName;\n    metadataCallTimestamp = timestamp;\n  }\n\n  const metadataStartTime = Date.now();\n  const metadataResponse = await withLlmTimeout(\n    llmClient.createChatCompletion<MetadataResponse>({\n      options: {\n        messages: metadataCallMessages,\n        response_model: {\n          name: \"Metadata\",\n          schema: metadataSchema,\n        },\n        top_p: 1,\n        frequency_penalty: 0,\n        presence_penalty: 0,\n      },\n      logger,\n    }),\n    \"extract metadata\",\n  );\n  const metadataEndTime = Date.now();\n\n  const {\n    data: {\n      completed: metadataResponseCompleted,\n      progress: metadataResponseProgress,\n    },\n    usage: metadataResponseUsage,\n  } = metadataResponse;\n\n  let metadataResponseFile: string;\n  if (logInferenceToFile) {\n    const { fileName } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"metadata_response\",\n      {\n        modelResponse: \"metadata\",\n        completed: metadataResponseCompleted,\n        progress: metadataResponseProgress,\n      },\n    );\n    metadataResponseFile = fileName;\n\n    appendSummary(\"extract\", {\n      extract_inference_type: \"metadata\",\n      timestamp: metadataCallTimestamp,\n      LLM_input_file: metadataCallFile,\n      LLM_output_file: metadataResponseFile,\n      prompt_tokens: metadataResponseUsage?.prompt_tokens ?? 0,\n      completion_tokens: metadataResponseUsage?.completion_tokens ?? 0,\n      reasoning_tokens: metadataResponseUsage?.reasoning_tokens ?? 0,\n      cached_input_tokens: metadataResponseUsage?.cached_input_tokens ?? 0,\n      inference_time_ms: metadataEndTime - metadataStartTime,\n    });\n  }\n\n  const totalPromptTokens =\n    (extractUsage?.prompt_tokens ?? 0) +\n    (metadataResponseUsage?.prompt_tokens ?? 0);\n\n  const totalCompletionTokens =\n    (extractUsage?.completion_tokens ?? 0) +\n    (metadataResponseUsage?.completion_tokens ?? 0);\n\n  const totalInferenceTimeMs =\n    extractEndTime - extractStartTime + (metadataEndTime - metadataStartTime);\n  const totalReasoningTokens =\n    (extractUsage?.reasoning_tokens ?? 0) +\n    (metadataResponseUsage?.reasoning_tokens ?? 0);\n  const totalCachedInputTokens =\n    (extractUsage?.cached_input_tokens ?? 0) +\n    (metadataResponseUsage?.cached_input_tokens ?? 0);\n\n  return {\n    ...extractedData,\n    metadata: {\n      completed: metadataResponseCompleted,\n      progress: metadataResponseProgress,\n    },\n    prompt_tokens: totalPromptTokens,\n    completion_tokens: totalCompletionTokens,\n    reasoning_tokens: totalReasoningTokens,\n    cached_input_tokens: totalCachedInputTokens,\n    inference_time_ms: totalInferenceTimeMs,\n  };\n}\n\nexport async function observe({\n  instruction,\n  domElements,\n  llmClient,\n  userProvidedInstructions,\n  logger,\n  logInferenceToFile = false,\n  supportedActions,\n  variables,\n}: {\n  instruction: string;\n  domElements: string;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n  supportedActions?: string[];\n  variables?: Variables;\n}) {\n  const observeSchema = z.object({\n    elements: z\n      .array(\n        z.object({\n          elementId: z\n            .string()\n            .regex(/^\\d+-\\d+$/)\n            .describe(\n              \"the exact ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'. For example, if the accessibility tree shows [0-18372], return '0-18372', not '18372'.\",\n            ),\n          description: z\n            .string()\n            .describe(\n              \"a description of the accessible element and its purpose\",\n            ),\n          method: z\n            .enum(\n              // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly\n              Object.values(SupportedUnderstudyAction) as unknown as readonly [\n                string,\n                ...string[],\n              ],\n            )\n            .describe(\n              `the candidate method/action to interact with the element. Select one of the available Understudy interaction methods.`,\n            ),\n          arguments: z.array(\n            z\n              .string()\n              .describe(\n                \"the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.\",\n              ),\n          ),\n        }),\n      )\n      .describe(\"an array of accessible elements that match the instruction\"),\n  });\n\n  type ObserveResponse = z.infer<typeof observeSchema>;\n\n  const messages: ChatMessage[] = [\n    buildObserveSystemPrompt(\n      userProvidedInstructions,\n      supportedActions,\n      variables,\n    ),\n    buildObserveUserMessage(instruction, domElements),\n  ];\n\n  let callTimestamp = \"\";\n  let callFile = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      `observe_summary`,\n      `observe_call`,\n      {\n        modelCall: \"observe\",\n        messages,\n      },\n    );\n    callFile = fileName;\n    callTimestamp = timestamp;\n  }\n\n  const start = Date.now();\n  const rawResponse = await llmClient.createChatCompletion<ObserveResponse>({\n    options: {\n      messages,\n      response_model: {\n        schema: observeSchema,\n        name: \"Observation\",\n      },\n      top_p: 1,\n      frequency_penalty: 0,\n      presence_penalty: 0,\n    },\n    logger,\n  });\n  const end = Date.now();\n  const usageTimeMs = end - start;\n\n  const { data: observeData, usage: observeUsage } = rawResponse;\n  const promptTokens = observeUsage?.prompt_tokens ?? 0;\n  const completionTokens = observeUsage?.completion_tokens ?? 0;\n  const reasoningTokens = observeUsage?.reasoning_tokens ?? 0;\n  const cachedInputTokens = observeUsage?.cached_input_tokens ?? 0;\n\n  let responseFile: string;\n  if (logInferenceToFile) {\n    const { fileName: responseFileName } = writeTimestampedTxtFile(\n      `observe_summary`,\n      `observe_response`,\n      {\n        modelResponse: \"observe\",\n        rawResponse: observeData,\n      },\n    );\n    responseFile = responseFileName;\n\n    appendSummary(\"observe\", {\n      [`observe_inference_type`]: \"observe\",\n      timestamp: callTimestamp,\n      LLM_input_file: callFile,\n      LLM_output_file: responseFile,\n      prompt_tokens: promptTokens,\n      completion_tokens: completionTokens,\n      reasoning_tokens: reasoningTokens,\n      cached_input_tokens: cachedInputTokens,\n      inference_time_ms: usageTimeMs,\n    });\n  }\n\n  const parsedElements =\n    observeData.elements?.map((el) => {\n      const base = {\n        elementId: el.elementId,\n        description: String(el.description),\n        method: String(el.method),\n        arguments: el.arguments,\n      };\n      return base;\n    }) ?? [];\n\n  return {\n    elements: parsedElements,\n    prompt_tokens: promptTokens,\n    completion_tokens: completionTokens,\n    reasoning_tokens: reasoningTokens,\n    cached_input_tokens: cachedInputTokens,\n    inference_time_ms: usageTimeMs,\n  };\n}\n\nexport async function act({\n  instruction,\n  domElements,\n  llmClient,\n  userProvidedInstructions,\n  logger,\n  logInferenceToFile = false,\n}: {\n  instruction: string;\n  domElements: string;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n}) {\n  const actionSchema = z.object({\n    elementId: z\n      .string()\n      .regex(/^\\d+-\\d+$/)\n      .describe(\n        \"the ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'. for example, '0-76' or '16-21'\",\n      ),\n    description: z\n      .string()\n      .describe(\"a description of the accessible element and its purpose\"),\n    method: z\n      .enum(\n        // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly\n        Object.values(SupportedUnderstudyAction) as unknown as readonly [\n          string,\n          ...string[],\n        ],\n      )\n      .describe(\n        \"the candidate method/action to interact with the element. Select one of the available Understudy interaction methods.\",\n      ),\n    arguments: z.array(\n      z\n        .string()\n        .describe(\n          \"the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.\",\n        ),\n    ),\n  });\n\n  const actSchema = z.object({\n    // OpenAI strict mode requires additionalProperties: false on every object.\n    // The AI SDK's Zod 4 converter omits it inside anyOf, so set it in the JSON\n    // schema only; parsing still strips unknown keys. .meta() is Zod 4 only;\n    // the Zod 3 converter already emits it.\n    action: (typeof actionSchema.meta === \"function\"\n      ? actionSchema.meta({ additionalProperties: false })\n      : actionSchema\n    )\n      .nullable()\n      .describe(\n        \"The element to act on. Return null if no element on the page matches the instruction — do NOT fabricate or guess an element, and never emit empty strings or placeholder values.\",\n      ),\n    twoStep: z.boolean(),\n  });\n\n  type ActResponse = z.infer<typeof actSchema>;\n\n  const messages: ChatMessage[] = [\n    buildActSystemPrompt(userProvidedInstructions),\n    buildObserveUserMessage(instruction, domElements),\n  ];\n\n  let callTimestamp = \"\";\n  let callFile = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      `act_summary`,\n      `act_call`,\n      {\n        modelCall: \"act\",\n        messages,\n      },\n    );\n    callFile = fileName;\n    callTimestamp = timestamp;\n  }\n\n  const start = Date.now();\n  const rawResponse = await llmClient.createChatCompletion<ActResponse>({\n    options: {\n      messages,\n      response_model: {\n        schema: actSchema,\n        name: \"act\",\n      },\n      top_p: 1,\n      frequency_penalty: 0,\n      presence_penalty: 0,\n    },\n    logger,\n  });\n  const end = Date.now();\n  const usageTimeMs = end - start;\n\n  const { data: actData, usage: actUsage } = rawResponse;\n  const promptTokens = actUsage?.prompt_tokens ?? 0;\n  const completionTokens = actUsage?.completion_tokens ?? 0;\n  const reasoningTokens = actUsage?.reasoning_tokens ?? 0;\n  const cachedInputTokens = actUsage?.cached_input_tokens ?? 0;\n\n  let responseFile: string;\n  if (logInferenceToFile) {\n    const { fileName: responseFileName } = writeTimestampedTxtFile(\n      `act_summary`,\n      `act_response`,\n      {\n        modelResponse: \"act\",\n        rawResponse: actData,\n      },\n    );\n    responseFile = responseFileName;\n\n    appendSummary(\"act\", {\n      [`act_inference_type`]: \"act\",\n      timestamp: callTimestamp,\n      LLM_input_file: callFile,\n      LLM_output_file: responseFile,\n      prompt_tokens: promptTokens,\n      completion_tokens: completionTokens,\n      reasoning_tokens: reasoningTokens,\n      cached_input_tokens: cachedInputTokens,\n      inference_time_ms: usageTimeMs,\n    });\n  }\n\n  const parsedElement = actData.action\n    ? {\n        elementId: actData.action.elementId,\n        description: String(actData.action.description),\n        method: String(actData.action.method),\n        arguments: actData.action.arguments,\n      }\n    : undefined;\n\n  return {\n    element: parsedElement,\n    prompt_tokens: promptTokens,\n    completion_tokens: completionTokens,\n    reasoning_tokens: reasoningTokens,\n    cached_input_tokens: cachedInputTokens,\n    inference_time_ms: usageTimeMs,\n    twoStep: actData.twoStep,\n  };\n}\n"]}

package/dist/cjs/lib/v3/agent/tools/click.js

@@ -19,7 +19,7 @@ const clickTool = (v3, provider) => (0, ai_1.tool)({
     execute: async ({ describe, coordinates }) => {
         try {
             const page = await v3.context.awaitActivePage();
-            const processed = (0, coordinateNormalization_js_1.processCoordinates)(coordinates[0], coordinates[1], provider, v3);
+            const processed = await (0, coordinateNormalization_js_1.processCoordinates)(coordinates[0], coordinates[1], provider, v3);
             v3.logger({
                 category: "agent",
                 message: `Agent calling tool: click`,

package/dist/cjs/lib/v3/agent/tools/click.js.map

@@ -1 +1 @@
-{"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAOxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AAElE,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAA,SAAI,EAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,OAAC;aACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,0EAA0E;YAC1E,6EAA6E;YAC7E,4EAA4E;YAC5E,4EAA4E;YAC5E,MAAM,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;YAC9B,MAAM,KAAK,GAAG,QAAQ;gBACpB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;oBACvC,WAAW,EAAE,kBAAkB;iBAChC,CAAC;gBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;oBACzC,WAAW,EAAE,kBAAkB;iBAChC,CAAC,CAAC;YAEP,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,mEAAmE;wBACnE,oDAAoD;wBACpD,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO;wBAClC,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAmB,KAAK,CAAC,OAAO,EAAE;aAC1C,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC;AAlHQ,QAAA,SAAS,aAkHjB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  ClickToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: click`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        // On a touch session, actuate as a trusted tap rather than a mouse click:\n        // mobile layouts commonly gate their handlers on touch/pointer events, where\n        // a synthesized mouse click never registers (a size selector highlights but\n        // still reports \"please choose a size\"). Mirrors the CUA handler's routing.\n        const useTouch = v3.usesTouch;\n        const xpath = useTouch\n          ? await page.tap(processed.x, processed.y, {\n              returnXpath: shouldCollectXpath,\n            })\n          : await page.click(processed.x, processed.y, {\n              returnXpath: shouldCollectXpath,\n            });\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              // Replay the same input kind that was recorded, so a cached mobile\n              // run does not silently fall back to a mouse click.\n              method: useTouch ? \"tap\" : \"click\",\n              arguments: [],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          coordinates: [processed.x, processed.y],\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            coordinates: result.coordinates,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}
+{"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAOxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AAElE,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAA,SAAI,EAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,OAAC;aACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,MAAM,IAAA,+CAAkB,EACxC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,0EAA0E;YAC1E,6EAA6E;YAC7E,4EAA4E;YAC5E,4EAA4E;YAC5E,MAAM,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;YAC9B,MAAM,KAAK,GAAG,QAAQ;gBACpB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;oBACvC,WAAW,EAAE,kBAAkB;iBAChC,CAAC;gBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;oBACzC,WAAW,EAAE,kBAAkB;iBAChC,CAAC,CAAC;YAEP,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,mEAAmE;wBACnE,oDAAoD;wBACpD,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO;wBAClC,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAmB,KAAK,CAAC,OAAO,EAAE;aAC1C,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC;AAlHQ,QAAA,SAAS,aAkHjB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  ClickToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = await processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: click`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        // On a touch session, actuate as a trusted tap rather than a mouse click:\n        // mobile layouts commonly gate their handlers on touch/pointer events, where\n        // a synthesized mouse click never registers (a size selector highlights but\n        // still reports \"please choose a size\"). Mirrors the CUA handler's routing.\n        const useTouch = v3.usesTouch;\n        const xpath = useTouch\n          ? await page.tap(processed.x, processed.y, {\n              returnXpath: shouldCollectXpath,\n            })\n          : await page.click(processed.x, processed.y, {\n              returnXpath: shouldCollectXpath,\n            });\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              // Replay the same input kind that was recorded, so a cached mobile\n              // run does not silently fall back to a mouse click.\n              method: useTouch ? \"tap\" : \"click\",\n              arguments: [],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          coordinates: [processed.x, processed.y],\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            coordinates: result.coordinates,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/cjs/lib/v3/agent/tools/clickAndHold.js

@@ -21,7 +21,7 @@ const clickAndHoldTool = (v3, provider) => (0, ai_1.tool)({
     execute: async ({ describe, coordinates, duration }) => {
         try {
             const page = await v3.context.awaitActivePage();
-            const processed = (0, coordinateNormalization_js_1.processCoordinates)(coordinates[0], coordinates[1], provider, v3);
+            const processed = await (0, coordinateNormalization_js_1.processCoordinates)(coordinates[0], coordinates[1], provider, v3);
             v3.logger({
                 category: "agent",
                 message: `Agent calling tool: clickAndHold`,

package/dist/cjs/lib/v3/agent/tools/clickAndHold.js.map

@@ -1 +1 @@
-{"version":3,"file":"clickAndHold.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/clickAndHold.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAGxB,oFAAyE;AACzE,gDAAgD;AAEzC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAA,SAAI,EAAC;IACH,WAAW,EAAE,oDAAoD;IACjE,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CAAC,kDAAkD,CAAC;QAC/D,WAAW,EAAE,OAAC;aACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,EAAE;QACrD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,kCAAkC;gBAC3C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;4BACR,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YAEpD,sFAAsF;YACtF,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACpC,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACrD,CAAC;YAEF,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,cAAc;wBACtB,SAAS,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;qBAC9B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,+BAA+B,KAAK,CAAC,OAAO,EAAE;aACtD,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC,CAAC;AAhFQ,QAAA,gBAAgB,oBAgFxB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\n\nexport const clickAndHoldTool = (v3: V3, provider?: string) =>\n  tool({\n    description: \"Click and hold on an element using its coordinates\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      duration: z\n        .number()\n        .describe(\"The duration to hold the element in milliseconds\"),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates, duration }) => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: clickAndHold`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({\n                describe,\n                duration,\n              }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n\n        // Use dragAndDrop from same point to same point with delay to simulate click and hold\n        const [xpath] = await page.dragAndDrop(\n          processed.x,\n          processed.y,\n          processed.x,\n          processed.y,\n          { delay: duration, returnXpath: shouldCollectXpath },\n        );\n\n        // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"clickAndHold\",\n              arguments: [String(duration)],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return { success: true, describe };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking and holding: ${error.message}`,\n        };\n      }\n    },\n  });\n"]}
+{"version":3,"file":"clickAndHold.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/clickAndHold.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAGxB,oFAAyE;AACzE,gDAAgD;AAEzC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAA,SAAI,EAAC;IACH,WAAW,EAAE,oDAAoD;IACjE,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CAAC,kDAAkD,CAAC;QAC/D,WAAW,EAAE,OAAC;aACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,EAAE;QACrD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,MAAM,IAAA,+CAAkB,EACxC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,kCAAkC;gBAC3C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;4BACR,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YAEpD,sFAAsF;YACtF,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACpC,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACrD,CAAC;YAEF,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,cAAc;wBACtB,SAAS,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;qBAC9B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,+BAA+B,KAAK,CAAC,OAAO,EAAE;aACtD,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC,CAAC;AAhFQ,QAAA,gBAAgB,oBAgFxB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\n\nexport const clickAndHoldTool = (v3: V3, provider?: string) =>\n  tool({\n    description: \"Click and hold on an element using its coordinates\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      duration: z\n        .number()\n        .describe(\"The duration to hold the element in milliseconds\"),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates, duration }) => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = await processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: clickAndHold`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({\n                describe,\n                duration,\n              }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n\n        // Use dragAndDrop from same point to same point with delay to simulate click and hold\n        const [xpath] = await page.dragAndDrop(\n          processed.x,\n          processed.y,\n          processed.x,\n          processed.y,\n          { delay: duration, returnXpath: shouldCollectXpath },\n        );\n\n        // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"clickAndHold\",\n              arguments: [String(duration)],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return { success: true, describe };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking and holding: ${error.message}`,\n        };\n      }\n    },\n  });\n"]}

package/dist/cjs/lib/v3/agent/tools/dragAndDrop.js

@@ -20,8 +20,8 @@ const dragAndDropTool = (v3, provider) => (0, ai_1.tool)({
     execute: async ({ describe, startCoordinates, endCoordinates, }) => {
         try {
             const page = await v3.context.awaitActivePage();
-            const processedStart = (0, coordinateNormalization_js_1.processCoordinates)(startCoordinates[0], startCoordinates[1], provider, v3);
-            const processedEnd = (0, coordinateNormalization_js_1.processCoordinates)(endCoordinates[0], endCoordinates[1], provider, v3);
+            const processedStart = await (0, coordinateNormalization_js_1.processCoordinates)(startCoordinates[0], startCoordinates[1], provider, v3);
+            const processedEnd = await (0, coordinateNormalization_js_1.processCoordinates)(endCoordinates[0], endCoordinates[1], provider, v3);
             v3.logger({
                 category: "agent",
                 message: `Agent calling tool: dragAndDrop`,

package/dist/cjs/lib/v3/agent/tools/dragAndDrop.js.map

@@ -1 +1 @@
-{"version":3,"file":"dragAndDrop.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/dragAndDrop.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAOxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AAElE,MAAM,eAAe,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC3D,IAAA,SAAI,EAAC;IACH,WAAW,EACT,kNAAkN;IACpN,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACtE,gBAAgB,EAAE,OAAC;aAChB,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,wDAAwD,CAAC;QACrE,cAAc,EAAE,OAAC;aACd,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oDAAoD,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,gBAAgB,EAChB,cAAc,GACf,EAAkC,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,IAAA,+CAAkB,EACvC,gBAAgB,CAAC,CAAC,CAAC,EACnB,gBAAgB,CAAC,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;YACF,MAAM,YAAY,GAAG,IAAA,+CAAkB,EACrC,cAAc,CAAC,CAAC,CAAC,EACjB,cAAc,CAAC,CAAC,CAAC,EACjB,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,iCAAiC;gBAC1C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACjD,cAAc,CAAC,CAAC,EAChB,cAAc,CAAC,CAAC,EAChB,YAAY,CAAC,CAAC,EACd,YAAY,CAAC,CAAC,EACd,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;YAE9D,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,cAAc,GAAG,IAAA,sBAAW,EAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,IAAA,sBAAW,EAAC,OAAO,CAAC,CAAC;gBAC1C,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,cAAc;wBACxB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,aAAa;wBACrB,SAAS,EAAE,CAAC,YAAY,CAAC;qBAC1B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAmB,KAAK,CAAC,OAAO,EAAE;aAC1C,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBAC1B,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC;AArHQ,QAAA,eAAe,mBAqHvB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  DragAndDropToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const dragAndDropTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z.string().describe(\"Describe the element to drag and drop\"),\n      startCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to start the drag and drop from\"),\n      endCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to end the drag and drop at\"),\n    }),\n    execute: async ({\n      describe,\n      startCoordinates,\n      endCoordinates,\n    }): Promise<DragAndDropToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processedStart = processCoordinates(\n          startCoordinates[0],\n          startCoordinates[1],\n          provider,\n          v3,\n        );\n        const processedEnd = processCoordinates(\n          endCoordinates[0],\n          endCoordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: dragAndDrop`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({\n                describe,\n              }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const [fromXpath, toXpath] = await page.dragAndDrop(\n          processedStart.x,\n          processedStart.y,\n          processedEnd.x,\n          processedEnd.y,\n          { returnXpath: shouldCollectXpath },\n        );\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedFrom = ensureXPath(fromXpath);\n          const normalizedTo = ensureXPath(toXpath);\n          if (normalizedFrom && normalizedTo) {\n            const action: Action = {\n              selector: normalizedFrom,\n              description: describe,\n              method: \"dragAndDrop\",\n              arguments: [normalizedTo],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error dragging: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}
+{"version":3,"file":"dragAndDrop.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/dragAndDrop.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAOxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AAElE,MAAM,eAAe,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC3D,IAAA,SAAI,EAAC;IACH,WAAW,EACT,kNAAkN;IACpN,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACtE,gBAAgB,EAAE,OAAC;aAChB,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,wDAAwD,CAAC;QACrE,cAAc,EAAE,OAAC;aACd,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oDAAoD,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,gBAAgB,EAChB,cAAc,GACf,EAAkC,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,MAAM,IAAA,+CAAkB,EAC7C,gBAAgB,CAAC,CAAC,CAAC,EACnB,gBAAgB,CAAC,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;YACF,MAAM,YAAY,GAAG,MAAM,IAAA,+CAAkB,EAC3C,cAAc,CAAC,CAAC,CAAC,EACjB,cAAc,CAAC,CAAC,CAAC,EACjB,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,iCAAiC;gBAC1C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACjD,cAAc,CAAC,CAAC,EAChB,cAAc,CAAC,CAAC,EAChB,YAAY,CAAC,CAAC,EACd,YAAY,CAAC,CAAC,EACd,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;YAE9D,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,cAAc,GAAG,IAAA,sBAAW,EAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,IAAA,sBAAW,EAAC,OAAO,CAAC,CAAC;gBAC1C,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,cAAc;wBACxB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,aAAa;wBACrB,SAAS,EAAE,CAAC,YAAY,CAAC;qBAC1B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAmB,KAAK,CAAC,OAAO,EAAE;aAC1C,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBAC1B,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC;AArHQ,QAAA,eAAe,mBAqHvB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  DragAndDropToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const dragAndDropTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z.string().describe(\"Describe the element to drag and drop\"),\n      startCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to start the drag and drop from\"),\n      endCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to end the drag and drop at\"),\n    }),\n    execute: async ({\n      describe,\n      startCoordinates,\n      endCoordinates,\n    }): Promise<DragAndDropToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processedStart = await processCoordinates(\n          startCoordinates[0],\n          startCoordinates[1],\n          provider,\n          v3,\n        );\n        const processedEnd = await processCoordinates(\n          endCoordinates[0],\n          endCoordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: dragAndDrop`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({\n                describe,\n              }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const [fromXpath, toXpath] = await page.dragAndDrop(\n          processedStart.x,\n          processedStart.y,\n          processedEnd.x,\n          processedEnd.y,\n          { returnXpath: shouldCollectXpath },\n        );\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedFrom = ensureXPath(fromXpath);\n          const normalizedTo = ensureXPath(toXpath);\n          if (normalizedFrom && normalizedTo) {\n            const action: Action = {\n              selector: normalizedFrom,\n              description: describe,\n              method: \"dragAndDrop\",\n              arguments: [normalizedTo],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error dragging: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/cjs/lib/v3/agent/tools/fillFormVision.js

@@ -49,19 +49,22 @@ MANDATORY USE CASES (always use fillFormVision for these):
         execute: async ({ fields }) => {
             try {
                 const page = await v3.context.awaitActivePage();
+                const viewport = (0, coordinateNormalization_js_1.isGoogleProvider)(provider)
+                    ? await v3.resolveViewport()
+                    : undefined;
                 // Process coordinates per field. Keep the original `value` (with any
                 // `%variableName%` tokens) so substituted secrets never leak through
                 // the tool result, replay cache, returned AgentResult.actions, or
                 // anything that logs the action. Variables are substituted only at
                 // typing time below.
-                const safeFields = fields.map((field) => {
-                    const processed = (0, coordinateNormalization_js_1.processCoordinates)(field.coordinates.x, field.coordinates.y, provider, v3);
+                const safeFields = await Promise.all(fields.map(async (field) => {
+                    const processed = await (0, coordinateNormalization_js_1.processCoordinates)(field.coordinates.x, field.coordinates.y, provider, v3, viewport);
                     return {
                         action: field.action,
                         value: field.value,
                         coordinates: { x: processed.x, y: processed.y },
                     };
-                });
+                }));
                 v3.logger({
                     category: "agent",
                     message: `Agent calling tool: fillFormVision`,

package/dist/cjs/lib/v3/agent/tools/fillFormVision.js.map

@@ -1 +1 @@
-{"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAQxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AACzE,wDAA4D;AAErD,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,OAAC;iBACN,KAAK,CACJ,OAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,OAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,OAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAEhD,qEAAqE;gBACrE,qEAAqE;gBACrE,kEAAkE;gBAClE,mEAAmE;gBACnE,qBAAqB;gBACrB,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBACtC,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;oBACF,OAAO;wBACL,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;oBAC/B,uEAAuE;oBACvE,uEAAuE;oBACvE,yEAAyE;oBACzE,wEAAwE;oBACxE,qEAAqE;oBACrE,MAAM,KAAK,GAAG,EAAE,CAAC,SAAS;wBACxB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE;4BACvD,WAAW,EAAE,kBAAkB;yBAChC,CAAC;wBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE;4BACzD,WAAW,EAAE,kBAAkB;yBAChC,CAAC,CAAC;oBACP,oEAAoE;oBACpE,iCAAiC;oBACjC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAA,kCAAmB,EAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;oBAE7D,wEAAwE;oBACxE,sEAAsE;oBACtE,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;6BACzB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,UAAU;oBAC/B,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAuB,KAAK,CAAC,OAAO,EAAE;iBAC9C,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gCACvB,KAAK,EAAE,MAAM,CAAC,KAAK;6BACpB,CAAC;yBACH;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AArLW,QAAA,kBAAkB,sBAqL7B","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  FillFormVisionToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n  v3: V3,\n  provider?: string,\n  variables?: Variables,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const valueDescription = hasVariables\n    ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"Text to type into the target field\";\n\n  return tool({\n    description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                \"Description of the typing action, e.g. 'type foo into the bar field'\",\n              ),\n            value: z.string().describe(valueDescription),\n            coordinates: z\n              .object({\n                x: z.number(),\n                y: z.number(),\n              })\n              .describe(\"Coordinates of the target field\"),\n          }),\n        )\n        .min(2, \"Provide at least two fields to fill\"),\n    }),\n    execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n\n        // Process coordinates per field. Keep the original `value` (with any\n        // `%variableName%` tokens) so substituted secrets never leak through\n        // the tool result, replay cache, returned AgentResult.actions, or\n        // anything that logs the action. Variables are substituted only at\n        // typing time below.\n        const safeFields = fields.map((field) => {\n          const processed = processCoordinates(\n            field.coordinates.x,\n            field.coordinates.y,\n            provider,\n            v3,\n          );\n          return {\n            action: field.action,\n            value: field.value,\n            coordinates: { x: processed.x, y: processed.y },\n          };\n        });\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillFormVision`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ fields }), // Don't log substituted values\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const actions: Action[] = [];\n\n        for (const field of safeFields) {\n          // Focus the field, only requesting XPath when caching is enabled. On a\n          // touch session this must be a trusted tap — a synthesized mouse click\n          // never registers on touch-gated mobile layouts, so the field would stay\n          // unfocused and the value go nowhere. The recorded step stays a \"type\",\n          // whose replay focuses via the selector rather than a pointer event.\n          const xpath = v3.usesTouch\n            ? await page.tap(field.coordinates.x, field.coordinates.y, {\n                returnXpath: shouldCollectXpath,\n              })\n            : await page.click(field.coordinates.x, field.coordinates.y, {\n                returnXpath: shouldCollectXpath,\n              });\n          // Substitute variables only at the moment of typing so the resolved\n          // value never leaves this scope.\n          await page.type(substituteVariables(field.value, variables));\n\n          // Build Action with XPath for deterministic replay (only when caching).\n          // Store the placeholder value so cache entries don't contain secrets.\n          if (shouldCollectXpath) {\n            const normalizedXpath = ensureXPath(xpath);\n            if (normalizedXpath) {\n              actions.push({\n                selector: normalizedXpath,\n                description: field.action,\n                method: \"type\",\n                arguments: [field.value],\n              });\n            }\n          }\n\n          // Small delay between fields\n          await new Promise((resolve) => setTimeout(resolve, 100));\n        }\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n        // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n        if (shouldCollectXpath && actions.length > 0) {\n          v3.recordAgentReplayStep({\n            type: \"act\",\n            instruction: `Fill ${fields.length} form fields`,\n            actions,\n            actionDescription: `Fill ${fields.length} form fields`,\n          });\n        }\n\n        return {\n          success: true,\n          playwrightArguments: safeFields,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error filling form: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [\n            {\n              type: \"text\",\n              text: JSON.stringify({\n                success: result.success,\n                error: result.error,\n              }),\n            },\n          ],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            fieldsCount: result.playwrightArguments?.length ?? 0,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}
+{"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAQxB,oFAG6C;AAC7C,gDAAgD;AAChD,wEAAyE;AACzE,wDAA4D;AAErD,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,OAAC;iBACN,KAAK,CACJ,OAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,OAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,OAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,QAAQ,GAAG,IAAA,6CAAgB,EAAC,QAAQ,CAAC;oBACzC,CAAC,CAAC,MAAM,EAAE,CAAC,eAAe,EAAE;oBAC5B,CAAC,CAAC,SAAS,CAAC;gBAEd,qEAAqE;gBACrE,qEAAqE;gBACrE,kEAAkE;gBAClE,mEAAmE;gBACnE,qBAAqB;gBACrB,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAClC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;oBACzB,MAAM,SAAS,GAAG,MAAM,IAAA,+CAAkB,EACxC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,EACF,QAAQ,CACT,CAAC;oBACF,OAAO;wBACL,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CACH,CAAC;gBAEF,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;oBAC/B,uEAAuE;oBACvE,uEAAuE;oBACvE,yEAAyE;oBACzE,wEAAwE;oBACxE,qEAAqE;oBACrE,MAAM,KAAK,GAAG,EAAE,CAAC,SAAS;wBACxB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE;4BACvD,WAAW,EAAE,kBAAkB;yBAChC,CAAC;wBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,EAAE;4BACzD,WAAW,EAAE,kBAAkB;yBAChC,CAAC,CAAC;oBACP,oEAAoE;oBACpE,iCAAiC;oBACjC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAA,kCAAmB,EAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;oBAE7D,wEAAwE;oBACxE,sEAAsE;oBACtE,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;6BACzB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,UAAU;oBAC/B,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAuB,KAAK,CAAC,OAAO,EAAE;iBAC9C,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gCACvB,KAAK,EAAE,MAAM,CAAC,KAAK;6BACpB,CAAC;yBACH;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AA3LW,QAAA,kBAAkB,sBA2L7B","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  FillFormVisionToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport {\n  isGoogleProvider,\n  processCoordinates,\n} from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n  v3: V3,\n  provider?: string,\n  variables?: Variables,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const valueDescription = hasVariables\n    ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"Text to type into the target field\";\n\n  return tool({\n    description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                \"Description of the typing action, e.g. 'type foo into the bar field'\",\n              ),\n            value: z.string().describe(valueDescription),\n            coordinates: z\n              .object({\n                x: z.number(),\n                y: z.number(),\n              })\n              .describe(\"Coordinates of the target field\"),\n          }),\n        )\n        .min(2, \"Provide at least two fields to fill\"),\n    }),\n    execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const viewport = isGoogleProvider(provider)\n          ? await v3.resolveViewport()\n          : undefined;\n\n        // Process coordinates per field. Keep the original `value` (with any\n        // `%variableName%` tokens) so substituted secrets never leak through\n        // the tool result, replay cache, returned AgentResult.actions, or\n        // anything that logs the action. Variables are substituted only at\n        // typing time below.\n        const safeFields = await Promise.all(\n          fields.map(async (field) => {\n            const processed = await processCoordinates(\n              field.coordinates.x,\n              field.coordinates.y,\n              provider,\n              v3,\n              viewport,\n            );\n            return {\n              action: field.action,\n              value: field.value,\n              coordinates: { x: processed.x, y: processed.y },\n            };\n          }),\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillFormVision`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ fields }), // Don't log substituted values\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const actions: Action[] = [];\n\n        for (const field of safeFields) {\n          // Focus the field, only requesting XPath when caching is enabled. On a\n          // touch session this must be a trusted tap — a synthesized mouse click\n          // never registers on touch-gated mobile layouts, so the field would stay\n          // unfocused and the value go nowhere. The recorded step stays a \"type\",\n          // whose replay focuses via the selector rather than a pointer event.\n          const xpath = v3.usesTouch\n            ? await page.tap(field.coordinates.x, field.coordinates.y, {\n                returnXpath: shouldCollectXpath,\n              })\n            : await page.click(field.coordinates.x, field.coordinates.y, {\n                returnXpath: shouldCollectXpath,\n              });\n          // Substitute variables only at the moment of typing so the resolved\n          // value never leaves this scope.\n          await page.type(substituteVariables(field.value, variables));\n\n          // Build Action with XPath for deterministic replay (only when caching).\n          // Store the placeholder value so cache entries don't contain secrets.\n          if (shouldCollectXpath) {\n            const normalizedXpath = ensureXPath(xpath);\n            if (normalizedXpath) {\n              actions.push({\n                selector: normalizedXpath,\n                description: field.action,\n                method: \"type\",\n                arguments: [field.value],\n              });\n            }\n          }\n\n          // Small delay between fields\n          await new Promise((resolve) => setTimeout(resolve, 100));\n        }\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n        // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n        if (shouldCollectXpath && actions.length > 0) {\n          v3.recordAgentReplayStep({\n            type: \"act\",\n            instruction: `Fill ${fields.length} form fields`,\n            actions,\n            actionDescription: `Fill ${fields.length} form fields`,\n          });\n        }\n\n        return {\n          success: true,\n          playwrightArguments: safeFields,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error filling form: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [\n            {\n              type: \"text\",\n              text: JSON.stringify({\n                success: result.success,\n                error: result.error,\n              }),\n            },\n          ],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            fieldsCount: result.playwrightArguments?.length ?? 0,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}

package/dist/cjs/lib/v3/agent/tools/scroll.js

@@ -85,7 +85,7 @@ const scrollVisionTool = (v3, provider) => (0, ai_1.tool)({
         let cx;
         let cy;
         if (coordinates) {
-            const processed = (0, coordinateNormalization_js_1.processCoordinates)(coordinates[0], coordinates[1], provider, v3);
+            const processed = await (0, coordinateNormalization_js_1.processCoordinates)(coordinates[0], coordinates[1], provider, v3);
             cx = processed.x;
             cy = processed.y;
         }

package/dist/cjs/lib/v3/agent/tools/scroll.js.map

@@ -1 +1 @@
-{"version":3,"file":"scroll.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/scroll.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAOxB,oFAAyE;AACzE,wEAAyE;AAEzE;;;GAGG;AACI,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,EAAE,CACnC,IAAA,SAAI,EAAC;IACH,WAAW,EACT,iJAAiJ;IACnJ,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,UAAU,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,UAAU,GAAG,EAAE,GAChB,EAA6B,EAAE;QAC9B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;oBAChD,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,YAAY,UAAU,KAAK,SAAS,KAAK,cAAc,KAAK;YACrE,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;aACtC;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AApEQ,QAAA,UAAU,cAoElB;AAEL;;;GAGG;AACI,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAA,SAAI,EAAC;IACH,WAAW,EAAE,6SAA6S;IAC1T,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,WAAW,EAAE,OAAC;aACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,EAAE;aACV,QAAQ,CACP,4GAA4G,CAC7G;QACH,UAAU,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,WAAW,EACX,UAAU,GAAG,EAAE,GAChB,EAAmC,EAAE;QACpC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,iEAAiE;QACjE,IAAI,EAAU,CAAC;QACf,IAAI,EAAU,CAAC;QACf,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YACF,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;YACjB,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACvB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,SAAS;wBACT,WAAW;wBACX,UAAU;wBACV,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;qBACtB,CAAC;oBACF,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEnE,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,WAAW;gBAClB,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,QAAQ,EAAE,KAAK,EAAE,GAAG;gBAC1D,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,EAAE;YAC1C,cAAc,EAAE,cAAc;YAC9B,gBAAgB;SACjB,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;iBACtC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC;AA7GQ,QAAA,gBAAgB,oBA6GxB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type {\n  ScrollToolResult,\n  ScrollVisionToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\n/**\n * Simple scroll tool for DOM mode (non-grounding models).\n * No coordinates - scrolls from viewport center.\n */\nexport const scrollTool = (v3: V3) =>\n  tool({\n    description:\n      \"Scroll the page up or down by a percentage of the viewport height. Default is 80%, and what should be typically used for general page scrolling\",\n    inputSchema: z.object({\n      direction: z.enum([\"up\", \"down\"]),\n      percentage: z.number().min(1).max(200).optional(),\n    }),\n    execute: async ({\n      direction,\n      percentage = 80,\n    }): Promise<ScrollToolResult> => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: scroll`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({ direction, percentage }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const page = await v3.context.awaitActivePage();\n\n      const { w, h } = await page.mainFrame().evaluate<{\n        w: number;\n        h: number;\n      }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n      const scrollDistance = Math.round((h * percentage) / 100);\n      const cx = Math.floor(w / 2);\n      const cy = Math.floor(h / 2);\n      const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n      await page.scroll(cx, cy, 0, deltaY);\n\n      v3.recordAgentReplayStep({\n        type: \"scroll\",\n        deltaX: 0,\n        deltaY,\n        anchor: { x: cx, y: cy },\n      });\n\n      return {\n        success: true,\n        message: `Scrolled ${percentage}% ${direction} (${scrollDistance}px)`,\n        scrolledPixels: scrollDistance,\n      };\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      return {\n        type: \"json\",\n        value: {\n          success: result.success,\n          message: result.message,\n          scrolledPixels: result.scrolledPixels,\n        },\n      };\n    },\n  });\n\n/**\n * Scroll tool for hybrid mode (grounding models).\n * Supports optional coordinates for scrolling within nested scrollable elements.\n */\nexport const scrollVisionTool = (v3: V3, provider?: string) =>\n  tool({\n    description: `Scroll the page up or down. For general page scrolling, no coordinates needed. Only provide coordinates when scrolling inside a nested scrollable element (e.g., a dropdown menu, modal with overflow, or scrollable sidebar). Default is 80%, and what should be typically used for general page scrolling`,\n    inputSchema: z.object({\n      direction: z.enum([\"up\", \"down\"]),\n      coordinates: z\n        .array(z.number())\n        .optional()\n        .describe(\n          \"Only use coordinates for scrolling inside a nested scrollable element - provide (x, y) within that element\",\n        ),\n      percentage: z.number().min(1).max(200).optional(),\n    }),\n    execute: async ({\n      direction,\n      coordinates,\n      percentage = 80,\n    }): Promise<ScrollVisionToolResult> => {\n      const page = await v3.context.awaitActivePage();\n\n      const { w, h } = await page.mainFrame().evaluate<{\n        w: number;\n        h: number;\n      }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n      // Process coordinates if provided, otherwise use viewport center\n      let cx: number;\n      let cy: number;\n      if (coordinates) {\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n        cx = processed.x;\n        cy = processed.y;\n      } else {\n        cx = Math.floor(w / 2);\n        cy = Math.floor(h / 2);\n      }\n\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: scroll`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({\n              direction,\n              coordinates,\n              percentage,\n              processed: { cx, cy },\n            }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const scrollDistance = Math.round((h * percentage) / 100);\n      const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n      await page.scroll(cx, cy, 0, deltaY);\n\n      const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n      v3.recordAgentReplayStep({\n        type: \"scroll\",\n        deltaX: 0,\n        deltaY,\n        anchor: { x: cx, y: cy },\n      });\n\n      return {\n        success: true,\n        message: coordinates\n          ? `Scrolled ${percentage}% ${direction} at (${cx}, ${cy})`\n          : `Scrolled ${percentage}% ${direction}`,\n        scrolledPixels: scrollDistance,\n        screenshotBase64,\n      };\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            message: result.message,\n            scrolledPixels: result.scrolledPixels,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}
+{"version":3,"file":"scroll.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/scroll.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAOxB,oFAAyE;AACzE,wEAAyE;AAEzE;;;GAGG;AACI,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,EAAE,CACnC,IAAA,SAAI,EAAC;IACH,WAAW,EACT,iJAAiJ;IACnJ,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,UAAU,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,UAAU,GAAG,EAAE,GAChB,EAA6B,EAAE;QAC9B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;oBAChD,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,YAAY,UAAU,KAAK,SAAS,KAAK,cAAc,KAAK;YACrE,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;aACtC;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AApEQ,QAAA,UAAU,cAoElB;AAEL;;;GAGG;AACI,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAA,SAAI,EAAC;IACH,WAAW,EAAE,6SAA6S;IAC1T,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,WAAW,EAAE,OAAC;aACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,EAAE;aACV,QAAQ,CACP,4GAA4G,CAC7G;QACH,UAAU,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,WAAW,EACX,UAAU,GAAG,EAAE,GAChB,EAAmC,EAAE;QACpC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,iEAAiE;QACjE,IAAI,EAAU,CAAC;QACf,IAAI,EAAU,CAAC;QACf,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,SAAS,GAAG,MAAM,IAAA,+CAAkB,EACxC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YACF,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;YACjB,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACvB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,SAAS;wBACT,WAAW;wBACX,UAAU;wBACV,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;qBACtB,CAAC;oBACF,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEnE,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,WAAW;gBAClB,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,QAAQ,EAAE,KAAK,EAAE,GAAG;gBAC1D,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,EAAE;YAC1C,cAAc,EAAE,cAAc;YAC9B,gBAAgB;SACjB,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;iBACtC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC;AA7GQ,QAAA,gBAAgB,oBA6GxB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type {\n  ScrollToolResult,\n  ScrollVisionToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\n/**\n * Simple scroll tool for DOM mode (non-grounding models).\n * No coordinates - scrolls from viewport center.\n */\nexport const scrollTool = (v3: V3) =>\n  tool({\n    description:\n      \"Scroll the page up or down by a percentage of the viewport height. Default is 80%, and what should be typically used for general page scrolling\",\n    inputSchema: z.object({\n      direction: z.enum([\"up\", \"down\"]),\n      percentage: z.number().min(1).max(200).optional(),\n    }),\n    execute: async ({\n      direction,\n      percentage = 80,\n    }): Promise<ScrollToolResult> => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: scroll`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({ direction, percentage }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const page = await v3.context.awaitActivePage();\n\n      const { w, h } = await page.mainFrame().evaluate<{\n        w: number;\n        h: number;\n      }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n      const scrollDistance = Math.round((h * percentage) / 100);\n      const cx = Math.floor(w / 2);\n      const cy = Math.floor(h / 2);\n      const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n      await page.scroll(cx, cy, 0, deltaY);\n\n      v3.recordAgentReplayStep({\n        type: \"scroll\",\n        deltaX: 0,\n        deltaY,\n        anchor: { x: cx, y: cy },\n      });\n\n      return {\n        success: true,\n        message: `Scrolled ${percentage}% ${direction} (${scrollDistance}px)`,\n        scrolledPixels: scrollDistance,\n      };\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      return {\n        type: \"json\",\n        value: {\n          success: result.success,\n          message: result.message,\n          scrolledPixels: result.scrolledPixels,\n        },\n      };\n    },\n  });\n\n/**\n * Scroll tool for hybrid mode (grounding models).\n * Supports optional coordinates for scrolling within nested scrollable elements.\n */\nexport const scrollVisionTool = (v3: V3, provider?: string) =>\n  tool({\n    description: `Scroll the page up or down. For general page scrolling, no coordinates needed. Only provide coordinates when scrolling inside a nested scrollable element (e.g., a dropdown menu, modal with overflow, or scrollable sidebar). Default is 80%, and what should be typically used for general page scrolling`,\n    inputSchema: z.object({\n      direction: z.enum([\"up\", \"down\"]),\n      coordinates: z\n        .array(z.number())\n        .optional()\n        .describe(\n          \"Only use coordinates for scrolling inside a nested scrollable element - provide (x, y) within that element\",\n        ),\n      percentage: z.number().min(1).max(200).optional(),\n    }),\n    execute: async ({\n      direction,\n      coordinates,\n      percentage = 80,\n    }): Promise<ScrollVisionToolResult> => {\n      const page = await v3.context.awaitActivePage();\n\n      const { w, h } = await page.mainFrame().evaluate<{\n        w: number;\n        h: number;\n      }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n      // Process coordinates if provided, otherwise use viewport center\n      let cx: number;\n      let cy: number;\n      if (coordinates) {\n        const processed = await processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n        cx = processed.x;\n        cy = processed.y;\n      } else {\n        cx = Math.floor(w / 2);\n        cy = Math.floor(h / 2);\n      }\n\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: scroll`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({\n              direction,\n              coordinates,\n              percentage,\n              processed: { cx, cy },\n            }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const scrollDistance = Math.round((h * percentage) / 100);\n      const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n      await page.scroll(cx, cy, 0, deltaY);\n\n      const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n      v3.recordAgentReplayStep({\n        type: \"scroll\",\n        deltaX: 0,\n        deltaY,\n        anchor: { x: cx, y: cy },\n      });\n\n      return {\n        success: true,\n        message: coordinates\n          ? `Scrolled ${percentage}% ${direction} at (${cx}, ${cy})`\n          : `Scrolled ${percentage}% ${direction}`,\n        scrolledPixels: scrollDistance,\n        screenshotBase64,\n      };\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            message: result.message,\n            scrolledPixels: result.scrolledPixels,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/cjs/lib/v3/agent/tools/type.js

@@ -26,7 +26,7 @@ const typeTool = (v3, provider, variables) => {
         execute: async ({ describe, coordinates, text, }) => {
             try {
                 const page = await v3.context.awaitActivePage();
-                const processed = (0, coordinateNormalization_js_1.processCoordinates)(coordinates[0], coordinates[1], provider, v3);
+                const processed = await (0, coordinateNormalization_js_1.processCoordinates)(coordinates[0], coordinates[1], provider, v3);
                 // Substitute any %variableName% tokens in the text
                 const actualText = (0, variables_js_1.substituteVariables)(text, variables);
                 v3.logger({

package/dist/cjs/lib/v3/agent/tools/type.js.map

@@ -1 +1 @@
-{"version":3,"file":"type.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/type.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAQxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AACzE,wDAA4D;AAErD,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,SAAqB,EAAE,EAAE;IAC3E,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,YAAY;QAClC,CAAC,CAAC,oGAAoG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACzI,CAAC,CAAC,mCAAmC,CAAC;IAExC,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EACT,2QAA2Q;QAC7Q,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,QAAQ,EAAE,OAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,CACP,4HAA4H,CAC7H;YACH,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;YAC1C,WAAW,EAAE,OAAC;iBACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,QAAQ,CAAC,iDAAiD,CAAC;SAC/D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,WAAW,EACX,IAAI,GACL,EAA2B,EAAE;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;gBAEF,mDAAmD;gBACnD,MAAM,UAAU,GAAG,IAAA,kCAAmB,EAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAExD,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,0BAA0B;oBACnC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;4BACzC,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,yEAAyE;gBACzE,0EAA0E;gBAC1E,sEAAsE;gBACtE,2EAA2E;gBAC3E,wDAAwD;gBACxD,MAAM,KAAK,GAAG,EAAE,CAAC,SAAS;oBACxB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;wBACvC,WAAW,EAAE,kBAAkB;qBAChC,CAAC;oBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;wBACzC,WAAW,EAAE,kBAAkB;qBAChC,CAAC,CAAC;gBAEP,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE5B,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;gBAE9D,0FAA0F;gBAC1F,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;oBAC3C,IAAI,eAAe,EAAE,CAAC;wBACpB,MAAM,MAAM,GAAW;4BACrB,QAAQ,EAAE,eAAe;4BACzB,WAAW,EAAE,QAAQ;4BACrB,MAAM,EAAE,MAAM;4BACd,SAAS,EAAE,CAAC,IAAI,CAAC;yBAClB,CAAC;wBACF,EAAE,CAAC,qBAAqB,CAAC;4BACvB,IAAI,EAAE,KAAK;4BACX,WAAW,EAAE,QAAQ;4BACrB,OAAO,EAAE,CAAC,MAAM,CAAC;4BACjB,iBAAiB,EAAE,QAAQ;yBAC5B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,QAAQ;oBACR,IAAI,EAAE,8FAA8F;oBACpG,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,iBAAiB,KAAK,CAAC,OAAO,EAAE;iBACxC,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;iBACxD,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AAhIW,QAAA,QAAQ,YAgInB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  TypeToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const typeTool = (v3: V3, provider?: string, variables?: Variables) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const textDescription = hasVariables\n    ? `The text to type into the element. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"The text to type into the element\";\n\n  return tool({\n    description:\n      \"Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      text: z.string().describe(textDescription),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to type into the element\"),\n    }),\n    execute: async ({\n      describe,\n      coordinates,\n      text,\n    }): Promise<TypeToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        // Substitute any %variableName% tokens in the text\n        const actualText = substituteVariables(text, variables);\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: type`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe, text }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        // Focus the field the way the session's input actually works: on a touch\n        // session a synthesized mouse click never registers on touch-gated mobile\n        // layouts, so the field would stay unfocused and the text go nowhere.\n        // Only the actuation changes — the recorded step is a \"type\", whose replay\n        // focuses via the selector rather than a pointer event.\n        const xpath = v3.usesTouch\n          ? await page.tap(processed.x, processed.y, {\n              returnXpath: shouldCollectXpath,\n            })\n          : await page.click(processed.x, processed.y, {\n              returnXpath: shouldCollectXpath,\n            });\n\n        await page.type(actualText);\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"type\",\n              arguments: [text],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          text, // Return original text (with %variableName% tokens) to avoid exposing sensitive values to LLM\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error typing: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            text: result.text,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}
+{"version":3,"file":"type.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/type.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAQxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AACzE,wDAA4D;AAErD,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,SAAqB,EAAE,EAAE;IAC3E,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,YAAY;QAClC,CAAC,CAAC,oGAAoG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACzI,CAAC,CAAC,mCAAmC,CAAC;IAExC,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EACT,2QAA2Q;QAC7Q,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,QAAQ,EAAE,OAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,CACP,4HAA4H,CAC7H;YACH,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;YAC1C,WAAW,EAAE,OAAC;iBACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,QAAQ,CAAC,iDAAiD,CAAC;SAC/D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,WAAW,EACX,IAAI,GACL,EAA2B,EAAE;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,MAAM,IAAA,+CAAkB,EACxC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;gBAEF,mDAAmD;gBACnD,MAAM,UAAU,GAAG,IAAA,kCAAmB,EAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAExD,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,0BAA0B;oBACnC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;4BACzC,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,yEAAyE;gBACzE,0EAA0E;gBAC1E,sEAAsE;gBACtE,2EAA2E;gBAC3E,wDAAwD;gBACxD,MAAM,KAAK,GAAG,EAAE,CAAC,SAAS;oBACxB,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;wBACvC,WAAW,EAAE,kBAAkB;qBAChC,CAAC;oBACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;wBACzC,WAAW,EAAE,kBAAkB;qBAChC,CAAC,CAAC;gBAEP,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE5B,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;gBAE9D,0FAA0F;gBAC1F,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;oBAC3C,IAAI,eAAe,EAAE,CAAC;wBACpB,MAAM,MAAM,GAAW;4BACrB,QAAQ,EAAE,eAAe;4BACzB,WAAW,EAAE,QAAQ;4BACrB,MAAM,EAAE,MAAM;4BACd,SAAS,EAAE,CAAC,IAAI,CAAC;yBAClB,CAAC;wBACF,EAAE,CAAC,qBAAqB,CAAC;4BACvB,IAAI,EAAE,KAAK;4BACX,WAAW,EAAE,QAAQ;4BACrB,OAAO,EAAE,CAAC,MAAM,CAAC;4BACjB,iBAAiB,EAAE,QAAQ;yBAC5B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,QAAQ;oBACR,IAAI,EAAE,8FAA8F;oBACpG,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,iBAAiB,KAAK,CAAC,OAAO,EAAE;iBACxC,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;iBACxD,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AAhIW,QAAA,QAAQ,YAgInB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  TypeToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const typeTool = (v3: V3, provider?: string, variables?: Variables) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const textDescription = hasVariables\n    ? `The text to type into the element. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"The text to type into the element\";\n\n  return tool({\n    description:\n      \"Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      text: z.string().describe(textDescription),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to type into the element\"),\n    }),\n    execute: async ({\n      describe,\n      coordinates,\n      text,\n    }): Promise<TypeToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = await processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        // Substitute any %variableName% tokens in the text\n        const actualText = substituteVariables(text, variables);\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: type`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe, text }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        // Focus the field the way the session's input actually works: on a touch\n        // session a synthesized mouse click never registers on touch-gated mobile\n        // layouts, so the field would stay unfocused and the text go nowhere.\n        // Only the actuation changes — the recorded step is a \"type\", whose replay\n        // focuses via the selector rather than a pointer event.\n        const xpath = v3.usesTouch\n          ? await page.tap(processed.x, processed.y, {\n              returnXpath: shouldCollectXpath,\n            })\n          : await page.click(processed.x, processed.y, {\n              returnXpath: shouldCollectXpath,\n            });\n\n        await page.type(actualText);\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"type\",\n              arguments: [text],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          text, // Return original text (with %variableName% tokens) to avoid exposing sensitive values to LLM\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error typing: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            text: result.text,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}

package/dist/cjs/lib/v3/agent/utils/coordinateNormalization.d.ts

@@ -7,7 +7,10 @@ export declare function normalizeGoogleCoordinates(x: number, y: number, viewpor
     x: number;
     y: number;
 };
-export declare function processCoordinates(x: number, y: number, provider?: string, v3?: V3): {
+export declare function processCoordinates(x: number, y: number, provider?: string, v3?: V3, resolvedViewport?: {
+    width: number;
+    height: number;
+}): Promise<{
     x: number;
     y: number;
-};
+}>;

package/dist/cjs/lib/v3/agent/utils/coordinateNormalization.js

@@ -3,8 +3,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.isGoogleProvider = isGoogleProvider;
 exports.normalizeGoogleCoordinates = normalizeGoogleCoordinates;
 exports.processCoordinates = processCoordinates;
-// Default viewport for Browserbase managed fingerprinting mode
-const STEALTH_VIEWPORT = { width: 1288, height: 711 };
 function isGoogleProvider(provider) {
     if (!provider)
         return false;
@@ -20,11 +18,11 @@ function normalizeGoogleCoordinates(x, y, viewport) {
         y: Math.floor((clampedY / 1000) * viewport.height),
     };
 }
-function processCoordinates(x, y, provider, v3) {
-    if (isGoogleProvider(provider) && v3) {
-        // Browserbase managed fingerprinting uses a fixed viewport fallback.
-        const viewport = v3.isVerified ? STEALTH_VIEWPORT : v3.configuredViewport;
-        return normalizeGoogleCoordinates(x, y, viewport);
+async function processCoordinates(x, y, provider, v3, resolvedViewport) {
+    if (isGoogleProvider(provider)) {
+        const viewport = resolvedViewport ?? (v3 ? await v3.resolveViewport() : undefined);
+        if (viewport)
+            return normalizeGoogleCoordinates(x, y, viewport);
     }
     return { x, y };
 }

package/dist/cjs/lib/v3/agent/utils/coordinateNormalization.js.map

@@ -1 +1 @@
-{"version":3,"file":"coordinateNormalization.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/coordinateNormalization.ts"],"names":[],"mappings":";;AAKA,4CAGC;AAID,gEAWC;AAED,gDAYC;AAnCD,+DAA+D;AAC/D,MAAM,gBAAgB,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;AAEtD,SAAgB,gBAAgB,CAAC,QAAiB;IAChD,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5B,OAAO,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AACnD,CAAC;AAED,qEAAqE;AACrE,kCAAkC;AAClC,SAAgB,0BAA0B,CACxC,CAAS,EACT,CAAS,EACT,QAA2C;IAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO;QACL,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC;QACjD,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;KACnD,CAAC;AACJ,CAAC;AAED,SAAgB,kBAAkB,CAChC,CAAS,EACT,CAAS,EACT,QAAiB,EACjB,EAAO;IAEP,IAAI,gBAAgB,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QACrC,qEAAqE;QACrE,MAAM,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC;QAC1E,OAAO,0BAA0B,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;AAClB,CAAC","sourcesContent":["import type { V3 } from \"../../v3.js\";\n\n// Default viewport for Browserbase managed fingerprinting mode\nconst STEALTH_VIEWPORT = { width: 1288, height: 711 };\n\nexport function isGoogleProvider(provider?: string): boolean {\n  if (!provider) return false;\n  return provider.toLowerCase().includes(\"google\");\n}\n\n// Google returns coordinates in a 0-1000 range, we need to normalize\n// them to the viewport dimensions\nexport function normalizeGoogleCoordinates(\n  x: number,\n  y: number,\n  viewport: { width: number; height: number },\n): { x: number; y: number } {\n  const clampedX = Math.min(999, Math.max(0, x));\n  const clampedY = Math.min(999, Math.max(0, y));\n  return {\n    x: Math.floor((clampedX / 1000) * viewport.width),\n    y: Math.floor((clampedY / 1000) * viewport.height),\n  };\n}\n\nexport function processCoordinates(\n  x: number,\n  y: number,\n  provider?: string,\n  v3?: V3,\n): { x: number; y: number } {\n  if (isGoogleProvider(provider) && v3) {\n    // Browserbase managed fingerprinting uses a fixed viewport fallback.\n    const viewport = v3.isVerified ? STEALTH_VIEWPORT : v3.configuredViewport;\n    return normalizeGoogleCoordinates(x, y, viewport);\n  }\n  return { x, y };\n}\n"]}
+{"version":3,"file":"coordinateNormalization.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/coordinateNormalization.ts"],"names":[],"mappings":";;AAEA,4CAGC;AAID,gEAWC;AAED,gDAaC;AAjCD,SAAgB,gBAAgB,CAAC,QAAiB;IAChD,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5B,OAAO,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AACnD,CAAC;AAED,qEAAqE;AACrE,kCAAkC;AAClC,SAAgB,0BAA0B,CACxC,CAAS,EACT,CAAS,EACT,QAA2C;IAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO;QACL,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC;QACjD,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;KACnD,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,kBAAkB,CACtC,CAAS,EACT,CAAS,EACT,QAAiB,EACjB,EAAO,EACP,gBAAoD;IAEpD,IAAI,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/B,MAAM,QAAQ,GACZ,gBAAgB,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACpE,IAAI,QAAQ;YAAE,OAAO,0BAA0B,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;AAClB,CAAC","sourcesContent":["import type { V3 } from \"../../v3.js\";\n\nexport function isGoogleProvider(provider?: string): boolean {\n  if (!provider) return false;\n  return provider.toLowerCase().includes(\"google\");\n}\n\n// Google returns coordinates in a 0-1000 range, we need to normalize\n// them to the viewport dimensions\nexport function normalizeGoogleCoordinates(\n  x: number,\n  y: number,\n  viewport: { width: number; height: number },\n): { x: number; y: number } {\n  const clampedX = Math.min(999, Math.max(0, x));\n  const clampedY = Math.min(999, Math.max(0, y));\n  return {\n    x: Math.floor((clampedX / 1000) * viewport.width),\n    y: Math.floor((clampedY / 1000) * viewport.height),\n  };\n}\n\nexport async function processCoordinates(\n  x: number,\n  y: number,\n  provider?: string,\n  v3?: V3,\n  resolvedViewport?: { width: number; height: number },\n): Promise<{ x: number; y: number }> {\n  if (isGoogleProvider(provider)) {\n    const viewport =\n      resolvedViewport ?? (v3 ? await v3.resolveViewport() : undefined);\n    if (viewport) return normalizeGoogleCoordinates(x, y, viewport);\n  }\n  return { x, y };\n}\n"]}