@browserbasehq/orca 3.1.0-patch.4 → 3.2.0-preview.1

package/dist/cjs/lib/inference.js

@@ -9,10 +9,7 @@ const prompt_js_1 = require("./prompt.js");
 const inferenceLogUtils_js_1 = require("./inferenceLogUtils.js");
 const handlers_js_1 = require("./v3/types/private/handlers.js");
 function withLlmTimeout(promise, operation) {
-    const timeoutMs = (0, timeoutConfig_js_1.getEnvTimeoutMs)("LLM_MAX_MS");
-    if (!timeoutMs)
-        return promise;
-    return (0, timeoutConfig_js_1.withTimeout)(promise, timeoutMs, `LLM ${operation}`);
+    return (0, timeoutConfig_js_1.withTimeout)(promise, (0, timeoutConfig_js_1.getEnvTimeoutMs)("LLM_MAX_MS"), `LLM ${operation}`);
 }
 async function extract({ instruction, domElements, schema, llmClient, logger, userProvidedInstructions, logInferenceToFile = false, }) {
     const metadataSchema = zod_1.z.object({

package/dist/cjs/lib/inference.js.map

@@ -1 +1 @@
-{"version":3,"file":"inference.js","sourceRoot":"","sources":["../../../lib/inference.ts"],"names":[],"mappings":";;AA6BA,0BA8MC;AAED,0BAmJC;AAED,kBAuIC;AAzgBD,6BAAwB;AAGxB,4DAAqE;AACrE,2CAQqB;AACrB,iEAAgF;AAKhF,gEAA2E;AAK3E,SAAS,cAAc,CAAI,OAAmB,EAAE,SAAiB;IAC/D,MAAM,SAAS,GAAG,IAAA,kCAAe,EAAC,YAAY,CAAC,CAAC;IAChD,IAAI,CAAC,SAAS;QAAE,OAAO,OAAO,CAAC;IAC/B,OAAO,IAAA,8BAAW,EAAC,OAAO,EAAE,SAAS,EAAE,OAAO,SAAS,EAAE,CAAC,CAAC;AAC7D,CAAC;AAEM,KAAK,UAAU,OAAO,CAA+B,EAC1D,WAAW,EACX,WAAW,EACX,MAAM,EACN,SAAS,EACT,MAAM,EACN,wBAAwB,EACxB,kBAAkB,GAAG,KAAK,GAS3B;IACC,MAAM,cAAc,GAAG,OAAC,CAAC,MAAM,CAAC;QAC9B,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,oEAAoE,CACrE;QACH,SAAS,EAAE,OAAC;aACT,OAAO,EAAE;aACT,QAAQ,CACP,iHAAiH,CAClH;KACJ,CAAC,CAAC;IAKH,MAAM,gBAAgB,GAAG,SAAS,CAAC,IAAI,KAAK,WAAW,CAAC;IACxD,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,yEAAyE;IAE/H,MAAM,mBAAmB,GAAkB;QACzC,IAAA,oCAAwB,EAAC,gBAAgB,EAAE,wBAAwB,CAAC;QACpE,IAAA,kCAAsB,EAAC,WAAW,EAAE,WAAW,EAAE,gBAAgB,CAAC;KACnE,CAAC;IAEF,IAAI,eAAe,GAAG,EAAE,CAAC;IACzB,IAAI,oBAAoB,GAAG,EAAE,CAAC;IAC9B,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,iBAAiB,EACjB,cAAc,EACd;YACE,SAAS,EAAE,SAAS;YACpB,QAAQ,EAAE,mBAAmB;SAC9B,CACF,CAAC;QACF,eAAe,GAAG,QAAQ,CAAC;QAC3B,oBAAoB,GAAG,SAAS,CAAC;IACnC,CAAC;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,MAAM,kBAAkB,GAAG,MAAM,cAAc,CAC7C,SAAS,CAAC,oBAAoB,CAAqB;QACjD,OAAO,EAAE;YACP,QAAQ,EAAE,mBAAmB;YAC7B,cAAc,EAAE;gBACd,MAAM;gBACN,IAAI,EAAE,YAAY;aACnB;YACD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;YAC7B,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,EACF,SAAS,CACV,CAAC;IACF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,kBAAkB,CAAC;IAExE,IAAI,mBAA2B,CAAC;IAChC,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAA,8CAAuB,EAC1C,iBAAiB,EACjB,kBAAkB,EAClB;YACE,aAAa,EAAE,SAAS;YACxB,WAAW,EAAE,aAAa;SAC3B,CACF,CAAC;QACF,mBAAmB,GAAG,QAAQ,CAAC;QAE/B,IAAA,oCAAa,EAAC,SAAS,EAAE;YACvB,sBAAsB,EAAE,SAAS;YACjC,SAAS,EAAE,oBAAoB;YAC/B,cAAc,EAAE,eAAe;YAC/B,eAAe,EAAE,mBAAmB;YACpC,aAAa,EAAE,YAAY,EAAE,aAAa,IAAI,CAAC;YAC/C,iBAAiB,EAAE,YAAY,EAAE,iBAAiB,IAAI,CAAC;YACvD,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,IAAI,CAAC;YACrD,mBAAmB,EAAE,YAAY,EAAE,mBAAmB,IAAI,CAAC;YAC3D,iBAAiB,EAAE,cAAc,GAAG,gBAAgB;SACrD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,oBAAoB,GAAkB;QAC1C,IAAA,qCAAyB,GAAE;QAC3B,IAAA,+BAAmB,EAAC,WAAW,EAAE,aAAa,CAAC;KAChD,CAAC;IAEF,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAC1B,IAAI,qBAAqB,GAAG,EAAE,CAAC;IAC/B,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,iBAAiB,EACjB,eAAe,EACf;YACE,SAAS,EAAE,UAAU;YACrB,QAAQ,EAAE,oBAAoB;SAC/B,CACF,CAAC;QACF,gBAAgB,GAAG,QAAQ,CAAC;QAC5B,qBAAqB,GAAG,SAAS,CAAC;IACpC,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACrC,MAAM,gBAAgB,GAAG,MAAM,cAAc,CAC3C,SAAS,CAAC,oBAAoB,CAAmB;QAC/C,OAAO,EAAE;YACP,QAAQ,EAAE,oBAAoB;YAC9B,cAAc,EAAE;gBACd,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,cAAc;aACvB;YACD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;YAC7B,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,EACF,kBAAkB,CACnB,CAAC;IACF,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEnC,MAAM,EACJ,IAAI,EAAE,EACJ,SAAS,EAAE,yBAAyB,EACpC,QAAQ,EAAE,wBAAwB,GACnC,EACD,KAAK,EAAE,qBAAqB,GAC7B,GAAG,gBAAgB,CAAC;IAErB,IAAI,oBAA4B,CAAC;IACjC,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAA,8CAAuB,EAC1C,iBAAiB,EACjB,mBAAmB,EACnB;YACE,aAAa,EAAE,UAAU;YACzB,SAAS,EAAE,yBAAyB;YACpC,QAAQ,EAAE,wBAAwB;SACnC,CACF,CAAC;QACF,oBAAoB,GAAG,QAAQ,CAAC;QAEhC,IAAA,oCAAa,EAAC,SAAS,EAAE;YACvB,sBAAsB,EAAE,UAAU;YAClC,SAAS,EAAE,qBAAqB;YAChC,cAAc,EAAE,gBAAgB;YAChC,eAAe,EAAE,oBAAoB;YACrC,aAAa,EAAE,qBAAqB,EAAE,aAAa,IAAI,CAAC;YACxD,iBAAiB,EAAE,qBAAqB,EAAE,iBAAiB,IAAI,CAAC;YAChE,gBAAgB,EAAE,qBAAqB,EAAE,gBAAgB,IAAI,CAAC;YAC9D,mBAAmB,EAAE,qBAAqB,EAAE,mBAAmB,IAAI,CAAC;YACpE,iBAAiB,EAAE,eAAe,GAAG,iBAAiB;SACvD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,iBAAiB,GACrB,CAAC,YAAY,EAAE,aAAa,IAAI,CAAC,CAAC;QAClC,CAAC,qBAAqB,EAAE,aAAa,IAAI,CAAC,CAAC,CAAC;IAE9C,MAAM,qBAAqB,GACzB,CAAC,YAAY,EAAE,iBAAiB,IAAI,CAAC,CAAC;QACtC,CAAC,qBAAqB,EAAE,iBAAiB,IAAI,CAAC,CAAC,CAAC;IAElD,MAAM,oBAAoB,GACxB,cAAc,GAAG,gBAAgB,GAAG,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAAC;IAC5E,MAAM,oBAAoB,GACxB,CAAC,YAAY,EAAE,gBAAgB,IAAI,CAAC,CAAC;QACrC,CAAC,qBAAqB,EAAE,gBAAgB,IAAI,CAAC,CAAC,CAAC;IACjD,MAAM,sBAAsB,GAC1B,CAAC,YAAY,EAAE,mBAAmB,IAAI,CAAC,CAAC;QACxC,CAAC,qBAAqB,EAAE,mBAAmB,IAAI,CAAC,CAAC,CAAC;IAEpD,OAAO;QACL,GAAG,aAAa;QAChB,QAAQ,EAAE;YACR,SAAS,EAAE,yBAAyB;YACpC,QAAQ,EAAE,wBAAwB;SACnC;QACD,aAAa,EAAE,iBAAiB;QAChC,iBAAiB,EAAE,qBAAqB;QACxC,gBAAgB,EAAE,oBAAoB;QACtC,mBAAmB,EAAE,sBAAsB;QAC3C,iBAAiB,EAAE,oBAAoB;KACxC,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,OAAO,CAAC,EAC5B,WAAW,EACX,WAAW,EACX,SAAS,EACT,wBAAwB,EACxB,MAAM,EACN,kBAAkB,GAAG,KAAK,EAC1B,gBAAgB,GASjB;IACC,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,yEAAyE;IAE/H,MAAM,aAAa,GAAG,OAAC,CAAC,MAAM,CAAC;QAC7B,QAAQ,EAAE,OAAC;aACR,KAAK,CACJ,OAAC,CAAC,MAAM,CAAC;YACP,SAAS,EAAE,OAAC;iBACT,MAAM,EAAE;iBACR,KAAK,CAAC,WAAW,CAAC;iBAClB,QAAQ,CACP,6IAA6I,CAC9I;YACH,WAAW,EAAE,OAAC;iBACX,MAAM,EAAE;iBACR,QAAQ,CACP,yDAAyD,CAC1D;YACH,MAAM,EAAE,OAAC;iBACN,IAAI;YACH,yGAAyG;YACzG,MAAM,CAAC,MAAM,CAAC,uCAAyB,CAGtC,CACF;iBACA,QAAQ,CACP,uHAAuH,CACxH;YACH,SAAS,EAAE,OAAC,CAAC,KAAK,CAChB,OAAC;iBACE,MAAM,EAAE;iBACR,QAAQ,CACP,iJAAiJ,CAClJ,CACJ;SACF,CAAC,CACH;aACA,QAAQ,CAAC,4DAA4D,CAAC;KAC1E,CAAC,CAAC;IAIH,MAAM,QAAQ,GAAkB;QAC9B,IAAA,oCAAwB,EAAC,wBAAwB,EAAE,gBAAgB,CAAC;QACpE,IAAA,mCAAuB,EAAC,WAAW,EAAE,WAAW,CAAC;KAClD,CAAC;IAEF,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,iBAAiB,EACjB,cAAc,EACd;YACE,SAAS,EAAE,SAAS;YACpB,QAAQ;SACT,CACF,CAAC;QACF,QAAQ,GAAG,QAAQ,CAAC;QACpB,aAAa,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAkB;QACxE,OAAO,EAAE;YACP,QAAQ;YACR,cAAc,EAAE;gBACd,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,aAAa;aACpB;YACD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;YAC7B,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,WAAW,GAAG,GAAG,GAAG,KAAK,CAAC;IAEhC,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;IAC/D,MAAM,YAAY,GAAG,YAAY,EAAE,aAAa,IAAI,CAAC,CAAC;IACtD,MAAM,gBAAgB,GAAG,YAAY,EAAE,iBAAiB,IAAI,CAAC,CAAC;IAC9D,MAAM,eAAe,GAAG,YAAY,EAAE,gBAAgB,IAAI,CAAC,CAAC;IAC5D,MAAM,iBAAiB,GAAG,YAAY,EAAE,mBAAmB,IAAI,CAAC,CAAC;IAEjE,IAAI,YAAoB,CAAC;IACzB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAA,8CAAuB,EAC5D,iBAAiB,EACjB,kBAAkB,EAClB;YACE,aAAa,EAAE,SAAS;YACxB,WAAW,EAAE,WAAW;SACzB,CACF,CAAC;QACF,YAAY,GAAG,gBAAgB,CAAC;QAEhC,IAAA,oCAAa,EAAC,SAAS,EAAE;YACvB,CAAC,wBAAwB,CAAC,EAAE,SAAS;YACrC,SAAS,EAAE,aAAa;YACxB,cAAc,EAAE,QAAQ;YACxB,eAAe,EAAE,YAAY;YAC7B,aAAa,EAAE,YAAY;YAC3B,iBAAiB,EAAE,gBAAgB;YACnC,gBAAgB,EAAE,eAAe;YACjC,mBAAmB,EAAE,iBAAiB;YACtC,iBAAiB,EAAE,WAAW;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,cAAc,GAClB,WAAW,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG;YACX,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,WAAW,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC;YACnC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;YACzB,SAAS,EAAE,EAAE,CAAC,SAAS;SACxB,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,IAAI,EAAE,CAAC;IAEX,OAAO;QACL,QAAQ,EAAE,cAAc;QACxB,aAAa,EAAE,YAAY;QAC3B,iBAAiB,EAAE,gBAAgB;QACnC,gBAAgB,EAAE,eAAe;QACjC,mBAAmB,EAAE,iBAAiB;QACtC,iBAAiB,EAAE,WAAW;KAC/B,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,GAAG,CAAC,EACxB,WAAW,EACX,WAAW,EACX,SAAS,EACT,wBAAwB,EACxB,MAAM,EACN,kBAAkB,GAAG,KAAK,GAQ3B;IACC,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,yEAAyE;IAE/H,MAAM,SAAS,GAAG,OAAC,CAAC,MAAM,CAAC;QACzB,SAAS,EAAE,OAAC;aACT,MAAM,EAAE;aACR,KAAK,CAAC,WAAW,CAAC;aAClB,QAAQ,CACP,6IAA6I,CAC9I;QACH,WAAW,EAAE,OAAC;aACX,MAAM,EAAE;aACR,QAAQ,CAAC,yDAAyD,CAAC;QACtE,MAAM,EAAE,OAAC;aACN,IAAI;QACH,yGAAyG;QACzG,MAAM,CAAC,MAAM,CAAC,uCAAyB,CAGtC,CACF;aACA,QAAQ,CACP,uHAAuH,CACxH;QACH,SAAS,EAAE,OAAC,CAAC,KAAK,CAChB,OAAC;aACE,MAAM,EAAE;aACR,QAAQ,CACP,iJAAiJ,CAClJ,CACJ;QACD,OAAO,EAAE,OAAC,CAAC,OAAO,EAAE;KACrB,CAAC,CAAC;IAIH,MAAM,QAAQ,GAAkB;QAC9B,IAAA,gCAAoB,EAAC,wBAAwB,CAAC;QAC9C,IAAA,mCAAuB,EAAC,WAAW,EAAE,WAAW,CAAC;KAClD,CAAC;IAEF,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,aAAa,EACb,UAAU,EACV;YACE,SAAS,EAAE,KAAK;YAChB,QAAQ;SACT,CACF,CAAC;QACF,QAAQ,GAAG,QAAQ,CAAC;QACpB,aAAa,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAc;QACpE,OAAO,EAAE;YACP,QAAQ;YACR,cAAc,EAAE;gBACd,MAAM,EAAE,SAAS;gBACjB,IAAI,EAAE,KAAK;aACZ;YACD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;YAC7B,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,WAAW,GAAG,GAAG,GAAG,KAAK,CAAC;IAEhC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC;IACvD,MAAM,YAAY,GAAG,QAAQ,EAAE,aAAa,IAAI,CAAC,CAAC;IAClD,MAAM,gBAAgB,GAAG,QAAQ,EAAE,iBAAiB,IAAI,CAAC,CAAC;IAC1D,MAAM,eAAe,GAAG,QAAQ,EAAE,gBAAgB,IAAI,CAAC,CAAC;IACxD,MAAM,iBAAiB,GAAG,QAAQ,EAAE,mBAAmB,IAAI,CAAC,CAAC;IAE7D,IAAI,YAAoB,CAAC;IACzB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAA,8CAAuB,EAC5D,aAAa,EACb,cAAc,EACd;YACE,aAAa,EAAE,KAAK;YACpB,WAAW,EAAE,OAAO;SACrB,CACF,CAAC;QACF,YAAY,GAAG,gBAAgB,CAAC;QAEhC,IAAA,oCAAa,EAAC,KAAK,EAAE;YACnB,CAAC,oBAAoB,CAAC,EAAE,KAAK;YAC7B,SAAS,EAAE,aAAa;YACxB,cAAc,EAAE,QAAQ;YACxB,eAAe,EAAE,YAAY;YAC7B,aAAa,EAAE,YAAY;YAC3B,iBAAiB,EAAE,gBAAgB;YACnC,gBAAgB,EAAE,eAAe;YACjC,mBAAmB,EAAE,iBAAiB;YACtC,iBAAiB,EAAE,WAAW;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,aAAa,GAAG;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;QACxC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;QAC9B,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,aAAa;QACtB,aAAa,EAAE,YAAY;QAC3B,iBAAiB,EAAE,gBAAgB;QACnC,gBAAgB,EAAE,eAAe;QACjC,mBAAmB,EAAE,iBAAiB;QACtC,iBAAiB,EAAE,WAAW;QAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC;AACJ,CAAC","sourcesContent":["import { z } from \"zod\";\nimport { LogLine } from \"./v3/types/public/logs.js\";\nimport { ChatMessage, LLMClient } from \"./v3/llm/LLMClient.js\";\nimport { getEnvTimeoutMs, withTimeout } from \"./v3/timeoutConfig.js\";\nimport {\n  buildActSystemPrompt,\n  buildExtractSystemPrompt,\n  buildExtractUserPrompt,\n  buildMetadataPrompt,\n  buildMetadataSystemPrompt,\n  buildObserveSystemPrompt,\n  buildObserveUserMessage,\n} from \"./prompt.js\";\nimport { appendSummary, writeTimestampedTxtFile } from \"./inferenceLogUtils.js\";\nimport type {\n  InferStagehandSchema,\n  StagehandZodObject,\n} from \"./v3/zodCompat.js\";\nimport { SupportedUnderstudyAction } from \"./v3/types/private/handlers.js\";\n\n// Re-export for backward compatibility\nexport type { LLMParsedResponse, LLMUsage } from \"./v3/llm/LLMClient.js\";\n\nfunction withLlmTimeout<T>(promise: Promise<T>, operation: string): Promise<T> {\n  const timeoutMs = getEnvTimeoutMs(\"LLM_MAX_MS\");\n  if (!timeoutMs) return promise;\n  return withTimeout(promise, timeoutMs, `LLM ${operation}`);\n}\n\nexport async function extract<T extends StagehandZodObject>({\n  instruction,\n  domElements,\n  schema,\n  llmClient,\n  logger,\n  userProvidedInstructions,\n  logInferenceToFile = false,\n}: {\n  instruction: string;\n  domElements: string;\n  schema: T;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n}) {\n  const metadataSchema = z.object({\n    progress: z\n      .string()\n      .describe(\n        \"progress of what has been extracted so far, as concise as possible\",\n      ),\n    completed: z\n      .boolean()\n      .describe(\n        \"true if the goal is now accomplished. Use this conservatively, only when sure that the goal has been completed.\",\n      ),\n  });\n\n  type ExtractionResponse = InferStagehandSchema<T>;\n  type MetadataResponse = z.infer<typeof metadataSchema>;\n\n  const isUsingAnthropic = llmClient.type === \"anthropic\";\n  const isGPT5 = llmClient.modelName.includes(\"gpt-5\"); // TODO: remove this as we update support for gpt-5 configuration options\n\n  const extractCallMessages: ChatMessage[] = [\n    buildExtractSystemPrompt(isUsingAnthropic, userProvidedInstructions),\n    buildExtractUserPrompt(instruction, domElements, isUsingAnthropic),\n  ];\n\n  let extractCallFile = \"\";\n  let extractCallTimestamp = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"extract_call\",\n      {\n        modelCall: \"extract\",\n        messages: extractCallMessages,\n      },\n    );\n    extractCallFile = fileName;\n    extractCallTimestamp = timestamp;\n  }\n\n  const extractStartTime = Date.now();\n  const extractionResponse = await withLlmTimeout(\n    llmClient.createChatCompletion<ExtractionResponse>({\n      options: {\n        messages: extractCallMessages,\n        response_model: {\n          schema,\n          name: \"Extraction\",\n        },\n        temperature: isGPT5 ? 1 : 0.1,\n        top_p: 1,\n        frequency_penalty: 0,\n        presence_penalty: 0,\n      },\n      logger,\n    }),\n    \"extract\",\n  );\n  const extractEndTime = Date.now();\n\n  const { data: extractedData, usage: extractUsage } = extractionResponse;\n\n  let extractResponseFile: string;\n  if (logInferenceToFile) {\n    const { fileName } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"extract_response\",\n      {\n        modelResponse: \"extract\",\n        rawResponse: extractedData,\n      },\n    );\n    extractResponseFile = fileName;\n\n    appendSummary(\"extract\", {\n      extract_inference_type: \"extract\",\n      timestamp: extractCallTimestamp,\n      LLM_input_file: extractCallFile,\n      LLM_output_file: extractResponseFile,\n      prompt_tokens: extractUsage?.prompt_tokens ?? 0,\n      completion_tokens: extractUsage?.completion_tokens ?? 0,\n      reasoning_tokens: extractUsage?.reasoning_tokens ?? 0,\n      cached_input_tokens: extractUsage?.cached_input_tokens ?? 0,\n      inference_time_ms: extractEndTime - extractStartTime,\n    });\n  }\n\n  const metadataCallMessages: ChatMessage[] = [\n    buildMetadataSystemPrompt(),\n    buildMetadataPrompt(instruction, extractedData),\n  ];\n\n  let metadataCallFile = \"\";\n  let metadataCallTimestamp = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"metadata_call\",\n      {\n        modelCall: \"metadata\",\n        messages: metadataCallMessages,\n      },\n    );\n    metadataCallFile = fileName;\n    metadataCallTimestamp = timestamp;\n  }\n\n  const metadataStartTime = Date.now();\n  const metadataResponse = await withLlmTimeout(\n    llmClient.createChatCompletion<MetadataResponse>({\n      options: {\n        messages: metadataCallMessages,\n        response_model: {\n          name: \"Metadata\",\n          schema: metadataSchema,\n        },\n        temperature: isGPT5 ? 1 : 0.1,\n        top_p: 1,\n        frequency_penalty: 0,\n        presence_penalty: 0,\n      },\n      logger,\n    }),\n    \"extract metadata\",\n  );\n  const metadataEndTime = Date.now();\n\n  const {\n    data: {\n      completed: metadataResponseCompleted,\n      progress: metadataResponseProgress,\n    },\n    usage: metadataResponseUsage,\n  } = metadataResponse;\n\n  let metadataResponseFile: string;\n  if (logInferenceToFile) {\n    const { fileName } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"metadata_response\",\n      {\n        modelResponse: \"metadata\",\n        completed: metadataResponseCompleted,\n        progress: metadataResponseProgress,\n      },\n    );\n    metadataResponseFile = fileName;\n\n    appendSummary(\"extract\", {\n      extract_inference_type: \"metadata\",\n      timestamp: metadataCallTimestamp,\n      LLM_input_file: metadataCallFile,\n      LLM_output_file: metadataResponseFile,\n      prompt_tokens: metadataResponseUsage?.prompt_tokens ?? 0,\n      completion_tokens: metadataResponseUsage?.completion_tokens ?? 0,\n      reasoning_tokens: metadataResponseUsage?.reasoning_tokens ?? 0,\n      cached_input_tokens: metadataResponseUsage?.cached_input_tokens ?? 0,\n      inference_time_ms: metadataEndTime - metadataStartTime,\n    });\n  }\n\n  const totalPromptTokens =\n    (extractUsage?.prompt_tokens ?? 0) +\n    (metadataResponseUsage?.prompt_tokens ?? 0);\n\n  const totalCompletionTokens =\n    (extractUsage?.completion_tokens ?? 0) +\n    (metadataResponseUsage?.completion_tokens ?? 0);\n\n  const totalInferenceTimeMs =\n    extractEndTime - extractStartTime + (metadataEndTime - metadataStartTime);\n  const totalReasoningTokens =\n    (extractUsage?.reasoning_tokens ?? 0) +\n    (metadataResponseUsage?.reasoning_tokens ?? 0);\n  const totalCachedInputTokens =\n    (extractUsage?.cached_input_tokens ?? 0) +\n    (metadataResponseUsage?.cached_input_tokens ?? 0);\n\n  return {\n    ...extractedData,\n    metadata: {\n      completed: metadataResponseCompleted,\n      progress: metadataResponseProgress,\n    },\n    prompt_tokens: totalPromptTokens,\n    completion_tokens: totalCompletionTokens,\n    reasoning_tokens: totalReasoningTokens,\n    cached_input_tokens: totalCachedInputTokens,\n    inference_time_ms: totalInferenceTimeMs,\n  };\n}\n\nexport async function observe({\n  instruction,\n  domElements,\n  llmClient,\n  userProvidedInstructions,\n  logger,\n  logInferenceToFile = false,\n  supportedActions,\n}: {\n  instruction: string;\n  domElements: string;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n  supportedActions?: string[];\n}) {\n  const isGPT5 = llmClient.modelName.includes(\"gpt-5\"); // TODO: remove this as we update support for gpt-5 configuration options\n\n  const observeSchema = z.object({\n    elements: z\n      .array(\n        z.object({\n          elementId: z\n            .string()\n            .regex(/^\\d+-\\d+$/)\n            .describe(\n              \"the ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'.\",\n            ),\n          description: z\n            .string()\n            .describe(\n              \"a description of the accessible element and its purpose\",\n            ),\n          method: z\n            .enum(\n              // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly\n              Object.values(SupportedUnderstudyAction) as unknown as readonly [\n                string,\n                ...string[],\n              ],\n            )\n            .describe(\n              `the candidate method/action to interact with the element. Select one of the available Understudy interaction methods.`,\n            ),\n          arguments: z.array(\n            z\n              .string()\n              .describe(\n                \"the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.\",\n              ),\n          ),\n        }),\n      )\n      .describe(\"an array of accessible elements that match the instruction\"),\n  });\n\n  type ObserveResponse = z.infer<typeof observeSchema>;\n\n  const messages: ChatMessage[] = [\n    buildObserveSystemPrompt(userProvidedInstructions, supportedActions),\n    buildObserveUserMessage(instruction, domElements),\n  ];\n\n  let callTimestamp = \"\";\n  let callFile = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      `observe_summary`,\n      `observe_call`,\n      {\n        modelCall: \"observe\",\n        messages,\n      },\n    );\n    callFile = fileName;\n    callTimestamp = timestamp;\n  }\n\n  const start = Date.now();\n  const rawResponse = await llmClient.createChatCompletion<ObserveResponse>({\n    options: {\n      messages,\n      response_model: {\n        schema: observeSchema,\n        name: \"Observation\",\n      },\n      temperature: isGPT5 ? 1 : 0.1,\n      top_p: 1,\n      frequency_penalty: 0,\n      presence_penalty: 0,\n    },\n    logger,\n  });\n  const end = Date.now();\n  const usageTimeMs = end - start;\n\n  const { data: observeData, usage: observeUsage } = rawResponse;\n  const promptTokens = observeUsage?.prompt_tokens ?? 0;\n  const completionTokens = observeUsage?.completion_tokens ?? 0;\n  const reasoningTokens = observeUsage?.reasoning_tokens ?? 0;\n  const cachedInputTokens = observeUsage?.cached_input_tokens ?? 0;\n\n  let responseFile: string;\n  if (logInferenceToFile) {\n    const { fileName: responseFileName } = writeTimestampedTxtFile(\n      `observe_summary`,\n      `observe_response`,\n      {\n        modelResponse: \"observe\",\n        rawResponse: observeData,\n      },\n    );\n    responseFile = responseFileName;\n\n    appendSummary(\"observe\", {\n      [`observe_inference_type`]: \"observe\",\n      timestamp: callTimestamp,\n      LLM_input_file: callFile,\n      LLM_output_file: responseFile,\n      prompt_tokens: promptTokens,\n      completion_tokens: completionTokens,\n      reasoning_tokens: reasoningTokens,\n      cached_input_tokens: cachedInputTokens,\n      inference_time_ms: usageTimeMs,\n    });\n  }\n\n  const parsedElements =\n    observeData.elements?.map((el) => {\n      const base = {\n        elementId: el.elementId,\n        description: String(el.description),\n        method: String(el.method),\n        arguments: el.arguments,\n      };\n      return base;\n    }) ?? [];\n\n  return {\n    elements: parsedElements,\n    prompt_tokens: promptTokens,\n    completion_tokens: completionTokens,\n    reasoning_tokens: reasoningTokens,\n    cached_input_tokens: cachedInputTokens,\n    inference_time_ms: usageTimeMs,\n  };\n}\n\nexport async function act({\n  instruction,\n  domElements,\n  llmClient,\n  userProvidedInstructions,\n  logger,\n  logInferenceToFile = false,\n}: {\n  instruction: string;\n  domElements: string;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n}) {\n  const isGPT5 = llmClient.modelName.includes(\"gpt-5\"); // TODO: remove this as we update support for gpt-5 configuration options\n\n  const actSchema = z.object({\n    elementId: z\n      .string()\n      .regex(/^\\d+-\\d+$/)\n      .describe(\n        \"the ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'.\",\n      ),\n    description: z\n      .string()\n      .describe(\"a description of the accessible element and its purpose\"),\n    method: z\n      .enum(\n        // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly\n        Object.values(SupportedUnderstudyAction) as unknown as readonly [\n          string,\n          ...string[],\n        ],\n      )\n      .describe(\n        \"the candidate method/action to interact with the element. Select one of the available Understudy interaction methods.\",\n      ),\n    arguments: z.array(\n      z\n        .string()\n        .describe(\n          \"the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.\",\n        ),\n    ),\n    twoStep: z.boolean(),\n  });\n\n  type ActResponse = z.infer<typeof actSchema>;\n\n  const messages: ChatMessage[] = [\n    buildActSystemPrompt(userProvidedInstructions),\n    buildObserveUserMessage(instruction, domElements),\n  ];\n\n  let callTimestamp = \"\";\n  let callFile = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      `act_summary`,\n      `act_call`,\n      {\n        modelCall: \"act\",\n        messages,\n      },\n    );\n    callFile = fileName;\n    callTimestamp = timestamp;\n  }\n\n  const start = Date.now();\n  const rawResponse = await llmClient.createChatCompletion<ActResponse>({\n    options: {\n      messages,\n      response_model: {\n        schema: actSchema,\n        name: \"act\",\n      },\n      temperature: isGPT5 ? 1 : 0.1,\n      top_p: 1,\n      frequency_penalty: 0,\n      presence_penalty: 0,\n    },\n    logger,\n  });\n  const end = Date.now();\n  const usageTimeMs = end - start;\n\n  const { data: actData, usage: actUsage } = rawResponse;\n  const promptTokens = actUsage?.prompt_tokens ?? 0;\n  const completionTokens = actUsage?.completion_tokens ?? 0;\n  const reasoningTokens = actUsage?.reasoning_tokens ?? 0;\n  const cachedInputTokens = actUsage?.cached_input_tokens ?? 0;\n\n  let responseFile: string;\n  if (logInferenceToFile) {\n    const { fileName: responseFileName } = writeTimestampedTxtFile(\n      `act_summary`,\n      `act_response`,\n      {\n        modelResponse: \"act\",\n        rawResponse: actData,\n      },\n    );\n    responseFile = responseFileName;\n\n    appendSummary(\"act\", {\n      [`act_inference_type`]: \"act\",\n      timestamp: callTimestamp,\n      LLM_input_file: callFile,\n      LLM_output_file: responseFile,\n      prompt_tokens: promptTokens,\n      completion_tokens: completionTokens,\n      reasoning_tokens: reasoningTokens,\n      cached_input_tokens: cachedInputTokens,\n      inference_time_ms: usageTimeMs,\n    });\n  }\n\n  const parsedElement = {\n    elementId: actData.elementId,\n    description: String(actData.description),\n    method: String(actData.method),\n    arguments: actData.arguments,\n  };\n\n  return {\n    element: parsedElement,\n    prompt_tokens: promptTokens,\n    completion_tokens: completionTokens,\n    reasoning_tokens: reasoningTokens,\n    cached_input_tokens: cachedInputTokens,\n    inference_time_ms: usageTimeMs,\n    twoStep: actData.twoStep,\n  };\n}\n"]}
+{"version":3,"file":"inference.js","sourceRoot":"","sources":["../../../lib/inference.ts"],"names":[],"mappings":";;AA+BA,0BA8MC;AAED,0BAmJC;AAED,kBAuIC;AA3gBD,6BAAwB;AAGxB,4DAAqE;AACrE,2CAQqB;AACrB,iEAAgF;AAKhF,gEAA2E;AAK3E,SAAS,cAAc,CAAI,OAAmB,EAAE,SAAiB;IAC/D,OAAO,IAAA,8BAAW,EAChB,OAAO,EACP,IAAA,kCAAe,EAAC,YAAY,CAAC,EAC7B,OAAO,SAAS,EAAE,CACnB,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,OAAO,CAA+B,EAC1D,WAAW,EACX,WAAW,EACX,MAAM,EACN,SAAS,EACT,MAAM,EACN,wBAAwB,EACxB,kBAAkB,GAAG,KAAK,GAS3B;IACC,MAAM,cAAc,GAAG,OAAC,CAAC,MAAM,CAAC;QAC9B,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,oEAAoE,CACrE;QACH,SAAS,EAAE,OAAC;aACT,OAAO,EAAE;aACT,QAAQ,CACP,iHAAiH,CAClH;KACJ,CAAC,CAAC;IAKH,MAAM,gBAAgB,GAAG,SAAS,CAAC,IAAI,KAAK,WAAW,CAAC;IACxD,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,yEAAyE;IAE/H,MAAM,mBAAmB,GAAkB;QACzC,IAAA,oCAAwB,EAAC,gBAAgB,EAAE,wBAAwB,CAAC;QACpE,IAAA,kCAAsB,EAAC,WAAW,EAAE,WAAW,EAAE,gBAAgB,CAAC;KACnE,CAAC;IAEF,IAAI,eAAe,GAAG,EAAE,CAAC;IACzB,IAAI,oBAAoB,GAAG,EAAE,CAAC;IAC9B,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,iBAAiB,EACjB,cAAc,EACd;YACE,SAAS,EAAE,SAAS;YACpB,QAAQ,EAAE,mBAAmB;SAC9B,CACF,CAAC;QACF,eAAe,GAAG,QAAQ,CAAC;QAC3B,oBAAoB,GAAG,SAAS,CAAC;IACnC,CAAC;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,MAAM,kBAAkB,GAAG,MAAM,cAAc,CAC7C,SAAS,CAAC,oBAAoB,CAAqB;QACjD,OAAO,EAAE;YACP,QAAQ,EAAE,mBAAmB;YAC7B,cAAc,EAAE;gBACd,MAAM;gBACN,IAAI,EAAE,YAAY;aACnB;YACD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;YAC7B,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,EACF,SAAS,CACV,CAAC;IACF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,kBAAkB,CAAC;IAExE,IAAI,mBAA2B,CAAC;IAChC,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAA,8CAAuB,EAC1C,iBAAiB,EACjB,kBAAkB,EAClB;YACE,aAAa,EAAE,SAAS;YACxB,WAAW,EAAE,aAAa;SAC3B,CACF,CAAC;QACF,mBAAmB,GAAG,QAAQ,CAAC;QAE/B,IAAA,oCAAa,EAAC,SAAS,EAAE;YACvB,sBAAsB,EAAE,SAAS;YACjC,SAAS,EAAE,oBAAoB;YAC/B,cAAc,EAAE,eAAe;YAC/B,eAAe,EAAE,mBAAmB;YACpC,aAAa,EAAE,YAAY,EAAE,aAAa,IAAI,CAAC;YAC/C,iBAAiB,EAAE,YAAY,EAAE,iBAAiB,IAAI,CAAC;YACvD,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,IAAI,CAAC;YACrD,mBAAmB,EAAE,YAAY,EAAE,mBAAmB,IAAI,CAAC;YAC3D,iBAAiB,EAAE,cAAc,GAAG,gBAAgB;SACrD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,oBAAoB,GAAkB;QAC1C,IAAA,qCAAyB,GAAE;QAC3B,IAAA,+BAAmB,EAAC,WAAW,EAAE,aAAa,CAAC;KAChD,CAAC;IAEF,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAC1B,IAAI,qBAAqB,GAAG,EAAE,CAAC;IAC/B,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,iBAAiB,EACjB,eAAe,EACf;YACE,SAAS,EAAE,UAAU;YACrB,QAAQ,EAAE,oBAAoB;SAC/B,CACF,CAAC;QACF,gBAAgB,GAAG,QAAQ,CAAC;QAC5B,qBAAqB,GAAG,SAAS,CAAC;IACpC,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACrC,MAAM,gBAAgB,GAAG,MAAM,cAAc,CAC3C,SAAS,CAAC,oBAAoB,CAAmB;QAC/C,OAAO,EAAE;YACP,QAAQ,EAAE,oBAAoB;YAC9B,cAAc,EAAE;gBACd,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,cAAc;aACvB;YACD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;YAC7B,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,EACF,kBAAkB,CACnB,CAAC;IACF,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEnC,MAAM,EACJ,IAAI,EAAE,EACJ,SAAS,EAAE,yBAAyB,EACpC,QAAQ,EAAE,wBAAwB,GACnC,EACD,KAAK,EAAE,qBAAqB,GAC7B,GAAG,gBAAgB,CAAC;IAErB,IAAI,oBAA4B,CAAC;IACjC,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAA,8CAAuB,EAC1C,iBAAiB,EACjB,mBAAmB,EACnB;YACE,aAAa,EAAE,UAAU;YACzB,SAAS,EAAE,yBAAyB;YACpC,QAAQ,EAAE,wBAAwB;SACnC,CACF,CAAC;QACF,oBAAoB,GAAG,QAAQ,CAAC;QAEhC,IAAA,oCAAa,EAAC,SAAS,EAAE;YACvB,sBAAsB,EAAE,UAAU;YAClC,SAAS,EAAE,qBAAqB;YAChC,cAAc,EAAE,gBAAgB;YAChC,eAAe,EAAE,oBAAoB;YACrC,aAAa,EAAE,qBAAqB,EAAE,aAAa,IAAI,CAAC;YACxD,iBAAiB,EAAE,qBAAqB,EAAE,iBAAiB,IAAI,CAAC;YAChE,gBAAgB,EAAE,qBAAqB,EAAE,gBAAgB,IAAI,CAAC;YAC9D,mBAAmB,EAAE,qBAAqB,EAAE,mBAAmB,IAAI,CAAC;YACpE,iBAAiB,EAAE,eAAe,GAAG,iBAAiB;SACvD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,iBAAiB,GACrB,CAAC,YAAY,EAAE,aAAa,IAAI,CAAC,CAAC;QAClC,CAAC,qBAAqB,EAAE,aAAa,IAAI,CAAC,CAAC,CAAC;IAE9C,MAAM,qBAAqB,GACzB,CAAC,YAAY,EAAE,iBAAiB,IAAI,CAAC,CAAC;QACtC,CAAC,qBAAqB,EAAE,iBAAiB,IAAI,CAAC,CAAC,CAAC;IAElD,MAAM,oBAAoB,GACxB,cAAc,GAAG,gBAAgB,GAAG,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAAC;IAC5E,MAAM,oBAAoB,GACxB,CAAC,YAAY,EAAE,gBAAgB,IAAI,CAAC,CAAC;QACrC,CAAC,qBAAqB,EAAE,gBAAgB,IAAI,CAAC,CAAC,CAAC;IACjD,MAAM,sBAAsB,GAC1B,CAAC,YAAY,EAAE,mBAAmB,IAAI,CAAC,CAAC;QACxC,CAAC,qBAAqB,EAAE,mBAAmB,IAAI,CAAC,CAAC,CAAC;IAEpD,OAAO;QACL,GAAG,aAAa;QAChB,QAAQ,EAAE;YACR,SAAS,EAAE,yBAAyB;YACpC,QAAQ,EAAE,wBAAwB;SACnC;QACD,aAAa,EAAE,iBAAiB;QAChC,iBAAiB,EAAE,qBAAqB;QACxC,gBAAgB,EAAE,oBAAoB;QACtC,mBAAmB,EAAE,sBAAsB;QAC3C,iBAAiB,EAAE,oBAAoB;KACxC,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,OAAO,CAAC,EAC5B,WAAW,EACX,WAAW,EACX,SAAS,EACT,wBAAwB,EACxB,MAAM,EACN,kBAAkB,GAAG,KAAK,EAC1B,gBAAgB,GASjB;IACC,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,yEAAyE;IAE/H,MAAM,aAAa,GAAG,OAAC,CAAC,MAAM,CAAC;QAC7B,QAAQ,EAAE,OAAC;aACR,KAAK,CACJ,OAAC,CAAC,MAAM,CAAC;YACP,SAAS,EAAE,OAAC;iBACT,MAAM,EAAE;iBACR,KAAK,CAAC,WAAW,CAAC;iBAClB,QAAQ,CACP,6IAA6I,CAC9I;YACH,WAAW,EAAE,OAAC;iBACX,MAAM,EAAE;iBACR,QAAQ,CACP,yDAAyD,CAC1D;YACH,MAAM,EAAE,OAAC;iBACN,IAAI;YACH,yGAAyG;YACzG,MAAM,CAAC,MAAM,CAAC,uCAAyB,CAGtC,CACF;iBACA,QAAQ,CACP,uHAAuH,CACxH;YACH,SAAS,EAAE,OAAC,CAAC,KAAK,CAChB,OAAC;iBACE,MAAM,EAAE;iBACR,QAAQ,CACP,iJAAiJ,CAClJ,CACJ;SACF,CAAC,CACH;aACA,QAAQ,CAAC,4DAA4D,CAAC;KAC1E,CAAC,CAAC;IAIH,MAAM,QAAQ,GAAkB;QAC9B,IAAA,oCAAwB,EAAC,wBAAwB,EAAE,gBAAgB,CAAC;QACpE,IAAA,mCAAuB,EAAC,WAAW,EAAE,WAAW,CAAC;KAClD,CAAC;IAEF,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,iBAAiB,EACjB,cAAc,EACd;YACE,SAAS,EAAE,SAAS;YACpB,QAAQ;SACT,CACF,CAAC;QACF,QAAQ,GAAG,QAAQ,CAAC;QACpB,aAAa,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAkB;QACxE,OAAO,EAAE;YACP,QAAQ;YACR,cAAc,EAAE;gBACd,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,aAAa;aACpB;YACD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;YAC7B,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,WAAW,GAAG,GAAG,GAAG,KAAK,CAAC;IAEhC,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC;IAC/D,MAAM,YAAY,GAAG,YAAY,EAAE,aAAa,IAAI,CAAC,CAAC;IACtD,MAAM,gBAAgB,GAAG,YAAY,EAAE,iBAAiB,IAAI,CAAC,CAAC;IAC9D,MAAM,eAAe,GAAG,YAAY,EAAE,gBAAgB,IAAI,CAAC,CAAC;IAC5D,MAAM,iBAAiB,GAAG,YAAY,EAAE,mBAAmB,IAAI,CAAC,CAAC;IAEjE,IAAI,YAAoB,CAAC;IACzB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAA,8CAAuB,EAC5D,iBAAiB,EACjB,kBAAkB,EAClB;YACE,aAAa,EAAE,SAAS;YACxB,WAAW,EAAE,WAAW;SACzB,CACF,CAAC;QACF,YAAY,GAAG,gBAAgB,CAAC;QAEhC,IAAA,oCAAa,EAAC,SAAS,EAAE;YACvB,CAAC,wBAAwB,CAAC,EAAE,SAAS;YACrC,SAAS,EAAE,aAAa;YACxB,cAAc,EAAE,QAAQ;YACxB,eAAe,EAAE,YAAY;YAC7B,aAAa,EAAE,YAAY;YAC3B,iBAAiB,EAAE,gBAAgB;YACnC,gBAAgB,EAAE,eAAe;YACjC,mBAAmB,EAAE,iBAAiB;YACtC,iBAAiB,EAAE,WAAW;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,cAAc,GAClB,WAAW,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG;YACX,SAAS,EAAE,EAAE,CAAC,SAAS;YACvB,WAAW,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC;YACnC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;YACzB,SAAS,EAAE,EAAE,CAAC,SAAS;SACxB,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,IAAI,EAAE,CAAC;IAEX,OAAO;QACL,QAAQ,EAAE,cAAc;QACxB,aAAa,EAAE,YAAY;QAC3B,iBAAiB,EAAE,gBAAgB;QACnC,gBAAgB,EAAE,eAAe;QACjC,mBAAmB,EAAE,iBAAiB;QACtC,iBAAiB,EAAE,WAAW;KAC/B,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,GAAG,CAAC,EACxB,WAAW,EACX,WAAW,EACX,SAAS,EACT,wBAAwB,EACxB,MAAM,EACN,kBAAkB,GAAG,KAAK,GAQ3B;IACC,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,yEAAyE;IAE/H,MAAM,SAAS,GAAG,OAAC,CAAC,MAAM,CAAC;QACzB,SAAS,EAAE,OAAC;aACT,MAAM,EAAE;aACR,KAAK,CAAC,WAAW,CAAC;aAClB,QAAQ,CACP,6IAA6I,CAC9I;QACH,WAAW,EAAE,OAAC;aACX,MAAM,EAAE;aACR,QAAQ,CAAC,yDAAyD,CAAC;QACtE,MAAM,EAAE,OAAC;aACN,IAAI;QACH,yGAAyG;QACzG,MAAM,CAAC,MAAM,CAAC,uCAAyB,CAGtC,CACF;aACA,QAAQ,CACP,uHAAuH,CACxH;QACH,SAAS,EAAE,OAAC,CAAC,KAAK,CAChB,OAAC;aACE,MAAM,EAAE;aACR,QAAQ,CACP,iJAAiJ,CAClJ,CACJ;QACD,OAAO,EAAE,OAAC,CAAC,OAAO,EAAE;KACrB,CAAC,CAAC;IAIH,MAAM,QAAQ,GAAkB;QAC9B,IAAA,gCAAoB,EAAC,wBAAwB,CAAC;QAC9C,IAAA,mCAAuB,EAAC,WAAW,EAAE,WAAW,CAAC;KAClD,CAAC;IAEF,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,8CAAuB,EACrD,aAAa,EACb,UAAU,EACV;YACE,SAAS,EAAE,KAAK;YAChB,QAAQ;SACT,CACF,CAAC;QACF,QAAQ,GAAG,QAAQ,CAAC;QACpB,aAAa,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAc;QACpE,OAAO,EAAE;YACP,QAAQ;YACR,cAAc,EAAE;gBACd,MAAM,EAAE,SAAS;gBACjB,IAAI,EAAE,KAAK;aACZ;YACD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;YAC7B,KAAK,EAAE,CAAC;YACR,iBAAiB,EAAE,CAAC;YACpB,gBAAgB,EAAE,CAAC;SACpB;QACD,MAAM;KACP,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,WAAW,GAAG,GAAG,GAAG,KAAK,CAAC;IAEhC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC;IACvD,MAAM,YAAY,GAAG,QAAQ,EAAE,aAAa,IAAI,CAAC,CAAC;IAClD,MAAM,gBAAgB,GAAG,QAAQ,EAAE,iBAAiB,IAAI,CAAC,CAAC;IAC1D,MAAM,eAAe,GAAG,QAAQ,EAAE,gBAAgB,IAAI,CAAC,CAAC;IACxD,MAAM,iBAAiB,GAAG,QAAQ,EAAE,mBAAmB,IAAI,CAAC,CAAC;IAE7D,IAAI,YAAoB,CAAC;IACzB,IAAI,kBAAkB,EAAE,CAAC;QACvB,MAAM,EAAE,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAA,8CAAuB,EAC5D,aAAa,EACb,cAAc,EACd;YACE,aAAa,EAAE,KAAK;YACpB,WAAW,EAAE,OAAO;SACrB,CACF,CAAC;QACF,YAAY,GAAG,gBAAgB,CAAC;QAEhC,IAAA,oCAAa,EAAC,KAAK,EAAE;YACnB,CAAC,oBAAoB,CAAC,EAAE,KAAK;YAC7B,SAAS,EAAE,aAAa;YACxB,cAAc,EAAE,QAAQ;YACxB,eAAe,EAAE,YAAY;YAC7B,aAAa,EAAE,YAAY;YAC3B,iBAAiB,EAAE,gBAAgB;YACnC,gBAAgB,EAAE,eAAe;YACjC,mBAAmB,EAAE,iBAAiB;YACtC,iBAAiB,EAAE,WAAW;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,aAAa,GAAG;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;QACxC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;QAC9B,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,aAAa;QACtB,aAAa,EAAE,YAAY;QAC3B,iBAAiB,EAAE,gBAAgB;QACnC,gBAAgB,EAAE,eAAe;QACjC,mBAAmB,EAAE,iBAAiB;QACtC,iBAAiB,EAAE,WAAW;QAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC;AACJ,CAAC","sourcesContent":["import { z } from \"zod\";\nimport { LogLine } from \"./v3/types/public/logs.js\";\nimport { ChatMessage, LLMClient } from \"./v3/llm/LLMClient.js\";\nimport { getEnvTimeoutMs, withTimeout } from \"./v3/timeoutConfig.js\";\nimport {\n  buildActSystemPrompt,\n  buildExtractSystemPrompt,\n  buildExtractUserPrompt,\n  buildMetadataPrompt,\n  buildMetadataSystemPrompt,\n  buildObserveSystemPrompt,\n  buildObserveUserMessage,\n} from \"./prompt.js\";\nimport { appendSummary, writeTimestampedTxtFile } from \"./inferenceLogUtils.js\";\nimport type {\n  InferStagehandSchema,\n  StagehandZodObject,\n} from \"./v3/zodCompat.js\";\nimport { SupportedUnderstudyAction } from \"./v3/types/private/handlers.js\";\n\n// Re-export for backward compatibility\nexport type { LLMParsedResponse, LLMUsage } from \"./v3/llm/LLMClient.js\";\n\nfunction withLlmTimeout<T>(promise: Promise<T>, operation: string): Promise<T> {\n  return withTimeout(\n    promise,\n    getEnvTimeoutMs(\"LLM_MAX_MS\"),\n    `LLM ${operation}`,\n  );\n}\n\nexport async function extract<T extends StagehandZodObject>({\n  instruction,\n  domElements,\n  schema,\n  llmClient,\n  logger,\n  userProvidedInstructions,\n  logInferenceToFile = false,\n}: {\n  instruction: string;\n  domElements: string;\n  schema: T;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n}) {\n  const metadataSchema = z.object({\n    progress: z\n      .string()\n      .describe(\n        \"progress of what has been extracted so far, as concise as possible\",\n      ),\n    completed: z\n      .boolean()\n      .describe(\n        \"true if the goal is now accomplished. Use this conservatively, only when sure that the goal has been completed.\",\n      ),\n  });\n\n  type ExtractionResponse = InferStagehandSchema<T>;\n  type MetadataResponse = z.infer<typeof metadataSchema>;\n\n  const isUsingAnthropic = llmClient.type === \"anthropic\";\n  const isGPT5 = llmClient.modelName.includes(\"gpt-5\"); // TODO: remove this as we update support for gpt-5 configuration options\n\n  const extractCallMessages: ChatMessage[] = [\n    buildExtractSystemPrompt(isUsingAnthropic, userProvidedInstructions),\n    buildExtractUserPrompt(instruction, domElements, isUsingAnthropic),\n  ];\n\n  let extractCallFile = \"\";\n  let extractCallTimestamp = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"extract_call\",\n      {\n        modelCall: \"extract\",\n        messages: extractCallMessages,\n      },\n    );\n    extractCallFile = fileName;\n    extractCallTimestamp = timestamp;\n  }\n\n  const extractStartTime = Date.now();\n  const extractionResponse = await withLlmTimeout(\n    llmClient.createChatCompletion<ExtractionResponse>({\n      options: {\n        messages: extractCallMessages,\n        response_model: {\n          schema,\n          name: \"Extraction\",\n        },\n        temperature: isGPT5 ? 1 : 0.1,\n        top_p: 1,\n        frequency_penalty: 0,\n        presence_penalty: 0,\n      },\n      logger,\n    }),\n    \"extract\",\n  );\n  const extractEndTime = Date.now();\n\n  const { data: extractedData, usage: extractUsage } = extractionResponse;\n\n  let extractResponseFile: string;\n  if (logInferenceToFile) {\n    const { fileName } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"extract_response\",\n      {\n        modelResponse: \"extract\",\n        rawResponse: extractedData,\n      },\n    );\n    extractResponseFile = fileName;\n\n    appendSummary(\"extract\", {\n      extract_inference_type: \"extract\",\n      timestamp: extractCallTimestamp,\n      LLM_input_file: extractCallFile,\n      LLM_output_file: extractResponseFile,\n      prompt_tokens: extractUsage?.prompt_tokens ?? 0,\n      completion_tokens: extractUsage?.completion_tokens ?? 0,\n      reasoning_tokens: extractUsage?.reasoning_tokens ?? 0,\n      cached_input_tokens: extractUsage?.cached_input_tokens ?? 0,\n      inference_time_ms: extractEndTime - extractStartTime,\n    });\n  }\n\n  const metadataCallMessages: ChatMessage[] = [\n    buildMetadataSystemPrompt(),\n    buildMetadataPrompt(instruction, extractedData),\n  ];\n\n  let metadataCallFile = \"\";\n  let metadataCallTimestamp = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"metadata_call\",\n      {\n        modelCall: \"metadata\",\n        messages: metadataCallMessages,\n      },\n    );\n    metadataCallFile = fileName;\n    metadataCallTimestamp = timestamp;\n  }\n\n  const metadataStartTime = Date.now();\n  const metadataResponse = await withLlmTimeout(\n    llmClient.createChatCompletion<MetadataResponse>({\n      options: {\n        messages: metadataCallMessages,\n        response_model: {\n          name: \"Metadata\",\n          schema: metadataSchema,\n        },\n        temperature: isGPT5 ? 1 : 0.1,\n        top_p: 1,\n        frequency_penalty: 0,\n        presence_penalty: 0,\n      },\n      logger,\n    }),\n    \"extract metadata\",\n  );\n  const metadataEndTime = Date.now();\n\n  const {\n    data: {\n      completed: metadataResponseCompleted,\n      progress: metadataResponseProgress,\n    },\n    usage: metadataResponseUsage,\n  } = metadataResponse;\n\n  let metadataResponseFile: string;\n  if (logInferenceToFile) {\n    const { fileName } = writeTimestampedTxtFile(\n      \"extract_summary\",\n      \"metadata_response\",\n      {\n        modelResponse: \"metadata\",\n        completed: metadataResponseCompleted,\n        progress: metadataResponseProgress,\n      },\n    );\n    metadataResponseFile = fileName;\n\n    appendSummary(\"extract\", {\n      extract_inference_type: \"metadata\",\n      timestamp: metadataCallTimestamp,\n      LLM_input_file: metadataCallFile,\n      LLM_output_file: metadataResponseFile,\n      prompt_tokens: metadataResponseUsage?.prompt_tokens ?? 0,\n      completion_tokens: metadataResponseUsage?.completion_tokens ?? 0,\n      reasoning_tokens: metadataResponseUsage?.reasoning_tokens ?? 0,\n      cached_input_tokens: metadataResponseUsage?.cached_input_tokens ?? 0,\n      inference_time_ms: metadataEndTime - metadataStartTime,\n    });\n  }\n\n  const totalPromptTokens =\n    (extractUsage?.prompt_tokens ?? 0) +\n    (metadataResponseUsage?.prompt_tokens ?? 0);\n\n  const totalCompletionTokens =\n    (extractUsage?.completion_tokens ?? 0) +\n    (metadataResponseUsage?.completion_tokens ?? 0);\n\n  const totalInferenceTimeMs =\n    extractEndTime - extractStartTime + (metadataEndTime - metadataStartTime);\n  const totalReasoningTokens =\n    (extractUsage?.reasoning_tokens ?? 0) +\n    (metadataResponseUsage?.reasoning_tokens ?? 0);\n  const totalCachedInputTokens =\n    (extractUsage?.cached_input_tokens ?? 0) +\n    (metadataResponseUsage?.cached_input_tokens ?? 0);\n\n  return {\n    ...extractedData,\n    metadata: {\n      completed: metadataResponseCompleted,\n      progress: metadataResponseProgress,\n    },\n    prompt_tokens: totalPromptTokens,\n    completion_tokens: totalCompletionTokens,\n    reasoning_tokens: totalReasoningTokens,\n    cached_input_tokens: totalCachedInputTokens,\n    inference_time_ms: totalInferenceTimeMs,\n  };\n}\n\nexport async function observe({\n  instruction,\n  domElements,\n  llmClient,\n  userProvidedInstructions,\n  logger,\n  logInferenceToFile = false,\n  supportedActions,\n}: {\n  instruction: string;\n  domElements: string;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n  supportedActions?: string[];\n}) {\n  const isGPT5 = llmClient.modelName.includes(\"gpt-5\"); // TODO: remove this as we update support for gpt-5 configuration options\n\n  const observeSchema = z.object({\n    elements: z\n      .array(\n        z.object({\n          elementId: z\n            .string()\n            .regex(/^\\d+-\\d+$/)\n            .describe(\n              \"the ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'.\",\n            ),\n          description: z\n            .string()\n            .describe(\n              \"a description of the accessible element and its purpose\",\n            ),\n          method: z\n            .enum(\n              // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly\n              Object.values(SupportedUnderstudyAction) as unknown as readonly [\n                string,\n                ...string[],\n              ],\n            )\n            .describe(\n              `the candidate method/action to interact with the element. Select one of the available Understudy interaction methods.`,\n            ),\n          arguments: z.array(\n            z\n              .string()\n              .describe(\n                \"the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.\",\n              ),\n          ),\n        }),\n      )\n      .describe(\"an array of accessible elements that match the instruction\"),\n  });\n\n  type ObserveResponse = z.infer<typeof observeSchema>;\n\n  const messages: ChatMessage[] = [\n    buildObserveSystemPrompt(userProvidedInstructions, supportedActions),\n    buildObserveUserMessage(instruction, domElements),\n  ];\n\n  let callTimestamp = \"\";\n  let callFile = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      `observe_summary`,\n      `observe_call`,\n      {\n        modelCall: \"observe\",\n        messages,\n      },\n    );\n    callFile = fileName;\n    callTimestamp = timestamp;\n  }\n\n  const start = Date.now();\n  const rawResponse = await llmClient.createChatCompletion<ObserveResponse>({\n    options: {\n      messages,\n      response_model: {\n        schema: observeSchema,\n        name: \"Observation\",\n      },\n      temperature: isGPT5 ? 1 : 0.1,\n      top_p: 1,\n      frequency_penalty: 0,\n      presence_penalty: 0,\n    },\n    logger,\n  });\n  const end = Date.now();\n  const usageTimeMs = end - start;\n\n  const { data: observeData, usage: observeUsage } = rawResponse;\n  const promptTokens = observeUsage?.prompt_tokens ?? 0;\n  const completionTokens = observeUsage?.completion_tokens ?? 0;\n  const reasoningTokens = observeUsage?.reasoning_tokens ?? 0;\n  const cachedInputTokens = observeUsage?.cached_input_tokens ?? 0;\n\n  let responseFile: string;\n  if (logInferenceToFile) {\n    const { fileName: responseFileName } = writeTimestampedTxtFile(\n      `observe_summary`,\n      `observe_response`,\n      {\n        modelResponse: \"observe\",\n        rawResponse: observeData,\n      },\n    );\n    responseFile = responseFileName;\n\n    appendSummary(\"observe\", {\n      [`observe_inference_type`]: \"observe\",\n      timestamp: callTimestamp,\n      LLM_input_file: callFile,\n      LLM_output_file: responseFile,\n      prompt_tokens: promptTokens,\n      completion_tokens: completionTokens,\n      reasoning_tokens: reasoningTokens,\n      cached_input_tokens: cachedInputTokens,\n      inference_time_ms: usageTimeMs,\n    });\n  }\n\n  const parsedElements =\n    observeData.elements?.map((el) => {\n      const base = {\n        elementId: el.elementId,\n        description: String(el.description),\n        method: String(el.method),\n        arguments: el.arguments,\n      };\n      return base;\n    }) ?? [];\n\n  return {\n    elements: parsedElements,\n    prompt_tokens: promptTokens,\n    completion_tokens: completionTokens,\n    reasoning_tokens: reasoningTokens,\n    cached_input_tokens: cachedInputTokens,\n    inference_time_ms: usageTimeMs,\n  };\n}\n\nexport async function act({\n  instruction,\n  domElements,\n  llmClient,\n  userProvidedInstructions,\n  logger,\n  logInferenceToFile = false,\n}: {\n  instruction: string;\n  domElements: string;\n  llmClient: LLMClient;\n  userProvidedInstructions?: string;\n  logger: (message: LogLine) => void;\n  logInferenceToFile?: boolean;\n}) {\n  const isGPT5 = llmClient.modelName.includes(\"gpt-5\"); // TODO: remove this as we update support for gpt-5 configuration options\n\n  const actSchema = z.object({\n    elementId: z\n      .string()\n      .regex(/^\\d+-\\d+$/)\n      .describe(\n        \"the ID string associated with the element. Never include surrounding square brackets. This field must follow the format of 'number-number'.\",\n      ),\n    description: z\n      .string()\n      .describe(\"a description of the accessible element and its purpose\"),\n    method: z\n      .enum(\n        // Use Object.values() for Zod v3 compatibility - z.enum() in v3 doesn't accept TypeScript enums directly\n        Object.values(SupportedUnderstudyAction) as unknown as readonly [\n          string,\n          ...string[],\n        ],\n      )\n      .describe(\n        \"the candidate method/action to interact with the element. Select one of the available Understudy interaction methods.\",\n      ),\n    arguments: z.array(\n      z\n        .string()\n        .describe(\n          \"the arguments to pass to the method. For example, for a click, the arguments are empty, but for a fill, the arguments are the value to fill in.\",\n        ),\n    ),\n    twoStep: z.boolean(),\n  });\n\n  type ActResponse = z.infer<typeof actSchema>;\n\n  const messages: ChatMessage[] = [\n    buildActSystemPrompt(userProvidedInstructions),\n    buildObserveUserMessage(instruction, domElements),\n  ];\n\n  let callTimestamp = \"\";\n  let callFile = \"\";\n  if (logInferenceToFile) {\n    const { fileName, timestamp } = writeTimestampedTxtFile(\n      `act_summary`,\n      `act_call`,\n      {\n        modelCall: \"act\",\n        messages,\n      },\n    );\n    callFile = fileName;\n    callTimestamp = timestamp;\n  }\n\n  const start = Date.now();\n  const rawResponse = await llmClient.createChatCompletion<ActResponse>({\n    options: {\n      messages,\n      response_model: {\n        schema: actSchema,\n        name: \"act\",\n      },\n      temperature: isGPT5 ? 1 : 0.1,\n      top_p: 1,\n      frequency_penalty: 0,\n      presence_penalty: 0,\n    },\n    logger,\n  });\n  const end = Date.now();\n  const usageTimeMs = end - start;\n\n  const { data: actData, usage: actUsage } = rawResponse;\n  const promptTokens = actUsage?.prompt_tokens ?? 0;\n  const completionTokens = actUsage?.completion_tokens ?? 0;\n  const reasoningTokens = actUsage?.reasoning_tokens ?? 0;\n  const cachedInputTokens = actUsage?.cached_input_tokens ?? 0;\n\n  let responseFile: string;\n  if (logInferenceToFile) {\n    const { fileName: responseFileName } = writeTimestampedTxtFile(\n      `act_summary`,\n      `act_response`,\n      {\n        modelResponse: \"act\",\n        rawResponse: actData,\n      },\n    );\n    responseFile = responseFileName;\n\n    appendSummary(\"act\", {\n      [`act_inference_type`]: \"act\",\n      timestamp: callTimestamp,\n      LLM_input_file: callFile,\n      LLM_output_file: responseFile,\n      prompt_tokens: promptTokens,\n      completion_tokens: completionTokens,\n      reasoning_tokens: reasoningTokens,\n      cached_input_tokens: cachedInputTokens,\n      inference_time_ms: usageTimeMs,\n    });\n  }\n\n  const parsedElement = {\n    elementId: actData.elementId,\n    description: String(actData.description),\n    method: String(actData.method),\n    arguments: actData.arguments,\n  };\n\n  return {\n    element: parsedElement,\n    prompt_tokens: promptTokens,\n    completion_tokens: completionTokens,\n    reasoning_tokens: reasoningTokens,\n    cached_input_tokens: cachedInputTokens,\n    inference_time_ms: usageTimeMs,\n    twoStep: actData.twoStep,\n  };\n}\n"]}

package/dist/cjs/lib/v3/agent/AgentProvider.js

@@ -10,7 +10,6 @@ const MicrosoftCUAClient_js_1 = require("./MicrosoftCUAClient.js");
 exports.modelToAgentProviderMap = {
     "computer-use-preview": "openai",
     "computer-use-preview-2025-03-11": "openai",
-    "claude-3-7-sonnet-latest": "anthropic",
     "claude-sonnet-4-20250514": "anthropic",
     "claude-sonnet-4-5-20250929": "anthropic",
     "claude-opus-4-5-20251101": "anthropic",

package/dist/cjs/lib/v3/agent/AgentProvider.js.map

@@ -1 +1 @@
-{"version":3,"file":"AgentProvider.js","sourceRoot":"","sources":["../../../../../lib/v3/agent/AgentProvider.ts"],"names":[],"mappings":";;;AAIA,+DAGsC;AAEtC,mEAA6D;AAC7D,6DAAuD;AACvD,6DAAuD;AACvD,mEAA6D;AAE7D,0CAA0C;AAC7B,QAAA,uBAAuB,GAAsC;IACxE,sBAAsB,EAAE,QAAQ;IAChC,iCAAiC,EAAE,QAAQ;IAC3C,0BAA0B,EAAE,WAAW;IACvC,0BAA0B,EAAE,WAAW;IACvC,4BAA4B,EAAE,WAAW;IACzC,0BAA0B,EAAE,WAAW;IACvC,iBAAiB,EAAE,WAAW;IAC9B,mBAAmB,EAAE,WAAW;IAChC,2BAA2B,EAAE,WAAW;IACxC,yCAAyC,EAAE,QAAQ;IACnD,wBAAwB,EAAE,QAAQ;IAClC,sBAAsB,EAAE,QAAQ;IAChC,SAAS,EAAE,WAAW;CACvB,CAAC;AAEF;;;;GAIG;AACH,MAAa,aAAa;IAChB,MAAM,CAA6B;IAE3C;;OAEG;IACH,YAAY,MAAkC;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,SAAS,CACP,SAAiB,EACjB,aAA6B,EAC7B,wBAAiC,EACjC,KAAe;QAEf,uDAAuD;QACvD,MAAM,gBAAgB,GAAG,aAAa,EAAE,QAE3B,CAAC;QACd,MAAM,IAAI,GAAG,gBAAgB,IAAI,aAAa,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAE3E,IAAI,CAAC,MAAM,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,kCAAkC,IAAI,YAAY,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,EAAE;YACvH,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,QAAQ,IAAI,EAAE,CAAC;gBACb,KAAK,QAAQ;oBACX,OAAO,IAAI,oCAAe,CACxB,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,WAAW;oBACd,OAAO,IAAI,0CAAkB,CAC3B,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,QAAQ;oBACX,OAAO,IAAI,oCAAe,CACxB,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,WAAW;oBACd,OAAO,IAAI,0CAAkB,CAC3B,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,CACd,CAAC;gBACJ;oBACE,MAAM,IAAI,4CAA6B,CACrC,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,CAAC,EAC9C,oBAAoB,CACrB,CAAC;YACN,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,IAAI,CAAC,MAAM,CAAC;gBACV,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,gCAAgC,YAAY,EAAE;gBACvD,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,MAAM,CAAC,gBAAgB,CAAC,SAAiB;QACvC,MAAM,UAAU,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;YACxC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACzB,CAAC,CAAC,SAAS,CAAC;QAEd,IAAI,UAAU,IAAI,+BAAuB,EAAE,CAAC;YAC1C,OAAO,+BAAuB,CAAC,UAAU,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,IAAI,oCAAqB,CAC7B,MAAM,CAAC,IAAI,CAAC,+BAAuB,CAAC,EACpC,oBAAoB,CACrB,CAAC;IACJ,CAAC;CACF;AA7FD,sCA6FC","sourcesContent":["import { ToolSet } from \"ai/dist\";\nimport { AgentProviderType } from \"../types/public/agent.js\";\nimport { LogLine } from \"../types/public/logs.js\";\nimport { ClientOptions } from \"../types/public/model.js\";\nimport {\n  UnsupportedModelError,\n  UnsupportedModelProviderError,\n} from \"../types/public/sdkErrors.js\";\nimport { AgentClient } from \"./AgentClient.js\";\nimport { AnthropicCUAClient } from \"./AnthropicCUAClient.js\";\nimport { OpenAICUAClient } from \"./OpenAICUAClient.js\";\nimport { GoogleCUAClient } from \"./GoogleCUAClient.js\";\nimport { MicrosoftCUAClient } from \"./MicrosoftCUAClient.js\";\n\n// Map model names to their provider types\nexport const modelToAgentProviderMap: Record<string, AgentProviderType> = {\n  \"computer-use-preview\": \"openai\",\n  \"computer-use-preview-2025-03-11\": \"openai\",\n  \"claude-3-7-sonnet-latest\": \"anthropic\",\n  \"claude-sonnet-4-20250514\": \"anthropic\",\n  \"claude-sonnet-4-5-20250929\": \"anthropic\",\n  \"claude-opus-4-5-20251101\": \"anthropic\",\n  \"claude-opus-4-6\": \"anthropic\",\n  \"claude-sonnet-4-6\": \"anthropic\",\n  \"claude-haiku-4-5-20251001\": \"anthropic\",\n  \"gemini-2.5-computer-use-preview-10-2025\": \"google\",\n  \"gemini-3-flash-preview\": \"google\",\n  \"gemini-3-pro-preview\": \"google\",\n  \"fara-7b\": \"microsoft\",\n};\n\n/**\n * Provider for agent clients\n * This class is responsible for creating the appropriate agent client\n * based on the provider type\n */\nexport class AgentProvider {\n  private logger: (message: LogLine) => void;\n\n  /**\n   * Create a new agent provider\n   */\n  constructor(logger: (message: LogLine) => void) {\n    this.logger = logger;\n  }\n\n  getClient(\n    modelName: string,\n    clientOptions?: ClientOptions,\n    userProvidedInstructions?: string,\n    tools?: ToolSet,\n  ): AgentClient {\n    // Check if provider is explicitly set in clientOptions\n    const explicitProvider = clientOptions?.provider as\n      | AgentProviderType\n      | undefined;\n    const type = explicitProvider || AgentProvider.getAgentProvider(modelName);\n\n    this.logger({\n      category: \"agent\",\n      message: `Getting agent client for type: ${type}, model: ${modelName}${explicitProvider ? \" (explicit provider)\" : \"\"}`,\n      level: 2,\n    });\n\n    try {\n      switch (type) {\n        case \"openai\":\n          return new OpenAICUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"anthropic\":\n          return new AnthropicCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"google\":\n          return new GoogleCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"microsoft\":\n          return new MicrosoftCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n          );\n        default:\n          throw new UnsupportedModelProviderError(\n            [\"openai\", \"anthropic\", \"google\", \"microsoft\"],\n            \"Computer Use Agent\",\n          );\n      }\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      this.logger({\n        category: \"agent\",\n        message: `Error creating agent client: ${errorMessage}`,\n        level: 0,\n      });\n      throw error;\n    }\n  }\n\n  static getAgentProvider(modelName: string): AgentProviderType {\n    const normalized = modelName.includes(\"/\")\n      ? modelName.split(\"/\")[1]\n      : modelName;\n\n    if (normalized in modelToAgentProviderMap) {\n      return modelToAgentProviderMap[normalized];\n    }\n\n    throw new UnsupportedModelError(\n      Object.keys(modelToAgentProviderMap),\n      \"Computer Use Agent\",\n    );\n  }\n}\n"]}
+{"version":3,"file":"AgentProvider.js","sourceRoot":"","sources":["../../../../../lib/v3/agent/AgentProvider.ts"],"names":[],"mappings":";;;AAIA,+DAGsC;AAEtC,mEAA6D;AAC7D,6DAAuD;AACvD,6DAAuD;AACvD,mEAA6D;AAE7D,0CAA0C;AAC7B,QAAA,uBAAuB,GAAsC;IACxE,sBAAsB,EAAE,QAAQ;IAChC,iCAAiC,EAAE,QAAQ;IAC3C,0BAA0B,EAAE,WAAW;IACvC,4BAA4B,EAAE,WAAW;IACzC,0BAA0B,EAAE,WAAW;IACvC,iBAAiB,EAAE,WAAW;IAC9B,mBAAmB,EAAE,WAAW;IAChC,2BAA2B,EAAE,WAAW;IACxC,yCAAyC,EAAE,QAAQ;IACnD,wBAAwB,EAAE,QAAQ;IAClC,sBAAsB,EAAE,QAAQ;IAChC,SAAS,EAAE,WAAW;CACvB,CAAC;AAEF;;;;GAIG;AACH,MAAa,aAAa;IAChB,MAAM,CAA6B;IAE3C;;OAEG;IACH,YAAY,MAAkC;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,SAAS,CACP,SAAiB,EACjB,aAA6B,EAC7B,wBAAiC,EACjC,KAAe;QAEf,uDAAuD;QACvD,MAAM,gBAAgB,GAAG,aAAa,EAAE,QAE3B,CAAC;QACd,MAAM,IAAI,GAAG,gBAAgB,IAAI,aAAa,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAE3E,IAAI,CAAC,MAAM,CAAC;YACV,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,kCAAkC,IAAI,YAAY,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,EAAE;YACvH,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,QAAQ,IAAI,EAAE,CAAC;gBACb,KAAK,QAAQ;oBACX,OAAO,IAAI,oCAAe,CACxB,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,WAAW;oBACd,OAAO,IAAI,0CAAkB,CAC3B,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,QAAQ;oBACX,OAAO,IAAI,oCAAe,CACxB,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,EACb,KAAK,CACN,CAAC;gBACJ,KAAK,WAAW;oBACd,OAAO,IAAI,0CAAkB,CAC3B,IAAI,EACJ,SAAS,EACT,wBAAwB,EACxB,aAAa,CACd,CAAC;gBACJ;oBACE,MAAM,IAAI,4CAA6B,CACrC,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,CAAC,EAC9C,oBAAoB,CACrB,CAAC;YACN,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,IAAI,CAAC,MAAM,CAAC;gBACV,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,gCAAgC,YAAY,EAAE;gBACvD,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,MAAM,CAAC,gBAAgB,CAAC,SAAiB;QACvC,MAAM,UAAU,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;YACxC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACzB,CAAC,CAAC,SAAS,CAAC;QAEd,IAAI,UAAU,IAAI,+BAAuB,EAAE,CAAC;YAC1C,OAAO,+BAAuB,CAAC,UAAU,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,IAAI,oCAAqB,CAC7B,MAAM,CAAC,IAAI,CAAC,+BAAuB,CAAC,EACpC,oBAAoB,CACrB,CAAC;IACJ,CAAC;CACF;AA7FD,sCA6FC","sourcesContent":["import { ToolSet } from \"ai/dist\";\nimport { AgentProviderType } from \"../types/public/agent.js\";\nimport { LogLine } from \"../types/public/logs.js\";\nimport { ClientOptions } from \"../types/public/model.js\";\nimport {\n  UnsupportedModelError,\n  UnsupportedModelProviderError,\n} from \"../types/public/sdkErrors.js\";\nimport { AgentClient } from \"./AgentClient.js\";\nimport { AnthropicCUAClient } from \"./AnthropicCUAClient.js\";\nimport { OpenAICUAClient } from \"./OpenAICUAClient.js\";\nimport { GoogleCUAClient } from \"./GoogleCUAClient.js\";\nimport { MicrosoftCUAClient } from \"./MicrosoftCUAClient.js\";\n\n// Map model names to their provider types\nexport const modelToAgentProviderMap: Record<string, AgentProviderType> = {\n  \"computer-use-preview\": \"openai\",\n  \"computer-use-preview-2025-03-11\": \"openai\",\n  \"claude-sonnet-4-20250514\": \"anthropic\",\n  \"claude-sonnet-4-5-20250929\": \"anthropic\",\n  \"claude-opus-4-5-20251101\": \"anthropic\",\n  \"claude-opus-4-6\": \"anthropic\",\n  \"claude-sonnet-4-6\": \"anthropic\",\n  \"claude-haiku-4-5-20251001\": \"anthropic\",\n  \"gemini-2.5-computer-use-preview-10-2025\": \"google\",\n  \"gemini-3-flash-preview\": \"google\",\n  \"gemini-3-pro-preview\": \"google\",\n  \"fara-7b\": \"microsoft\",\n};\n\n/**\n * Provider for agent clients\n * This class is responsible for creating the appropriate agent client\n * based on the provider type\n */\nexport class AgentProvider {\n  private logger: (message: LogLine) => void;\n\n  /**\n   * Create a new agent provider\n   */\n  constructor(logger: (message: LogLine) => void) {\n    this.logger = logger;\n  }\n\n  getClient(\n    modelName: string,\n    clientOptions?: ClientOptions,\n    userProvidedInstructions?: string,\n    tools?: ToolSet,\n  ): AgentClient {\n    // Check if provider is explicitly set in clientOptions\n    const explicitProvider = clientOptions?.provider as\n      | AgentProviderType\n      | undefined;\n    const type = explicitProvider || AgentProvider.getAgentProvider(modelName);\n\n    this.logger({\n      category: \"agent\",\n      message: `Getting agent client for type: ${type}, model: ${modelName}${explicitProvider ? \" (explicit provider)\" : \"\"}`,\n      level: 2,\n    });\n\n    try {\n      switch (type) {\n        case \"openai\":\n          return new OpenAICUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"anthropic\":\n          return new AnthropicCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"google\":\n          return new GoogleCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n            tools,\n          );\n        case \"microsoft\":\n          return new MicrosoftCUAClient(\n            type,\n            modelName,\n            userProvidedInstructions,\n            clientOptions,\n          );\n        default:\n          throw new UnsupportedModelProviderError(\n            [\"openai\", \"anthropic\", \"google\", \"microsoft\"],\n            \"Computer Use Agent\",\n          );\n      }\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      this.logger({\n        category: \"agent\",\n        message: `Error creating agent client: ${errorMessage}`,\n        level: 0,\n      });\n      throw error;\n    }\n  }\n\n  static getAgentProvider(modelName: string): AgentProviderType {\n    const normalized = modelName.includes(\"/\")\n      ? modelName.split(\"/\")[1]\n      : modelName;\n\n    if (normalized in modelToAgentProviderMap) {\n      return modelToAgentProviderMap[normalized];\n    }\n\n    throw new UnsupportedModelError(\n      Object.keys(modelToAgentProviderMap),\n      \"Computer Use Agent\",\n    );\n  }\n}\n"]}

package/dist/cjs/lib/v3/agent/tools/act.d.ts

@@ -1,7 +1,7 @@
 import type { V3 } from "../../v3.js";
 import type { Action } from "../../types/public/methods.js";
 import type { AgentModelConfig, Variables } from "../../types/public/agent.js";
-export declare const actTool: (v3: V3, executionModel?: string | AgentModelConfig, variables?: Variables) => import("ai").Tool<{
+export declare const actTool: (v3: V3, executionModel?: string | AgentModelConfig, variables?: Variables, toolTimeout?: number) => import("ai").Tool<{
     action: string;
 }, {
     success: boolean;

package/dist/cjs/lib/v3/agent/tools/act.js

@@ -3,7 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.actTool = void 0;
 const ai_1 = require("ai");
 const zod_1 = require("zod");
-const actTool = (v3, executionModel, variables) => {
+const sdkErrors_js_1 = require("../../types/public/sdkErrors.js");
+const actTool = (v3, executionModel, variables, toolTimeout) => {
     const hasVariables = variables && Object.keys(variables).length > 0;
     const actionDescription = hasVariables
         ? `Describe what to click or type, e.g. "click the Login button" or "type %variableName% into the input". Available variables: ${Object.keys(variables).join(", ")}`
@@ -27,8 +28,8 @@ const actTool = (v3, executionModel, variables) => {
                     },
                 });
                 const options = executionModel
-                    ? { model: executionModel, variables }
-                    : { variables };
+                    ? { model: executionModel, variables, timeout: toolTimeout }
+                    : { variables, timeout: toolTimeout };
                 const result = await v3.act(action, options);
                 const actions = result.actions ?? [];
                 v3.recordAgentReplayStep({
@@ -50,7 +51,22 @@ const actTool = (v3, executionModel, variables) => {
                 return response;
             }
             catch (error) {
-                return { success: false, error: error?.message ?? String(error) };
+                if (error instanceof sdkErrors_js_1.TimeoutError) {
+                    const timeoutMessage = `TimeoutError while waiting for act() to complete (it may continue executing in the background)`;
+                    v3.logger({
+                        category: "agent",
+                        message: timeoutMessage,
+                        level: 0,
+                    });
+                    return {
+                        success: false,
+                        error: `${timeoutMessage} — try using a different description for the action`,
+                    };
+                }
+                return {
+                    success: false,
+                    error: error?.message ?? String(error),
+                };
             }
         },
     });

package/dist/cjs/lib/v3/agent/tools/act.js.map

@@ -1 +1 @@
-{"version":3,"file":"act.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/act.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAKjB,MAAM,OAAO,GAAG,CACrB,EAAM,EACN,cAA0C,EAC1C,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,iBAAiB,GAAG,YAAY;QACpC,CAAC,CAAC,+HAA+H,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACpK,CAAC,CAAC,0GAA0G,CAAC;IAE/G,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EACT,+GAA+G;QACjH,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;SAC/C,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC5B,IAAI,CAAC;gBACH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,yBAAyB;oBAClC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,MAAM;4BACb,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBACH,MAAM,OAAO,GAAG,cAAc;oBAC5B,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE;oBACtC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC;gBAClB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gBAC7C,MAAM,OAAO,GAAI,MAAM,CAAC,OAAgC,IAAI,EAAE,CAAC;gBAC/D,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,KAAK;oBACX,WAAW,EAAE,MAAM;oBACnB,OAAO;oBACP,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;oBAC3C,OAAO,EAAE,MAAM,CAAC,OAAO;iBACxB,CAAC,CAAC;gBACH,sDAAsD;gBACtD,6DAA6D;gBAC7D,MAAM,QAAQ,GAIV;oBACF,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;oBAC/B,MAAM,EAAE,MAAM,EAAE,iBAAiB,IAAI,MAAM;iBAC5C,CAAC;gBACF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,QAAQ,CAAC,mBAAmB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC5C,CAAC;gBACD,OAAO,QAAQ,CAAC;YAClB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YACpE,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AA5DW,QAAA,OAAO,WA4DlB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type { AgentModelConfig, Variables } from \"../../types/public/agent.js\";\n\nexport const actTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n  variables?: Variables,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const actionDescription = hasVariables\n    ? `Describe what to click or type, e.g. \"click the Login button\" or \"type %variableName% into the input\". Available variables: ${Object.keys(variables).join(\", \")}`\n    : 'Describe what to click or type, e.g. \"click the Login button\" or \"type \"John\" into the first name input\"';\n\n  return tool({\n    description:\n      \"Perform an action on the page (click, type). Provide a short, specific phrase that mentions the element type.\",\n    inputSchema: z.object({\n      action: z.string().describe(actionDescription),\n    }),\n    execute: async ({ action }) => {\n      try {\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: act`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: action,\n              type: \"string\",\n            },\n          },\n        });\n        const options = executionModel\n          ? { model: executionModel, variables }\n          : { variables };\n        const result = await v3.act(action, options);\n        const actions = (result.actions as Action[] | undefined) ?? [];\n        v3.recordAgentReplayStep({\n          type: \"act\",\n          instruction: action,\n          actions,\n          actionDescription: result.actionDescription,\n          message: result.message,\n        });\n        // Only include playwrightArguments when actions exist\n        // (undefined is not valid JSON and breaks AI SDK validation)\n        const response: {\n          success: boolean;\n          action: string;\n          playwrightArguments?: Action;\n        } = {\n          success: result.success ?? true,\n          action: result?.actionDescription ?? action,\n        };\n        if (actions.length > 0) {\n          response.playwrightArguments = actions[0];\n        }\n        return response;\n      } catch (error) {\n        return { success: false, error: error?.message ?? String(error) };\n      }\n    },\n  });\n};\n"]}
+{"version":3,"file":"act.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/act.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAIxB,kEAA+D;AAExD,MAAM,OAAO,GAAG,CACrB,EAAM,EACN,cAA0C,EAC1C,SAAqB,EACrB,WAAoB,EACpB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,iBAAiB,GAAG,YAAY;QACpC,CAAC,CAAC,+HAA+H,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACpK,CAAC,CAAC,0GAA0G,CAAC;IAE/G,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EACT,+GAA+G;QACjH,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;SAC/C,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC5B,IAAI,CAAC;gBACH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,yBAAyB;oBAClC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,MAAM;4BACb,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBACH,MAAM,OAAO,GAAG,cAAc;oBAC5B,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE;oBAC5D,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;gBAExC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gBAC7C,MAAM,OAAO,GAAI,MAAM,CAAC,OAAgC,IAAI,EAAE,CAAC;gBAC/D,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,KAAK;oBACX,WAAW,EAAE,MAAM;oBACnB,OAAO;oBACP,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;oBAC3C,OAAO,EAAE,MAAM,CAAC,OAAO;iBACxB,CAAC,CAAC;gBACH,sDAAsD;gBACtD,6DAA6D;gBAC7D,MAAM,QAAQ,GAIV;oBACF,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;oBAC/B,MAAM,EAAE,MAAM,EAAE,iBAAiB,IAAI,MAAM;iBAC5C,CAAC;gBACF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,QAAQ,CAAC,mBAAmB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC5C,CAAC;gBACD,OAAO,QAAQ,CAAC;YAClB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,2BAAY,EAAE,CAAC;oBAClC,MAAM,cAAc,GAAG,gGAAgG,CAAC;oBACxH,EAAE,CAAC,MAAM,CAAC;wBACR,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,cAAc;wBACvB,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,KAAK,EAAE,GAAG,cAAc,qDAAqD;qBAC9E,CAAC;gBACJ,CAAC;gBACD,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC;iBACvC,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AA7EW,QAAA,OAAO,WA6ElB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type { AgentModelConfig, Variables } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\nexport const actTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n  variables?: Variables,\n  toolTimeout?: number,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const actionDescription = hasVariables\n    ? `Describe what to click or type, e.g. \"click the Login button\" or \"type %variableName% into the input\". Available variables: ${Object.keys(variables).join(\", \")}`\n    : 'Describe what to click or type, e.g. \"click the Login button\" or \"type \"John\" into the first name input\"';\n\n  return tool({\n    description:\n      \"Perform an action on the page (click, type). Provide a short, specific phrase that mentions the element type.\",\n    inputSchema: z.object({\n      action: z.string().describe(actionDescription),\n    }),\n    execute: async ({ action }) => {\n      try {\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: act`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: action,\n              type: \"string\",\n            },\n          },\n        });\n        const options = executionModel\n          ? { model: executionModel, variables, timeout: toolTimeout }\n          : { variables, timeout: toolTimeout };\n\n        const result = await v3.act(action, options);\n        const actions = (result.actions as Action[] | undefined) ?? [];\n        v3.recordAgentReplayStep({\n          type: \"act\",\n          instruction: action,\n          actions,\n          actionDescription: result.actionDescription,\n          message: result.message,\n        });\n        // Only include playwrightArguments when actions exist\n        // (undefined is not valid JSON and breaks AI SDK validation)\n        const response: {\n          success: boolean;\n          action: string;\n          playwrightArguments?: Action;\n        } = {\n          success: result.success ?? true,\n          action: result?.actionDescription ?? action,\n        };\n        if (actions.length > 0) {\n          response.playwrightArguments = actions[0];\n        }\n        return response;\n      } catch (error) {\n        if (error instanceof TimeoutError) {\n          const timeoutMessage = `TimeoutError while waiting for act() to complete (it may continue executing in the background)`;\n          v3.logger({\n            category: \"agent\",\n            message: timeoutMessage,\n            level: 0,\n          });\n          return {\n            success: false,\n            error: `${timeoutMessage} — try using a different description for the action`,\n          };\n        }\n        return {\n          success: false,\n          error: error?.message ?? String(error),\n        };\n      }\n    },\n  });\n};\n"]}

package/dist/cjs/lib/v3/agent/tools/ariaTree.d.ts

@@ -1,5 +1,12 @@
 import type { V3 } from "../../v3.js";
-export declare const ariaTreeTool: (v3: V3) => import("ai").Tool<Record<string, never>, {
+export declare const ariaTreeTool: (v3: V3, toolTimeout?: number) => import("ai").Tool<Record<string, never>, {
+    success: boolean;
     content: string;
     pageUrl: string;
+    error?: undefined;
+} | {
+    content: string;
+    error: any;
+    success: boolean;
+    pageUrl: string;
 }>;

package/dist/cjs/lib/v3/agent/tools/ariaTree.js

@@ -3,33 +3,71 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.ariaTreeTool = void 0;
 const ai_1 = require("ai");
 const zod_1 = require("zod");
-const ariaTreeTool = (v3) => (0, ai_1.tool)({
+const sdkErrors_js_1 = require("../../types/public/sdkErrors.js");
+const ariaTreeTool = (v3, toolTimeout) => (0, ai_1.tool)({
     description: "gets the accessibility (ARIA) hybrid tree text for the current page. use this to understand structure and content.",
     inputSchema: zod_1.z.object({}),
     execute: async () => {
-        v3.logger({
-            category: "agent",
-            message: `Agent calling tool: ariaTree`,
-            level: 1,
-        });
-        const page = await v3.context.awaitActivePage();
-        const { pageText } = (await v3.extract());
-        const pageUrl = page.url();
-        let content = pageText;
-        const MAX_TOKENS = 70000; // rough cap, assume ~4 chars per token for conservative truncation
-        const estimatedTokens = Math.ceil(content.length / 4);
-        if (estimatedTokens > MAX_TOKENS) {
-            const maxChars = MAX_TOKENS * 4;
-            content =
-                content.substring(0, maxChars) +
-                    "\n\n[CONTENT TRUNCATED: Exceeded 70,000 token limit]";
+        try {
+            v3.logger({
+                category: "agent",
+                message: `Agent calling tool: ariaTree`,
+                level: 1,
+            });
+            const page = await v3.context.awaitActivePage();
+            const extractOptions = toolTimeout
+                ? { timeout: toolTimeout }
+                : undefined;
+            const { pageText } = (await v3.extract(extractOptions));
+            const pageUrl = page.url();
+            let content = pageText;
+            const MAX_TOKENS = 70000; // rough cap, assume ~4 chars per token for conservative truncation
+            const estimatedTokens = Math.ceil(content.length / 4);
+            if (estimatedTokens > MAX_TOKENS) {
+                const maxChars = MAX_TOKENS * 4;
+                content =
+                    content.substring(0, maxChars) +
+                        "\n\n[CONTENT TRUNCATED: Exceeded 70,000 token limit]";
+            }
+            return { success: true, content, pageUrl };
         }
-        return { content, pageUrl };
+        catch (error) {
+            if (error instanceof sdkErrors_js_1.TimeoutError) {
+                const timeoutMessage = `TimeoutError: ariaTree() timed out — the page may be too large`;
+                v3.logger({
+                    category: "agent",
+                    message: timeoutMessage,
+                    level: 0,
+                });
+                return {
+                    content: "",
+                    error: timeoutMessage,
+                    success: false,
+                    pageUrl: "",
+                };
+            }
+            return {
+                content: "",
+                error: error?.message ?? String(error),
+                success: false,
+                pageUrl: "",
+            };
+        }
+    },
+    toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
+        return {
+            type: "content",
+            value: [
+                { type: "text", text: `Accessibility Tree:\n${result.content}` },
+            ],
+        };
     },
-    toModelOutput: (result) => ({
-        type: "content",
-        value: [{ type: "text", text: `Accessibility Tree:\n${result.content}` }],
-    }),
 });
 exports.ariaTreeTool = ariaTreeTool;
 //# sourceMappingURL=ariaTree.js.map

package/dist/cjs/lib/v3/agent/tools/ariaTree.js.map

@@ -1 +1 @@
-{"version":3,"file":"ariaTree.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/ariaTree.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAGjB,MAAM,YAAY,GAAG,CAAC,EAAM,EAAE,EAAE,CACrC,IAAA,SAAI,EAAC;IACH,WAAW,EACT,oHAAoH;IACtH,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC,EAAE,CAAC;IACzB,OAAO,EAAE,KAAK,IAAI,EAAE;QAClB,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,8BAA8B;YACvC,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAChD,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,CAAyB,CAAC;QAClE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE3B,IAAI,OAAO,GAAG,QAAQ,CAAC;QACvB,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,mEAAmE;QAC7F,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACtD,IAAI,eAAe,GAAG,UAAU,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,UAAU,GAAG,CAAC,CAAC;YAChC,OAAO;gBACL,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC;oBAC9B,sDAAsD,CAAC;QAC3D,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC9B,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC1B,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,wBAAwB,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;KAC1E,CAAC;CACH,CAAC,CAAC;AA/BQ,QAAA,YAAY,gBA+BpB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const ariaTreeTool = (v3: V3) =>\n  tool({\n    description:\n      \"gets the accessibility (ARIA) hybrid tree text for the current page. use this to understand structure and content.\",\n    inputSchema: z.object({}),\n    execute: async () => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: ariaTree`,\n        level: 1,\n      });\n      const page = await v3.context.awaitActivePage();\n      const { pageText } = (await v3.extract()) as { pageText: string };\n      const pageUrl = page.url();\n\n      let content = pageText;\n      const MAX_TOKENS = 70000; // rough cap, assume ~4 chars per token for conservative truncation\n      const estimatedTokens = Math.ceil(content.length / 4);\n      if (estimatedTokens > MAX_TOKENS) {\n        const maxChars = MAX_TOKENS * 4;\n        content =\n          content.substring(0, maxChars) +\n          \"\\n\\n[CONTENT TRUNCATED: Exceeded 70,000 token limit]\";\n      }\n\n      return { content, pageUrl };\n    },\n    toModelOutput: (result) => ({\n      type: \"content\",\n      value: [{ type: \"text\", text: `Accessibility Tree:\\n${result.content}` }],\n    }),\n  });\n"]}
+{"version":3,"file":"ariaTree.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/ariaTree.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAExB,kEAA+D;AAExD,MAAM,YAAY,GAAG,CAAC,EAAM,EAAE,WAAoB,EAAE,EAAE,CAC3D,IAAA,SAAI,EAAC;IACH,WAAW,EACT,oHAAoH;IACtH,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC,EAAE,CAAC;IACzB,OAAO,EAAE,KAAK,IAAI,EAAE;QAClB,IAAI,CAAC;YACH,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,8BAA8B;gBACvC,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,WAAW;gBAChC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE;gBAC1B,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,CAErD,CAAC;YACF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE3B,IAAI,OAAO,GAAG,QAAQ,CAAC;YACvB,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,mEAAmE;YAC7F,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACtD,IAAI,eAAe,GAAG,UAAU,EAAE,CAAC;gBACjC,MAAM,QAAQ,GAAG,UAAU,GAAG,CAAC,CAAC;gBAChC,OAAO;oBACL,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC;wBAC9B,sDAAsD,CAAC;YAC3D,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,2BAAY,EAAE,CAAC;gBAClC,MAAM,cAAc,GAAG,gEAAgE,CAAC;gBACxF,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,cAAc;oBACvB,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBACH,OAAO;oBACL,OAAO,EAAE,EAAE;oBACX,KAAK,EAAE,cAAc;oBACrB,OAAO,EAAE,KAAK;oBACd,OAAO,EAAE,EAAE;iBACZ,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,OAAO,EAAE,EAAE;gBACX,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC;gBACtC,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,EAAE;aACZ,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE;gBACL,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,wBAAwB,MAAM,CAAC,OAAO,EAAE,EAAE;aACjE;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAtEQ,QAAA,YAAY,gBAsEpB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\nexport const ariaTreeTool = (v3: V3, toolTimeout?: number) =>\n  tool({\n    description:\n      \"gets the accessibility (ARIA) hybrid tree text for the current page. use this to understand structure and content.\",\n    inputSchema: z.object({}),\n    execute: async () => {\n      try {\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: ariaTree`,\n          level: 1,\n        });\n        const page = await v3.context.awaitActivePage();\n        const extractOptions = toolTimeout\n          ? { timeout: toolTimeout }\n          : undefined;\n        const { pageText } = (await v3.extract(extractOptions)) as {\n          pageText: string;\n        };\n        const pageUrl = page.url();\n\n        let content = pageText;\n        const MAX_TOKENS = 70000; // rough cap, assume ~4 chars per token for conservative truncation\n        const estimatedTokens = Math.ceil(content.length / 4);\n        if (estimatedTokens > MAX_TOKENS) {\n          const maxChars = MAX_TOKENS * 4;\n          content =\n            content.substring(0, maxChars) +\n            \"\\n\\n[CONTENT TRUNCATED: Exceeded 70,000 token limit]\";\n        }\n\n        return { success: true, content, pageUrl };\n      } catch (error) {\n        if (error instanceof TimeoutError) {\n          const timeoutMessage = `TimeoutError: ariaTree() timed out — the page may be too large`;\n          v3.logger({\n            category: \"agent\",\n            message: timeoutMessage,\n            level: 0,\n          });\n          return {\n            content: \"\",\n            error: timeoutMessage,\n            success: false,\n            pageUrl: \"\",\n          };\n        }\n        return {\n          content: \"\",\n          error: error?.message ?? String(error),\n          success: false,\n          pageUrl: \"\",\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      return {\n        type: \"content\",\n        value: [\n          { type: \"text\", text: `Accessibility Tree:\\n${result.content}` },\n        ],\n      };\n    },\n  });\n"]}

package/dist/cjs/lib/v3/agent/tools/click.js

@@ -70,38 +70,30 @@ const clickTool = (v3, provider) => (0, ai_1.tool)({
         }
     },
     toModelOutput: (result) => {
-        if (result.success) {
-            const content = [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        describe: result.describe,
-                        coordinates: result.coordinates,
-                    }),
-                },
-            ];
-            if (result.screenshotBase64) {
-                content.push({
-                    type: "media",
-                    mediaType: "image/png",
-                    data: result.screenshotBase64,
-                });
-            }
-            return { type: "content", value: content };
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
         }
-        return {
-            type: "content",
-            value: [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        error: result.error,
-                    }),
-                },
-            ],
-        };
+        const content = [
+            {
+                type: "text",
+                text: JSON.stringify({
+                    success: result.success,
+                    describe: result.describe,
+                    coordinates: result.coordinates,
+                }),
+            },
+        ];
+        if (result.screenshotBase64) {
+            content.push({
+                type: "media",
+                mediaType: "image/png",
+                data: result.screenshotBase64,
+            });
+        }
+        return { type: "content", value: content };
     },
 });
 exports.clickTool = clickTool;

package/dist/cjs/lib/v3/agent/tools/click.js.map

@@ -1 +1 @@
-{"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAOxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AAElE,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAA,SAAI,EAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,OAAC;aACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;gBACvD,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,OAAO;wBACf,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;qBAChC,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,KAAK,EAAE,MAAM,CAAC,KAAK;qBACpB,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AA9GQ,QAAA,SAAS,aA8GjB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  ClickToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: click`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const xpath = await page.click(processed.x, processed.y, {\n          returnXpath: shouldCollectXpath,\n        });\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"click\",\n              arguments: [],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          coordinates: [processed.x, processed.y],\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success) {\n        const content: ModelOutputContentItem[] = [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              describe: result.describe,\n              coordinates: result.coordinates,\n            }),\n          },\n        ];\n        if (result.screenshotBase64) {\n          content.push({\n            type: \"media\",\n            mediaType: \"image/png\",\n            data: result.screenshotBase64,\n          });\n        }\n        return { type: \"content\", value: content };\n      }\n      return {\n        type: \"content\",\n        value: [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              error: result.error,\n            }),\n          },\n        ],\n      };\n    },\n  });\n"]}
+{"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAOxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AAElE,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAA,SAAI,EAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,OAAC;aACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;gBACvD,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,OAAO;wBACf,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC;AAvGQ,QAAA,SAAS,aAuGjB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  ClickToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: click`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const xpath = await page.click(processed.x, processed.y, {\n          returnXpath: shouldCollectXpath,\n        });\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"click\",\n              arguments: [],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          coordinates: [processed.x, processed.y],\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            coordinates: result.coordinates,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/cjs/lib/v3/agent/tools/dragAndDrop.js

@@ -72,37 +72,29 @@ const dragAndDropTool = (v3, provider) => (0, ai_1.tool)({
         }
     },
     toModelOutput: (result) => {
-        if (result.success) {
-            const content = [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        describe: result.describe,
-                    }),
-                },
-            ];
-            if (result.screenshotBase64) {
-                content.push({
-                    type: "media",
-                    mediaType: "image/png",
-                    data: result.screenshotBase64,
-                });
-            }
-            return { type: "content", value: content };
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
         }
-        return {
-            type: "content",
-            value: [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        error: result.error,
-                    }),
-                },
-            ],
-        };
+        const content = [
+            {
+                type: "text",
+                text: JSON.stringify({
+                    success: result.success,
+                    describe: result.describe,
+                }),
+            },
+        ];
+        if (result.screenshotBase64) {
+            content.push({
+                type: "media",
+                mediaType: "image/png",
+                data: result.screenshotBase64,
+            });
+        }
+        return { type: "content", value: content };
     },
 });
 exports.dragAndDropTool = dragAndDropTool;

package/dist/cjs/lib/v3/agent/tools/dragAndDrop.js.map

@@ -1 +1 @@
-{"version":3,"file":"dragAndDrop.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/dragAndDrop.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAOxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AAElE,MAAM,eAAe,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC3D,IAAA,SAAI,EAAC;IACH,WAAW,EACT,kNAAkN;IACpN,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACtE,gBAAgB,EAAE,OAAC;aAChB,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,wDAAwD,CAAC;QACrE,cAAc,EAAE,OAAC;aACd,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oDAAoD,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,gBAAgB,EAChB,cAAc,GACf,EAAkC,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,IAAA,+CAAkB,EACvC,gBAAgB,CAAC,CAAC,CAAC,EACnB,gBAAgB,CAAC,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;YACF,MAAM,YAAY,GAAG,IAAA,+CAAkB,EACrC,cAAc,CAAC,CAAC,CAAC,EACjB,cAAc,CAAC,CAAC,CAAC,EACjB,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,iCAAiC;gBAC1C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACjD,cAAc,CAAC,CAAC,EAChB,cAAc,CAAC,CAAC,EAChB,YAAY,CAAC,CAAC,EACd,YAAY,CAAC,CAAC,EACd,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;YAE9D,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,cAAc,GAAG,IAAA,sBAAW,EAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,IAAA,sBAAW,EAAC,OAAO,CAAC,CAAC;gBAC1C,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,cAAc;wBACxB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,aAAa;wBACrB,SAAS,EAAE,CAAC,YAAY,CAAC;qBAC1B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;qBAC1B,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,KAAK,EAAE,MAAM,CAAC,KAAK;qBACpB,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AA5HQ,QAAA,eAAe,mBA4HvB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  DragAndDropToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const dragAndDropTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z.string().describe(\"Describe the element to drag and drop\"),\n      startCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to start the drag and drop from\"),\n      endCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to end the drag and drop at\"),\n    }),\n    execute: async ({\n      describe,\n      startCoordinates,\n      endCoordinates,\n    }): Promise<DragAndDropToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processedStart = processCoordinates(\n          startCoordinates[0],\n          startCoordinates[1],\n          provider,\n          v3,\n        );\n        const processedEnd = processCoordinates(\n          endCoordinates[0],\n          endCoordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: dragAndDrop`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({\n                describe,\n              }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const [fromXpath, toXpath] = await page.dragAndDrop(\n          processedStart.x,\n          processedStart.y,\n          processedEnd.x,\n          processedEnd.y,\n          { returnXpath: shouldCollectXpath },\n        );\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedFrom = ensureXPath(fromXpath);\n          const normalizedTo = ensureXPath(toXpath);\n          if (normalizedFrom && normalizedTo) {\n            const action: Action = {\n              selector: normalizedFrom,\n              description: describe,\n              method: \"dragAndDrop\",\n              arguments: [normalizedTo],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error dragging: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success) {\n        const content: ModelOutputContentItem[] = [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              describe: result.describe,\n            }),\n          },\n        ];\n        if (result.screenshotBase64) {\n          content.push({\n            type: \"media\",\n            mediaType: \"image/png\",\n            data: result.screenshotBase64,\n          });\n        }\n        return { type: \"content\", value: content };\n      }\n      return {\n        type: \"content\",\n        value: [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              error: result.error,\n            }),\n          },\n        ],\n      };\n    },\n  });\n"]}
+{"version":3,"file":"dragAndDrop.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/dragAndDrop.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAOxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AAElE,MAAM,eAAe,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC3D,IAAA,SAAI,EAAC;IACH,WAAW,EACT,kNAAkN;IACpN,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACtE,gBAAgB,EAAE,OAAC;aAChB,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,wDAAwD,CAAC;QACrE,cAAc,EAAE,OAAC;aACd,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oDAAoD,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,gBAAgB,EAChB,cAAc,GACf,EAAkC,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,IAAA,+CAAkB,EACvC,gBAAgB,CAAC,CAAC,CAAC,EACnB,gBAAgB,CAAC,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;YACF,MAAM,YAAY,GAAG,IAAA,+CAAkB,EACrC,cAAc,CAAC,CAAC,CAAC,EACjB,cAAc,CAAC,CAAC,CAAC,EACjB,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,iCAAiC;gBAC1C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACjD,cAAc,CAAC,CAAC,EAChB,cAAc,CAAC,CAAC,EAChB,YAAY,CAAC,CAAC,EACd,YAAY,CAAC,CAAC,EACd,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;YAE9D,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,cAAc,GAAG,IAAA,sBAAW,EAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,IAAA,sBAAW,EAAC,OAAO,CAAC,CAAC;gBAC1C,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,cAAc;wBACxB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,aAAa;wBACrB,SAAS,EAAE,CAAC,YAAY,CAAC;qBAC1B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBAC1B,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC;AArHQ,QAAA,eAAe,mBAqHvB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  DragAndDropToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const dragAndDropTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z.string().describe(\"Describe the element to drag and drop\"),\n      startCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to start the drag and drop from\"),\n      endCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to end the drag and drop at\"),\n    }),\n    execute: async ({\n      describe,\n      startCoordinates,\n      endCoordinates,\n    }): Promise<DragAndDropToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processedStart = processCoordinates(\n          startCoordinates[0],\n          startCoordinates[1],\n          provider,\n          v3,\n        );\n        const processedEnd = processCoordinates(\n          endCoordinates[0],\n          endCoordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: dragAndDrop`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({\n                describe,\n              }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const [fromXpath, toXpath] = await page.dragAndDrop(\n          processedStart.x,\n          processedStart.y,\n          processedEnd.x,\n          processedEnd.y,\n          { returnXpath: shouldCollectXpath },\n        );\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedFrom = ensureXPath(fromXpath);\n          const normalizedTo = ensureXPath(toXpath);\n          if (normalizedFrom && normalizedTo) {\n            const action: Action = {\n              selector: normalizedFrom,\n              description: describe,\n              method: \"dragAndDrop\",\n              arguments: [normalizedTo],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error dragging: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}